1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import android.os.Environment;
8
9import androidx.annotation.NonNull;
10import androidx.annotation.Nullable;
11
12import com.google.common.base.Joiner;
13import com.google.common.base.Optional;
14import com.google.common.base.Preconditions;
15import com.google.common.base.Stopwatch;
16import com.google.common.base.Strings;
17import com.google.common.base.Throwables;
18import com.google.common.collect.Collections2;
19import com.google.common.collect.ImmutableMultimap;
20import com.google.common.collect.ImmutableSet;
21import com.google.common.collect.Iterables;
22import com.google.common.collect.Maps;
23import com.google.common.collect.Sets;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import eu.siacs.conversations.BuildConfig;
30import eu.siacs.conversations.Config;
31import eu.siacs.conversations.crypto.axolotl.AxolotlService;
32import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
33import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
34import eu.siacs.conversations.entities.Account;
35import eu.siacs.conversations.entities.Conversation;
36import eu.siacs.conversations.entities.Conversational;
37import eu.siacs.conversations.entities.Message;
38import eu.siacs.conversations.entities.RtpSessionStatus;
39import eu.siacs.conversations.services.CallIntegration;
40import eu.siacs.conversations.services.CallIntegrationConnectionService;
41import eu.siacs.conversations.ui.RtpSessionActivity;
42import eu.siacs.conversations.xml.Element;
43import eu.siacs.conversations.xml.Namespace;
44import eu.siacs.conversations.xmpp.Jid;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
47import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
48import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
51import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
52import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
53import eu.siacs.conversations.xmpp.stanzas.IqPacket;
54import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
55
56import org.webrtc.DtmfSender;
57import org.webrtc.EglBase;
58import org.webrtc.IceCandidate;
59import org.webrtc.PeerConnection;
60import org.webrtc.VideoTrack;
61
62import java.io.File;
63import java.io.IOException;
64import java.util.Arrays;
65import java.util.Collection;
66import java.util.Collections;
67import java.util.LinkedList;
68import java.util.List;
69import java.util.Map;
70import java.util.Queue;
71import java.util.Set;
72import java.util.concurrent.ExecutionException;
73import java.util.concurrent.ScheduledFuture;
74import java.util.concurrent.TimeUnit;
75
76public class JingleRtpConnection extends AbstractJingleConnection
77 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
78
79 // TODO consider adding State.SESSION_INITIALIZED to ongoing call states for direct init mode
80 public static final List<State> STATES_SHOWING_ONGOING_CALL =
81 Arrays.asList(
82 State.PROPOSED,
83 State.PROCEED,
84 State.SESSION_INITIALIZED_PRE_APPROVED,
85 State.SESSION_ACCEPTED);
86 private static final long BUSY_TIME_OUT = 30;
87
88 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
89 private final Queue<
90 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
91 pendingIceCandidates = new LinkedList<>();
92 private final OmemoVerification omemoVerification = new OmemoVerification();
93 public final CallIntegration callIntegration;
94 private final Message message;
95
96 private Set<Media> proposedMedia;
97 private RtpContentMap initiatorRtpContentMap;
98 private RtpContentMap responderRtpContentMap;
99 private RtpContentMap incomingContentAdd;
100 private RtpContentMap outgoingContentAdd;
101 private IceUdpTransportInfo.Setup peerDtlsSetup;
102 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
103 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
104 private ScheduledFuture<?> ringingTimeoutFuture;
105 private final long created = System.currentTimeMillis() / 1000L;
106
107 JingleRtpConnection(
108 final JingleConnectionManager jingleConnectionManager,
109 final Id id,
110 final Jid initiator) {
111 this(
112 jingleConnectionManager,
113 id,
114 initiator,
115 new CallIntegration(
116 jingleConnectionManager
117 .getXmppConnectionService()
118 .getApplicationContext()));
119 this.callIntegration.setAddress(
120 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
121 this.callIntegration.setInitialized();
122 }
123
124 JingleRtpConnection(
125 final JingleConnectionManager jingleConnectionManager,
126 final Id id,
127 final Jid initiator,
128 final CallIntegration callIntegration) {
129 super(jingleConnectionManager, id, initiator);
130 final Conversation conversation =
131 jingleConnectionManager
132 .getXmppConnectionService()
133 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
134 this.message =
135 new Message(
136 conversation,
137 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
138 Message.TYPE_RTP_SESSION,
139 id.sessionId);
140 this.callIntegration = callIntegration;
141 this.callIntegration.setCallback(this);
142 }
143
144 @Override
145 synchronized void deliverPacket(final JinglePacket jinglePacket) {
146 switch (jinglePacket.getAction()) {
147 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
148 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
149 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
150 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
151 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
152 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
153 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
154 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
155 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
156 default -> {
157 respondOk(jinglePacket);
158 Log.d(
159 Config.LOGTAG,
160 String.format(
161 "%s: received unhandled jingle action %s",
162 id.account.getJid().asBareJid(), jinglePacket.getAction()));
163 }
164 }
165 }
166
167 @Override
168 synchronized void notifyRebound() {
169 if (isTerminated()) {
170 return;
171 }
172 webRTCWrapper.close();
173 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
174 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
175 }
176 if (isInState(
177 State.SESSION_INITIALIZED,
178 State.SESSION_INITIALIZED_PRE_APPROVED,
179 State.SESSION_ACCEPTED)) {
180 // we might have already changed resources (full jid) at this point; so this might not
181 // even reach the other party
182 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
183 } else {
184 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
185 finish();
186 }
187 }
188
189 public boolean applyDtmfTone(final String tone) {
190 return webRTCWrapper.applyDtmfTone(tone);
191 }
192
193 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
194 respondOk(jinglePacket);
195 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
196 final State previous = this.state;
197 Log.d(
198 Config.LOGTAG,
199 id.account.getJid().asBareJid()
200 + ": received session terminate reason="
201 + wrapper.reason
202 + "("
203 + Strings.nullToEmpty(wrapper.text)
204 + ") while in state "
205 + previous);
206 if (TERMINATED.contains(previous)) {
207 Log.d(
208 Config.LOGTAG,
209 id.account.getJid().asBareJid()
210 + ": ignoring session terminate because already in "
211 + previous);
212 return;
213 }
214 webRTCWrapper.close();
215 final State target = reasonToState(wrapper.reason);
216 transitionOrThrow(target);
217 writeLogMessage(target);
218 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
219 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
220 }
221 finish();
222 }
223
224 private void receiveTransportInfo(final JinglePacket jinglePacket) {
225 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
226 // INITIALIZED only after transport-info has been received
227 if (isInState(
228 State.NULL,
229 State.PROCEED,
230 State.SESSION_INITIALIZED,
231 State.SESSION_INITIALIZED_PRE_APPROVED,
232 State.SESSION_ACCEPTED)) {
233 final RtpContentMap contentMap;
234 try {
235 contentMap = RtpContentMap.of(jinglePacket);
236 } catch (final IllegalArgumentException | NullPointerException e) {
237 Log.d(
238 Config.LOGTAG,
239 id.account.getJid().asBareJid()
240 + ": improperly formatted contents; ignoring",
241 e);
242 respondOk(jinglePacket);
243 return;
244 }
245 receiveTransportInfo(jinglePacket, contentMap);
246 } else {
247 if (isTerminated()) {
248 respondOk(jinglePacket);
249 Log.d(
250 Config.LOGTAG,
251 id.account.getJid().asBareJid()
252 + ": ignoring out-of-order transport info; we where already terminated");
253 } else {
254 Log.d(
255 Config.LOGTAG,
256 id.account.getJid().asBareJid()
257 + ": received transport info while in state="
258 + this.state);
259 terminateWithOutOfOrder(jinglePacket);
260 }
261 }
262 }
263
264 private void receiveTransportInfo(
265 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
266 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
267 candidates = contentMap.contents.entrySet();
268 final RtpContentMap remote = getRemoteContentMap();
269 final Set<String> remoteContentIds =
270 remote == null ? Collections.emptySet() : remote.contents.keySet();
271 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
272 Log.d(
273 Config.LOGTAG,
274 "received transport-info for unknown contents "
275 + contentMap.contents.keySet()
276 + " (known: "
277 + remoteContentIds
278 + ")");
279 respondOk(jinglePacket);
280 pendingIceCandidates.addAll(candidates);
281 return;
282 }
283 if (this.state != State.SESSION_ACCEPTED) {
284 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
285 respondOk(jinglePacket);
286 pendingIceCandidates.addAll(candidates);
287 return;
288 }
289 // zero candidates + modified credentials are an ICE restart offer
290 if (checkForIceRestart(jinglePacket, contentMap)) {
291 return;
292 }
293 respondOk(jinglePacket);
294 try {
295 processCandidates(candidates);
296 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
297 Log.w(
298 Config.LOGTAG,
299 id.account.getJid().asBareJid()
300 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
301 }
302 }
303
304 private void receiveContentAdd(final JinglePacket jinglePacket) {
305 final RtpContentMap modification;
306 try {
307 modification = RtpContentMap.of(jinglePacket);
308 modification.requireContentDescriptions();
309 } catch (final RuntimeException e) {
310 Log.d(
311 Config.LOGTAG,
312 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
313 Throwables.getRootCause(e));
314 respondOk(jinglePacket);
315 webRTCWrapper.close();
316 sendSessionTerminate(Reason.of(e), e.getMessage());
317 return;
318 }
319 if (isInState(State.SESSION_ACCEPTED)) {
320 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
321 final ListenableFuture<RtpContentMap> future =
322 receiveRtpContentMap(
323 modification,
324 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
325 Futures.addCallback(
326 future,
327 new FutureCallback<>() {
328 @Override
329 public void onSuccess(final RtpContentMap rtpContentMap) {
330 receiveContentAdd(jinglePacket, rtpContentMap);
331 }
332
333 @Override
334 public void onFailure(@NonNull Throwable throwable) {
335 respondOk(jinglePacket);
336 final Throwable rootCause = Throwables.getRootCause(throwable);
337 Log.d(
338 Config.LOGTAG,
339 id.account.getJid().asBareJid()
340 + ": improperly formatted contents in content-add",
341 throwable);
342 webRTCWrapper.close();
343 sendSessionTerminate(
344 Reason.ofThrowable(rootCause), rootCause.getMessage());
345 }
346 },
347 MoreExecutors.directExecutor());
348 } else {
349 terminateWithOutOfOrder(jinglePacket);
350 }
351 }
352
353 private void receiveContentAdd(
354 final JinglePacket jinglePacket, final RtpContentMap modification) {
355 final RtpContentMap remote = getRemoteContentMap();
356 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
357 respondOk(jinglePacket);
358 this.webRTCWrapper.close();
359 sendSessionTerminate(
360 Reason.FAILED_APPLICATION,
361 String.format(
362 "contents with names %s already exists",
363 Joiner.on(", ").join(modification.getNames())));
364 return;
365 }
366 final ContentAddition contentAddition =
367 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
368
369 final RtpContentMap outgoing = this.outgoingContentAdd;
370 final Set<ContentAddition.Summary> outgoingContentAddSummary =
371 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
372
373 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
374 if (isInitiator()) {
375 Log.d(
376 Config.LOGTAG,
377 id.getAccount().getJid().asBareJid()
378 + ": respond with tie break to matching content-add offer");
379 respondWithTieBreak(jinglePacket);
380 } else {
381 Log.d(
382 Config.LOGTAG,
383 id.getAccount().getJid().asBareJid()
384 + ": automatically accept matching content-add offer");
385 acceptContentAdd(contentAddition.summary, modification);
386 }
387 return;
388 }
389
390 // once we can display multiple video tracks we can be more loose with this condition
391 // theoretically it should also be fine to automatically accept audio only contents
392 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
393 Log.d(
394 Config.LOGTAG,
395 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
396 this.incomingContentAdd = modification;
397 respondOk(jinglePacket);
398 updateEndUserState();
399 } else {
400 respondOk(jinglePacket);
401 // TODO do we want to add a reason?
402 rejectContentAdd(modification);
403 }
404 }
405
406 private void receiveContentAccept(final JinglePacket jinglePacket) {
407 final RtpContentMap receivedContentAccept;
408 try {
409 receivedContentAccept = RtpContentMap.of(jinglePacket);
410 receivedContentAccept.requireContentDescriptions();
411 } catch (final RuntimeException e) {
412 Log.d(
413 Config.LOGTAG,
414 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
415 Throwables.getRootCause(e));
416 respondOk(jinglePacket);
417 webRTCWrapper.close();
418 sendSessionTerminate(Reason.of(e), e.getMessage());
419 return;
420 }
421
422 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
423 if (outgoingContentAdd == null) {
424 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
425 terminateWithOutOfOrder(jinglePacket);
426 return;
427 }
428 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
429 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
430 this.outgoingContentAdd = null;
431 respondOk(jinglePacket);
432 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
433 final ListenableFuture<RtpContentMap> future =
434 receiveRtpContentMap(
435 receivedContentAccept,
436 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
437 Futures.addCallback(
438 future,
439 new FutureCallback<>() {
440 @Override
441 public void onSuccess(final RtpContentMap result) {
442 receiveContentAccept(result);
443 }
444
445 @Override
446 public void onFailure(@NonNull final Throwable throwable) {
447 webRTCWrapper.close();
448 sendSessionTerminate(
449 Reason.ofThrowable(throwable), throwable.getMessage());
450 }
451 },
452 MoreExecutors.directExecutor());
453 } else {
454 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
455 terminateWithOutOfOrder(jinglePacket);
456 }
457 }
458
459 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
460 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
461 final RtpContentMap modifiedContentMap =
462 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
463
464 setRemoteContentMap(modifiedContentMap);
465
466 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
467
468 final org.webrtc.SessionDescription sdp =
469 new org.webrtc.SessionDescription(
470 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
471
472 try {
473 this.webRTCWrapper.setRemoteDescription(sdp).get();
474 } catch (final Exception e) {
475 final Throwable cause = Throwables.getRootCause(e);
476 Log.d(
477 Config.LOGTAG,
478 id.getAccount().getJid().asBareJid()
479 + ": unable to set remote description after receiving content-accept",
480 cause);
481 webRTCWrapper.close();
482 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
483 return;
484 }
485 Log.d(
486 Config.LOGTAG,
487 id.getAccount().getJid().asBareJid()
488 + ": remote has accepted content-add "
489 + ContentAddition.summary(receivedContentAccept));
490 processCandidates(receivedContentAccept.contents.entrySet());
491 updateEndUserState();
492 }
493
494 private void receiveContentModify(final JinglePacket jinglePacket) {
495 if (this.state != State.SESSION_ACCEPTED) {
496 terminateWithOutOfOrder(jinglePacket);
497 return;
498 }
499 final Map<String, Content.Senders> modification =
500 Maps.transformEntries(
501 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
502 final boolean isInitiator = isInitiator();
503 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
504 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
505 final Set<String> currentOutgoingMediaIds =
506 currentOutgoing == null
507 ? Collections.emptySet()
508 : currentOutgoing.contents.keySet();
509 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
510 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
511 respondOk(jinglePacket);
512 final RtpContentMap modifiedContentMap;
513 try {
514 modifiedContentMap =
515 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
516 } catch (final IllegalArgumentException e) {
517 webRTCWrapper.close();
518 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
519 return;
520 }
521 this.outgoingContentAdd = modifiedContentMap;
522 Log.d(
523 Config.LOGTAG,
524 id.account.getJid().asBareJid()
525 + ": processed content-modification for pending content-add");
526 } else if (remoteContentMap != null
527 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
528 respondOk(jinglePacket);
529 final RtpContentMap modifiedRemoteContentMap;
530 try {
531 modifiedRemoteContentMap =
532 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
533 } catch (final IllegalArgumentException e) {
534 webRTCWrapper.close();
535 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
536 return;
537 }
538 final SessionDescription offer;
539 try {
540 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
541 } catch (final IllegalArgumentException | NullPointerException e) {
542 Log.d(
543 Config.LOGTAG,
544 id.getAccount().getJid().asBareJid()
545 + ": unable convert offer from content-modify to SDP",
546 e);
547 webRTCWrapper.close();
548 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
549 return;
550 }
551 Log.d(
552 Config.LOGTAG,
553 id.account.getJid().asBareJid() + ": auto accepting content-modification");
554 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
555 } else {
556 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
557 respondWithItemNotFound(jinglePacket);
558 }
559 }
560
561 private void autoAcceptContentModify(
562 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
563 this.setRemoteContentMap(modifiedRemoteContentMap);
564 final org.webrtc.SessionDescription sdp =
565 new org.webrtc.SessionDescription(
566 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
567 try {
568 this.webRTCWrapper.setRemoteDescription(sdp).get();
569 // auto accept is only done when we already have tracks
570 final SessionDescription answer = setLocalSessionDescription();
571 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
572 modifyLocalContentMap(rtpContentMap);
573 // we do not need to send an answer but do we have to resend the candidates currently in
574 // SDP?
575 // resendCandidatesFromSdp(answer);
576 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
577 } catch (final Exception e) {
578 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
579 webRTCWrapper.close();
580 sendSessionTerminate(Reason.FAILED_APPLICATION);
581 }
582 }
583
584 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
585 final SessionDescription answer) {
586 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
587 new ImmutableMultimap.Builder<>();
588 for (final SessionDescription.Media media : answer.media) {
589 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
590 if (Strings.isNullOrEmpty(mid)) {
591 continue;
592 }
593 for (final String sdpCandidate : media.attributes.get("candidate")) {
594 final IceUdpTransportInfo.Candidate candidate =
595 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
596 if (candidate != null) {
597 candidateBuilder.put(mid, candidate);
598 }
599 }
600 }
601 return candidateBuilder.build();
602 }
603
604 private void receiveContentReject(final JinglePacket jinglePacket) {
605 final RtpContentMap receivedContentReject;
606 try {
607 receivedContentReject = RtpContentMap.of(jinglePacket);
608 } catch (final RuntimeException e) {
609 Log.d(
610 Config.LOGTAG,
611 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
612 Throwables.getRootCause(e));
613 respondOk(jinglePacket);
614 this.webRTCWrapper.close();
615 sendSessionTerminate(Reason.of(e), e.getMessage());
616 return;
617 }
618
619 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
620 if (outgoingContentAdd == null) {
621 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
622 terminateWithOutOfOrder(jinglePacket);
623 return;
624 }
625 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
626 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
627 this.outgoingContentAdd = null;
628 respondOk(jinglePacket);
629 Log.d(Config.LOGTAG, jinglePacket.toString());
630 receiveContentReject(ourSummary);
631 } else {
632 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
633 terminateWithOutOfOrder(jinglePacket);
634 }
635 }
636
637 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
638 try {
639 this.webRTCWrapper.removeTrack(Media.VIDEO);
640 final RtpContentMap localContentMap = customRollback();
641 modifyLocalContentMap(localContentMap);
642 } catch (final Exception e) {
643 final Throwable cause = Throwables.getRootCause(e);
644 Log.d(
645 Config.LOGTAG,
646 id.getAccount().getJid().asBareJid()
647 + ": unable to rollback local description after receiving content-reject",
648 cause);
649 webRTCWrapper.close();
650 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
651 return;
652 }
653 Log.d(
654 Config.LOGTAG,
655 id.getAccount().getJid().asBareJid()
656 + ": remote has rejected our content-add "
657 + summary);
658 }
659
660 private void receiveContentRemove(final JinglePacket jinglePacket) {
661 final RtpContentMap receivedContentRemove;
662 try {
663 receivedContentRemove = RtpContentMap.of(jinglePacket);
664 receivedContentRemove.requireContentDescriptions();
665 } catch (final RuntimeException e) {
666 Log.d(
667 Config.LOGTAG,
668 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
669 Throwables.getRootCause(e));
670 respondOk(jinglePacket);
671 this.webRTCWrapper.close();
672 sendSessionTerminate(Reason.of(e), e.getMessage());
673 return;
674 }
675 respondOk(jinglePacket);
676 receiveContentRemove(receivedContentRemove);
677 }
678
679 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
680 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
681 final Set<ContentAddition.Summary> contentAddSummary =
682 incomingContentAdd == null
683 ? Collections.emptySet()
684 : ContentAddition.summary(incomingContentAdd);
685 final Set<ContentAddition.Summary> removeSummary =
686 ContentAddition.summary(receivedContentRemove);
687 if (contentAddSummary.equals(removeSummary)) {
688 this.incomingContentAdd = null;
689 updateEndUserState();
690 } else {
691 webRTCWrapper.close();
692 sendSessionTerminate(
693 Reason.FAILED_APPLICATION,
694 String.format(
695 "%s only supports %s as a means to retract a not yet accepted %s",
696 BuildConfig.APP_NAME,
697 JinglePacket.Action.CONTENT_REMOVE,
698 JinglePacket.Action.CONTENT_ADD));
699 }
700 }
701
702 public synchronized void retractContentAdd() {
703 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
704 if (outgoingContentAdd == null) {
705 throw new IllegalStateException("Not outgoing content add");
706 }
707 try {
708 webRTCWrapper.removeTrack(Media.VIDEO);
709 final RtpContentMap localContentMap = customRollback();
710 modifyLocalContentMap(localContentMap);
711 } catch (final Exception e) {
712 final Throwable cause = Throwables.getRootCause(e);
713 Log.d(
714 Config.LOGTAG,
715 id.getAccount().getJid().asBareJid()
716 + ": unable to rollback local description after trying to retract content-add",
717 cause);
718 webRTCWrapper.close();
719 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
720 return;
721 }
722 this.outgoingContentAdd = null;
723 final JinglePacket retract =
724 outgoingContentAdd
725 .toStub()
726 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
727 this.send(retract);
728 Log.d(
729 Config.LOGTAG,
730 id.getAccount().getJid()
731 + ": retract content-add "
732 + ContentAddition.summary(outgoingContentAdd));
733 }
734
735 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
736 final SessionDescription sdp = setLocalSessionDescription();
737 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
738 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
739 this.webRTCWrapper
740 .setRemoteDescription(
741 new org.webrtc.SessionDescription(
742 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
743 .get();
744 return localRtpContentMap;
745 }
746
747 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
748 final RtpContentMap currentRemote = getRemoteContentMap();
749 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
750 if (diff.isEmpty()) {
751 throw new IllegalStateException(
752 "Unexpected rollback condition. No difference between local and remote");
753 }
754 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
755 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
756 final RtpContentMap nextRemote =
757 currentRemote.addContent(
758 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
759 return SessionDescription.of(nextRemote, isResponder());
760 }
761 throw new IllegalStateException(
762 "Unexpected rollback condition. Senders were not uniformly none");
763 }
764
765 public synchronized void acceptContentAdd(
766 @NonNull final Set<ContentAddition.Summary> contentAddition) {
767 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
768 if (incomingContentAdd == null) {
769 throw new IllegalStateException("No incoming content add");
770 }
771
772 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
773 this.incomingContentAdd = null;
774 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
775 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
776 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
777 Log.d(
778 Config.LOGTAG,
779 "content addition is receive only. we want to upgrade to 'both'");
780 final RtpContentMap modifiedSenders =
781 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
782 final JinglePacket proposedContentModification =
783 modifiedSenders
784 .toStub()
785 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
786 proposedContentModification.setTo(id.with);
787 xmppConnectionService.sendIqPacket(
788 id.account,
789 proposedContentModification,
790 (account, response) -> {
791 if (response.getType() == IqPacket.TYPE.RESULT) {
792 Log.d(
793 Config.LOGTAG,
794 id.account.getJid().asBareJid()
795 + ": remote has accepted our upgrade to senders=both");
796 acceptContentAdd(
797 ContentAddition.summary(modifiedSenders), modifiedSenders);
798 } else {
799 Log.d(
800 Config.LOGTAG,
801 id.account.getJid().asBareJid()
802 + ": remote has rejected our upgrade to senders=both");
803 acceptContentAdd(contentAddition, incomingContentAdd);
804 }
805 });
806 } else {
807 acceptContentAdd(contentAddition, incomingContentAdd);
808 }
809 } else {
810 throw new IllegalStateException(
811 "Accepted content add does not match pending content-add");
812 }
813 }
814
815 private void acceptContentAdd(
816 @NonNull final Set<ContentAddition.Summary> contentAddition,
817 final RtpContentMap incomingContentAdd) {
818 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
819 final RtpContentMap modifiedContentMap =
820 getRemoteContentMap().addContent(incomingContentAdd, setup);
821 this.setRemoteContentMap(modifiedContentMap);
822
823 final SessionDescription offer;
824 try {
825 offer = SessionDescription.of(modifiedContentMap, isResponder());
826 } catch (final IllegalArgumentException | NullPointerException e) {
827 Log.d(
828 Config.LOGTAG,
829 id.getAccount().getJid().asBareJid()
830 + ": unable convert offer from content-add to SDP",
831 e);
832 webRTCWrapper.close();
833 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
834 return;
835 }
836 this.incomingContentAdd = null;
837 acceptContentAdd(contentAddition, offer);
838 }
839
840 private void acceptContentAdd(
841 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
842 final org.webrtc.SessionDescription sdp =
843 new org.webrtc.SessionDescription(
844 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
845 try {
846 this.webRTCWrapper.setRemoteDescription(sdp).get();
847
848 // TODO add tracks for 'media' where contentAddition.senders matches
849
850 // TODO if senders.sending(isInitiator())
851
852 this.webRTCWrapper.addTrack(Media.VIDEO);
853
854 // TODO add additional transceivers for recv only cases
855
856 final SessionDescription answer = setLocalSessionDescription();
857 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
858
859 final RtpContentMap contentAcceptMap =
860 rtpContentMap.toContentModification(
861 Collections2.transform(contentAddition, ca -> ca.name));
862
863 Log.d(
864 Config.LOGTAG,
865 id.getAccount().getJid().asBareJid()
866 + ": sending content-accept "
867 + ContentAddition.summary(contentAcceptMap));
868
869 addIceCandidatesFromBlackLog();
870
871 modifyLocalContentMap(rtpContentMap);
872 final ListenableFuture<RtpContentMap> future =
873 prepareOutgoingContentMap(contentAcceptMap);
874 Futures.addCallback(
875 future,
876 new FutureCallback<>() {
877 @Override
878 public void onSuccess(final RtpContentMap rtpContentMap) {
879 sendContentAccept(rtpContentMap);
880 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
881 }
882
883 @Override
884 public void onFailure(@NonNull final Throwable throwable) {
885 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
886 }
887 },
888 MoreExecutors.directExecutor());
889 } catch (final Exception e) {
890 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
891 webRTCWrapper.close();
892 sendSessionTerminate(Reason.FAILED_APPLICATION);
893 }
894 }
895
896 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
897 final JinglePacket jinglePacket =
898 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
899 send(jinglePacket);
900 }
901
902 public synchronized void rejectContentAdd() {
903 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
904 if (incomingContentAdd == null) {
905 throw new IllegalStateException("No incoming content add");
906 }
907 this.incomingContentAdd = null;
908 updateEndUserState();
909 rejectContentAdd(incomingContentAdd);
910 }
911
912 private void rejectContentAdd(final RtpContentMap contentMap) {
913 final JinglePacket jinglePacket =
914 contentMap
915 .toStub()
916 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
917 Log.d(
918 Config.LOGTAG,
919 id.getAccount().getJid().asBareJid()
920 + ": rejecting content "
921 + ContentAddition.summary(contentMap));
922 send(jinglePacket);
923 }
924
925 private boolean checkForIceRestart(
926 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
927 final RtpContentMap existing = getRemoteContentMap();
928 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
929 final IceUdpTransportInfo.Credentials newCredentials;
930 try {
931 existingCredentials = existing.getCredentials();
932 newCredentials = rtpContentMap.getDistinctCredentials();
933 } catch (final IllegalStateException e) {
934 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
935 return false;
936 }
937 if (existingCredentials.contains(newCredentials)) {
938 return false;
939 }
940 // TODO an alternative approach is to check if we already got an iq result to our
941 // ICE-restart
942 // and if that's the case we are seeing an answer.
943 // This might be more spec compliant but also more error prone potentially
944 final boolean isSignalStateStable =
945 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
946 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
947 final boolean isOffer = rtpContentMap.emptyCandidates();
948 final RtpContentMap restartContentMap;
949 try {
950 if (isOffer) {
951 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
952 restartContentMap =
953 existing.modifiedCredentials(
954 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
955 } else {
956 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
957 Log.d(
958 Config.LOGTAG,
959 "received confirmation of ICE restart"
960 + newCredentials
961 + " peer_setup="
962 + setup);
963 // DTLS setup attribute needs to be rewritten to reflect current peer state
964 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
965 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
966 }
967 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
968 return isOffer;
969 } else {
970 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
971 respondWithTieBreak(jinglePacket);
972 return true;
973 }
974 } catch (final Exception exception) {
975 respondOk(jinglePacket);
976 final Throwable rootCause = Throwables.getRootCause(exception);
977 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
978 // If this happens a termination is already in progress
979 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
980 return true;
981 }
982 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
983 webRTCWrapper.close();
984 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
985 return true;
986 }
987 }
988
989 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
990 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
991 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
992 throw new IllegalStateException("Invalid peer setup");
993 }
994 return peerSetup;
995 }
996
997 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
998 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
999 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
1000 }
1001 this.peerDtlsSetup = setup;
1002 }
1003
1004 private boolean applyIceRestart(
1005 final JinglePacket jinglePacket,
1006 final RtpContentMap restartContentMap,
1007 final boolean isOffer)
1008 throws ExecutionException, InterruptedException {
1009 final SessionDescription sessionDescription =
1010 SessionDescription.of(restartContentMap, isResponder());
1011 final org.webrtc.SessionDescription.Type type =
1012 isOffer
1013 ? org.webrtc.SessionDescription.Type.OFFER
1014 : org.webrtc.SessionDescription.Type.ANSWER;
1015 org.webrtc.SessionDescription sdp =
1016 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1017 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1018 if (isInitiator()) {
1019 // We ignore the offer and respond with tie-break. This will clause the responder
1020 // not to apply the content map
1021 return false;
1022 }
1023 }
1024 webRTCWrapper.setRemoteDescription(sdp).get();
1025 setRemoteContentMap(restartContentMap);
1026 if (isOffer) {
1027 final SessionDescription localSessionDescription = setLocalSessionDescription();
1028 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1029 // We need to respond OK before sending any candidates
1030 respondOk(jinglePacket);
1031 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1032 } else {
1033 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1034 }
1035 return true;
1036 }
1037
1038 private void processCandidates(
1039 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1040 contents) {
1041 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1042 content : contents) {
1043 processCandidate(content);
1044 }
1045 }
1046
1047 private void processCandidate(
1048 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1049 content) {
1050 final RtpContentMap rtpContentMap = getRemoteContentMap();
1051 final List<String> indices = toIdentificationTags(rtpContentMap);
1052 final String sdpMid = content.getKey(); // aka content name
1053 final IceUdpTransportInfo transport = content.getValue().transport;
1054 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1055
1056 // TODO check that credentials remained the same
1057
1058 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1059 final String sdp;
1060 try {
1061 sdp = candidate.toSdpAttribute(credentials.ufrag);
1062 } catch (final IllegalArgumentException e) {
1063 Log.d(
1064 Config.LOGTAG,
1065 id.account.getJid().asBareJid()
1066 + ": ignoring invalid ICE candidate "
1067 + e.getMessage());
1068 continue;
1069 }
1070 final int mLineIndex = indices.indexOf(sdpMid);
1071 if (mLineIndex < 0) {
1072 Log.w(
1073 Config.LOGTAG,
1074 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1075 }
1076 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1077 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1078 this.webRTCWrapper.addIceCandidate(iceCandidate);
1079 }
1080 }
1081
1082 private RtpContentMap getRemoteContentMap() {
1083 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1084 }
1085
1086 private RtpContentMap getLocalContentMap() {
1087 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1088 }
1089
1090 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1091 final Group originalGroup = rtpContentMap.group;
1092 final List<String> identificationTags =
1093 originalGroup == null
1094 ? rtpContentMap.getNames()
1095 : originalGroup.getIdentificationTags();
1096 if (identificationTags.size() == 0) {
1097 Log.w(
1098 Config.LOGTAG,
1099 id.account.getJid().asBareJid()
1100 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1101 }
1102 return identificationTags;
1103 }
1104
1105 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1106 final JinglePacket jinglePacket, final boolean expectVerification) {
1107 try {
1108 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1109 } catch (final Exception e) {
1110 return Futures.immediateFailedFuture(e);
1111 }
1112 }
1113
1114 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1115 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1116 Log.d(
1117 Config.LOGTAG,
1118 "receiveRtpContentMap("
1119 + receivedContentMap.getClass().getSimpleName()
1120 + ",expectVerification="
1121 + expectVerification
1122 + ")");
1123 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1124 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1125 id.account
1126 .getAxolotlService()
1127 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1128 return Futures.transform(
1129 future,
1130 omemoVerifiedPayload -> {
1131 // TODO test if an exception here triggers a correct abort
1132 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1133 Log.d(
1134 Config.LOGTAG,
1135 id.account.getJid().asBareJid()
1136 + ": received verifiable DTLS fingerprint via "
1137 + omemoVerification);
1138 return omemoVerifiedPayload.getPayload();
1139 },
1140 MoreExecutors.directExecutor());
1141 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1142 return Futures.immediateFailedFuture(
1143 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1144 } else {
1145 return Futures.immediateFuture(receivedContentMap);
1146 }
1147 }
1148
1149 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1150 if (isInitiator()) {
1151 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1152 return;
1153 }
1154 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1155 Futures.addCallback(
1156 future,
1157 new FutureCallback<>() {
1158 @Override
1159 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1160 receiveSessionInitiate(jinglePacket, rtpContentMap);
1161 }
1162
1163 @Override
1164 public void onFailure(@NonNull final Throwable throwable) {
1165 respondOk(jinglePacket);
1166 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1167 }
1168 },
1169 MoreExecutors.directExecutor());
1170 }
1171
1172 private void receiveSessionInitiate(
1173 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1174 try {
1175 contentMap.requireContentDescriptions();
1176 contentMap.requireDTLSFingerprint(true);
1177 } catch (final RuntimeException e) {
1178 Log.d(
1179 Config.LOGTAG,
1180 id.account.getJid().asBareJid() + ": improperly formatted contents",
1181 Throwables.getRootCause(e));
1182 respondOk(jinglePacket);
1183 sendSessionTerminate(Reason.of(e), e.getMessage());
1184 return;
1185 }
1186 Log.d(
1187 Config.LOGTAG,
1188 "processing session-init with " + contentMap.contents.size() + " contents");
1189 final State target;
1190 if (this.state == State.PROCEED) {
1191 Preconditions.checkState(
1192 proposedMedia != null && proposedMedia.size() > 0,
1193 "proposed media must be set when processing pre-approved session-initiate");
1194 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1195 sendSessionTerminate(
1196 Reason.SECURITY_ERROR,
1197 String.format(
1198 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1199 this.proposedMedia, contentMap.getMedia()));
1200 return;
1201 }
1202 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1203 } else {
1204 target = State.SESSION_INITIALIZED;
1205 setProposedMedia(contentMap.getMedia());
1206 }
1207 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1208 respondOk(jinglePacket);
1209 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1210 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1211 Log.d(
1212 Config.LOGTAG,
1213 id.account.getJid().asBareJid()
1214 + ": automatically accepting session-initiate");
1215 sendSessionAccept();
1216 } else {
1217 Log.d(
1218 Config.LOGTAG,
1219 id.account.getJid().asBareJid()
1220 + ": received not pre-approved session-initiate. start ringing");
1221 startRinging();
1222 }
1223 } else {
1224 Log.d(
1225 Config.LOGTAG,
1226 String.format(
1227 "%s: received session-initiate while in state %s",
1228 id.account.getJid().asBareJid(), state));
1229 terminateWithOutOfOrder(jinglePacket);
1230 }
1231 }
1232
1233 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1234 if (isResponder()) {
1235 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1236 return;
1237 }
1238 final ListenableFuture<RtpContentMap> future =
1239 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1240 Futures.addCallback(
1241 future,
1242 new FutureCallback<>() {
1243 @Override
1244 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1245 receiveSessionAccept(jinglePacket, rtpContentMap);
1246 }
1247
1248 @Override
1249 public void onFailure(@NonNull final Throwable throwable) {
1250 respondOk(jinglePacket);
1251 Log.d(
1252 Config.LOGTAG,
1253 id.account.getJid().asBareJid()
1254 + ": improperly formatted contents in session-accept",
1255 throwable);
1256 webRTCWrapper.close();
1257 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1258 }
1259 },
1260 MoreExecutors.directExecutor());
1261 }
1262
1263 private void receiveSessionAccept(
1264 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1265 try {
1266 contentMap.requireContentDescriptions();
1267 contentMap.requireDTLSFingerprint();
1268 } catch (final RuntimeException e) {
1269 respondOk(jinglePacket);
1270 Log.d(
1271 Config.LOGTAG,
1272 id.account.getJid().asBareJid()
1273 + ": improperly formatted contents in session-accept",
1274 e);
1275 webRTCWrapper.close();
1276 sendSessionTerminate(Reason.of(e), e.getMessage());
1277 return;
1278 }
1279 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1280 if (!initiatorMedia.equals(contentMap.getMedia())) {
1281 sendSessionTerminate(
1282 Reason.SECURITY_ERROR,
1283 String.format(
1284 "Your session-included included media %s but our session-initiate was %s",
1285 this.proposedMedia, contentMap.getMedia()));
1286 return;
1287 }
1288 Log.d(
1289 Config.LOGTAG,
1290 "processing session-accept with " + contentMap.contents.size() + " contents");
1291 if (transition(State.SESSION_ACCEPTED)) {
1292 respondOk(jinglePacket);
1293 receiveSessionAccept(contentMap);
1294 } else {
1295 Log.d(
1296 Config.LOGTAG,
1297 String.format(
1298 "%s: received session-accept while in state %s",
1299 id.account.getJid().asBareJid(), state));
1300 respondOk(jinglePacket);
1301 }
1302 }
1303
1304 private void receiveSessionAccept(final RtpContentMap contentMap) {
1305 this.responderRtpContentMap = contentMap;
1306 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1307 final SessionDescription sessionDescription;
1308 try {
1309 sessionDescription = SessionDescription.of(contentMap, false);
1310 } catch (final IllegalArgumentException | NullPointerException e) {
1311 Log.d(
1312 Config.LOGTAG,
1313 id.account.getJid().asBareJid()
1314 + ": unable convert offer from session-accept to SDP",
1315 e);
1316 webRTCWrapper.close();
1317 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1318 return;
1319 }
1320 final org.webrtc.SessionDescription answer =
1321 new org.webrtc.SessionDescription(
1322 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1323 try {
1324 this.webRTCWrapper.setRemoteDescription(answer).get();
1325 } catch (final Exception e) {
1326 Log.d(
1327 Config.LOGTAG,
1328 id.account.getJid().asBareJid()
1329 + ": unable to set remote description after receiving session-accept",
1330 Throwables.getRootCause(e));
1331 webRTCWrapper.close();
1332 sendSessionTerminate(
1333 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1334 return;
1335 }
1336 processCandidates(contentMap.contents.entrySet());
1337 }
1338
1339 private void sendSessionAccept() {
1340 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1341 if (rtpContentMap == null) {
1342 throw new IllegalStateException("initiator RTP Content Map has not been set");
1343 }
1344 final SessionDescription offer;
1345 try {
1346 offer = SessionDescription.of(rtpContentMap, true);
1347 } catch (final IllegalArgumentException | NullPointerException e) {
1348 Log.d(
1349 Config.LOGTAG,
1350 id.account.getJid().asBareJid()
1351 + ": unable convert offer from session-initiate to SDP",
1352 e);
1353 webRTCWrapper.close();
1354 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1355 return;
1356 }
1357 sendSessionAccept(rtpContentMap.getMedia(), offer);
1358 }
1359
1360 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1361 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1362 }
1363
1364 private synchronized void sendSessionAccept(
1365 final Set<Media> media,
1366 final SessionDescription offer,
1367 final List<PeerConnection.IceServer> iceServers) {
1368 if (isTerminated()) {
1369 Log.w(
1370 Config.LOGTAG,
1371 id.account.getJid().asBareJid()
1372 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1373 return;
1374 }
1375 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1376 try {
1377 setupWebRTC(media, iceServers, !includeCandidates);
1378 } catch (final WebRTCWrapper.InitializationException e) {
1379 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1380 webRTCWrapper.close();
1381 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1382 return;
1383 }
1384 final org.webrtc.SessionDescription sdp =
1385 new org.webrtc.SessionDescription(
1386 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1387 try {
1388 this.webRTCWrapper.setRemoteDescription(sdp).get();
1389 addIceCandidatesFromBlackLog();
1390 org.webrtc.SessionDescription webRTCSessionDescription =
1391 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1392 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1393 } catch (final Exception e) {
1394 failureToAcceptSession(e);
1395 }
1396 }
1397
1398 private void failureToAcceptSession(final Throwable throwable) {
1399 if (isTerminated()) {
1400 return;
1401 }
1402 final Throwable rootCause = Throwables.getRootCause(throwable);
1403 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1404 webRTCWrapper.close();
1405 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1406 }
1407
1408 private void failureToPerformAction(
1409 final JinglePacket.Action action, final Throwable throwable) {
1410 if (isTerminated()) {
1411 return;
1412 }
1413 final Throwable rootCause = Throwables.getRootCause(throwable);
1414 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1415 webRTCWrapper.close();
1416 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1417 }
1418
1419 private void addIceCandidatesFromBlackLog() {
1420 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1421 while ((foo = this.pendingIceCandidates.poll()) != null) {
1422 processCandidate(foo);
1423 Log.d(
1424 Config.LOGTAG,
1425 id.account.getJid().asBareJid() + ": added candidate from back log");
1426 }
1427 }
1428
1429 private void prepareSessionAccept(
1430 final org.webrtc.SessionDescription webRTCSessionDescription,
1431 final boolean includeCandidates) {
1432 final SessionDescription sessionDescription =
1433 SessionDescription.parse(webRTCSessionDescription.description);
1434 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1435 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1436 if (includeCandidates) {
1437 candidates = parseCandidates(sessionDescription);
1438 } else {
1439 candidates = ImmutableMultimap.of();
1440 }
1441 this.responderRtpContentMap = respondingRtpContentMap;
1442 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1443 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1444 prepareOutgoingContentMap(respondingRtpContentMap);
1445 Futures.addCallback(
1446 outgoingContentMapFuture,
1447 new FutureCallback<>() {
1448 @Override
1449 public void onSuccess(final RtpContentMap outgoingContentMap) {
1450 if (includeCandidates) {
1451 Log.d(
1452 Config.LOGTAG,
1453 "including "
1454 + candidates.size()
1455 + " candidates in session accept");
1456 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1457 } else {
1458 sendSessionAccept(outgoingContentMap);
1459 }
1460 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1461 }
1462
1463 @Override
1464 public void onFailure(@NonNull Throwable throwable) {
1465 failureToAcceptSession(throwable);
1466 }
1467 },
1468 MoreExecutors.directExecutor());
1469 }
1470
1471 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1472 if (isTerminated()) {
1473 Log.w(
1474 Config.LOGTAG,
1475 id.account.getJid().asBareJid()
1476 + ": preparing session accept was too slow. already terminated. nothing to do.");
1477 return;
1478 }
1479 transitionOrThrow(State.SESSION_ACCEPTED);
1480 final JinglePacket sessionAccept =
1481 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1482 send(sessionAccept);
1483 }
1484
1485 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1486 final RtpContentMap rtpContentMap) {
1487 if (this.omemoVerification.hasDeviceId()) {
1488 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1489 verifiedPayloadFuture =
1490 id.account
1491 .getAxolotlService()
1492 .encrypt(
1493 rtpContentMap,
1494 id.with,
1495 omemoVerification.getDeviceId());
1496 return Futures.transform(
1497 verifiedPayloadFuture,
1498 verifiedPayload -> {
1499 omemoVerification.setOrEnsureEqual(verifiedPayload);
1500 return verifiedPayload.getPayload();
1501 },
1502 MoreExecutors.directExecutor());
1503 } else {
1504 return Futures.immediateFuture(rtpContentMap);
1505 }
1506 }
1507
1508 synchronized void deliveryMessage(
1509 final Jid from,
1510 final Element message,
1511 final String serverMessageId,
1512 final long timestamp) {
1513 Log.d(
1514 Config.LOGTAG,
1515 id.account.getJid().asBareJid()
1516 + ": delivered message to JingleRtpConnection "
1517 + message);
1518 switch (message.getName()) {
1519 case "propose" -> receivePropose(
1520 from, Propose.upgrade(message), serverMessageId, timestamp);
1521 case "proceed" -> receiveProceed(
1522 from, Proceed.upgrade(message), serverMessageId, timestamp);
1523 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1524 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1525 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1526 }
1527 }
1528
1529 void deliverFailedProceed(final String message) {
1530 Log.d(
1531 Config.LOGTAG,
1532 id.account.getJid().asBareJid()
1533 + ": receive message error for proceed message ("
1534 + Strings.nullToEmpty(message)
1535 + ")");
1536 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1537 webRTCWrapper.close();
1538 Log.d(
1539 Config.LOGTAG,
1540 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1541 this.finish();
1542 }
1543 }
1544
1545 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1546 final boolean originatedFromMyself =
1547 from.asBareJid().equals(id.account.getJid().asBareJid());
1548 if (originatedFromMyself) {
1549 if (transition(State.ACCEPTED)) {
1550 acceptedOnOtherDevice(serverMsgId, timestamp);
1551 } else {
1552 Log.d(
1553 Config.LOGTAG,
1554 id.account.getJid().asBareJid()
1555 + ": unable to transition to accept because already in state="
1556 + this.state);
1557 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1558 }
1559 } else {
1560 Log.d(
1561 Config.LOGTAG,
1562 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1563 }
1564 }
1565
1566 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1567 if (serverMsgId != null) {
1568 this.message.setServerMsgId(serverMsgId);
1569 }
1570 this.message.setTime(timestamp);
1571 this.message.setCarbon(true); // indicate that call was accepted on other device
1572 this.writeLogMessageSuccess(0);
1573 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1574 this.finish();
1575 }
1576
1577 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1578 final boolean originatedFromMyself =
1579 from.asBareJid().equals(id.account.getJid().asBareJid());
1580 // reject from another one of my clients
1581 if (originatedFromMyself) {
1582 receiveRejectFromMyself(serverMsgId, timestamp);
1583 } else if (isInitiator()) {
1584 if (from.equals(id.with)) {
1585 receiveRejectFromResponder();
1586 } else {
1587 Log.d(
1588 Config.LOGTAG,
1589 id.account.getJid()
1590 + ": ignoring reject from "
1591 + from
1592 + " for session with "
1593 + id.with);
1594 }
1595 } else {
1596 Log.d(
1597 Config.LOGTAG,
1598 id.account.getJid()
1599 + ": ignoring reject from "
1600 + from
1601 + " for session with "
1602 + id.with);
1603 }
1604 }
1605
1606 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1607 if (transition(State.REJECTED)) {
1608 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1609 this.finish();
1610 if (serverMsgId != null) {
1611 this.message.setServerMsgId(serverMsgId);
1612 }
1613 this.message.setTime(timestamp);
1614 this.message.setCarbon(true); // indicate that call was rejected on other device
1615 writeLogMessageMissed();
1616 } else {
1617 Log.d(
1618 Config.LOGTAG,
1619 "not able to transition into REJECTED because already in " + this.state);
1620 }
1621 }
1622
1623 private void receiveRejectFromResponder() {
1624 if (isInState(State.PROCEED)) {
1625 Log.d(
1626 Config.LOGTAG,
1627 id.account.getJid()
1628 + ": received reject while still in proceed. callee reconsidered");
1629 closeTransitionLogFinish(State.REJECTED_RACED);
1630 return;
1631 }
1632 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1633 Log.d(
1634 Config.LOGTAG,
1635 id.account.getJid()
1636 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1637 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1638 return;
1639 }
1640 Log.d(
1641 Config.LOGTAG,
1642 id.account.getJid()
1643 + ": ignoring reject from responder because already in state "
1644 + this.state);
1645 }
1646
1647 private void receivePropose(
1648 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1649 final boolean originatedFromMyself =
1650 from.asBareJid().equals(id.account.getJid().asBareJid());
1651 if (originatedFromMyself) {
1652 Log.d(
1653 Config.LOGTAG,
1654 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1655 } else if (transition(
1656 State.PROPOSED,
1657 () -> {
1658 final Collection<RtpDescription> descriptions =
1659 Collections2.transform(
1660 Collections2.filter(
1661 propose.getDescriptions(),
1662 d -> d instanceof RtpDescription),
1663 input -> (RtpDescription) input);
1664 final Collection<Media> media =
1665 Collections2.transform(descriptions, RtpDescription::getMedia);
1666 Preconditions.checkState(
1667 !media.contains(Media.UNKNOWN),
1668 "RTP descriptions contain unknown media");
1669 Log.d(
1670 Config.LOGTAG,
1671 id.account.getJid().asBareJid()
1672 + ": received session proposal from "
1673 + from
1674 + " for "
1675 + media);
1676 this.setProposedMedia(Sets.newHashSet(media));
1677 })) {
1678 if (serverMsgId != null) {
1679 this.message.setServerMsgId(serverMsgId);
1680 }
1681 this.message.setTime(timestamp);
1682 startRinging();
1683 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1684 sendJingleMessage("ringing");
1685 }
1686 } else {
1687 Log.d(
1688 Config.LOGTAG,
1689 id.account.getJid()
1690 + ": ignoring session proposal because already in "
1691 + state);
1692 }
1693 }
1694
1695 private void startRinging() {
1696 this.callIntegration.setRinging();
1697 Log.d(
1698 Config.LOGTAG,
1699 id.account.getJid().asBareJid()
1700 + ": received call from "
1701 + id.with
1702 + ". start ringing");
1703 ringingTimeoutFuture =
1704 jingleConnectionManager.schedule(
1705 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1706 if (CallIntegration.selfManaged(xmppConnectionService)) {
1707 if (CallIntegrationConnectionService.addNewIncomingCall(xmppConnectionService, getId())) {
1708 return;
1709 }
1710 }
1711 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1712 }
1713
1714 private synchronized void ringingTimeout() {
1715 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1716 switch (this.state) {
1717 case PROPOSED -> {
1718 message.markUnread();
1719 rejectCallFromProposed();
1720 }
1721 case SESSION_INITIALIZED -> {
1722 message.markUnread();
1723 rejectCallFromSessionInitiate();
1724 }
1725 }
1726 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1727 }
1728
1729 private void cancelRingingTimeout() {
1730 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1731 if (future != null && !future.isCancelled()) {
1732 future.cancel(false);
1733 }
1734 }
1735
1736 private void receiveProceed(
1737 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1738 final Set<Media> media =
1739 Preconditions.checkNotNull(
1740 this.proposedMedia, "Proposed media has to be set before handling proceed");
1741 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1742 if (from.equals(id.with)) {
1743 if (isInitiator()) {
1744 if (transition(State.PROCEED)) {
1745 if (serverMsgId != null) {
1746 this.message.setServerMsgId(serverMsgId);
1747 }
1748 this.message.setTime(timestamp);
1749 final Integer remoteDeviceId = proceed.getDeviceId();
1750 if (isOmemoEnabled()) {
1751 this.omemoVerification.setDeviceId(remoteDeviceId);
1752 } else {
1753 if (remoteDeviceId != null) {
1754 Log.d(
1755 Config.LOGTAG,
1756 id.account.getJid().asBareJid()
1757 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1758 }
1759 this.omemoVerification.setDeviceId(null);
1760 }
1761 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1762 } else {
1763 Log.d(
1764 Config.LOGTAG,
1765 String.format(
1766 "%s: ignoring proceed because already in %s",
1767 id.account.getJid().asBareJid(), this.state));
1768 }
1769 } else {
1770 Log.d(
1771 Config.LOGTAG,
1772 String.format(
1773 "%s: ignoring proceed because we were not initializing",
1774 id.account.getJid().asBareJid()));
1775 }
1776 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1777 if (transition(State.ACCEPTED)) {
1778 Log.d(
1779 Config.LOGTAG,
1780 id.account.getJid().asBareJid()
1781 + ": moved session with "
1782 + id.with
1783 + " into state accepted after received carbon copied proceed");
1784 acceptedOnOtherDevice(serverMsgId, timestamp);
1785 }
1786 } else {
1787 Log.d(
1788 Config.LOGTAG,
1789 String.format(
1790 "%s: ignoring proceed from %s. was expected from %s",
1791 id.account.getJid().asBareJid(), from, id.with));
1792 }
1793 }
1794
1795 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1796 if (from.equals(id.with)) {
1797 final State target =
1798 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1799 if (transition(target)) {
1800 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1801 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1802 Log.d(
1803 Config.LOGTAG,
1804 id.account.getJid().asBareJid()
1805 + ": session with "
1806 + id.with
1807 + " has been retracted (serverMsgId="
1808 + serverMsgId
1809 + ")");
1810 if (serverMsgId != null) {
1811 this.message.setServerMsgId(serverMsgId);
1812 }
1813 this.message.setTime(timestamp);
1814 if (target == State.RETRACTED) {
1815 this.message.markUnread();
1816 }
1817 writeLogMessageMissed();
1818 finish();
1819 } else {
1820 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1821 }
1822 } else {
1823 // TODO parse retract from self
1824 Log.d(
1825 Config.LOGTAG,
1826 id.account.getJid().asBareJid()
1827 + ": received retract from "
1828 + from
1829 + ". expected retract from"
1830 + id.with
1831 + ". ignoring");
1832 }
1833 }
1834
1835 public void sendSessionInitiate() {
1836 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1837 }
1838
1839 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1840 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1841 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1842 }
1843
1844 private synchronized void sendSessionInitiate(
1845 final Set<Media> media,
1846 final State targetState,
1847 final List<PeerConnection.IceServer> iceServers) {
1848 if (isTerminated()) {
1849 Log.w(
1850 Config.LOGTAG,
1851 id.account.getJid().asBareJid()
1852 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1853 return;
1854 }
1855 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1856 try {
1857 setupWebRTC(media, iceServers, !includeCandidates);
1858 } catch (final WebRTCWrapper.InitializationException e) {
1859 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1860 webRTCWrapper.close();
1861 sendRetract(Reason.ofThrowable(e));
1862 return;
1863 }
1864 try {
1865 org.webrtc.SessionDescription webRTCSessionDescription =
1866 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1867 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1868 } catch (final Exception e) {
1869 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1870 // exceptions
1871 failureToInitiateSession(e, targetState);
1872 }
1873 }
1874
1875 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1876 if (isTerminated()) {
1877 return;
1878 }
1879 Log.d(
1880 Config.LOGTAG,
1881 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1882 Throwables.getRootCause(throwable));
1883 webRTCWrapper.close();
1884 final Reason reason = Reason.ofThrowable(throwable);
1885 if (isInState(targetState)) {
1886 sendSessionTerminate(reason, throwable.getMessage());
1887 } else {
1888 sendRetract(reason);
1889 }
1890 }
1891
1892 private void sendRetract(final Reason reason) {
1893 // TODO embed reason into retract
1894 sendJingleMessage("retract", id.with.asBareJid());
1895 transitionOrThrow(reasonToState(reason));
1896 this.finish();
1897 }
1898
1899 private void prepareSessionInitiate(
1900 final org.webrtc.SessionDescription webRTCSessionDescription,
1901 final boolean includeCandidates,
1902 final State targetState) {
1903 final SessionDescription sessionDescription =
1904 SessionDescription.parse(webRTCSessionDescription.description);
1905 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1906 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1907 if (includeCandidates) {
1908 candidates = parseCandidates(sessionDescription);
1909 } else {
1910 candidates = ImmutableMultimap.of();
1911 }
1912 this.initiatorRtpContentMap = rtpContentMap;
1913 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1914 encryptSessionInitiate(rtpContentMap);
1915 Futures.addCallback(
1916 outgoingContentMapFuture,
1917 new FutureCallback<>() {
1918 @Override
1919 public void onSuccess(final RtpContentMap outgoingContentMap) {
1920 if (includeCandidates) {
1921 Log.d(
1922 Config.LOGTAG,
1923 "including "
1924 + candidates.size()
1925 + " candidates in session initiate");
1926 sendSessionInitiate(
1927 outgoingContentMap.withCandidates(candidates), targetState);
1928 } else {
1929 sendSessionInitiate(outgoingContentMap, targetState);
1930 }
1931 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1932 }
1933
1934 @Override
1935 public void onFailure(@NonNull final Throwable throwable) {
1936 failureToInitiateSession(throwable, targetState);
1937 }
1938 },
1939 MoreExecutors.directExecutor());
1940 }
1941
1942 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1943 if (isTerminated()) {
1944 Log.w(
1945 Config.LOGTAG,
1946 id.account.getJid().asBareJid()
1947 + ": preparing session was too slow. already terminated. nothing to do.");
1948 return;
1949 }
1950 this.transitionOrThrow(targetState);
1951 final JinglePacket sessionInitiate =
1952 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1953 send(sessionInitiate);
1954 }
1955
1956 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1957 final RtpContentMap rtpContentMap) {
1958 if (this.omemoVerification.hasDeviceId()) {
1959 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1960 verifiedPayloadFuture =
1961 id.account
1962 .getAxolotlService()
1963 .encrypt(
1964 rtpContentMap,
1965 id.with,
1966 omemoVerification.getDeviceId());
1967 final ListenableFuture<RtpContentMap> future =
1968 Futures.transform(
1969 verifiedPayloadFuture,
1970 verifiedPayload -> {
1971 omemoVerification.setSessionFingerprint(
1972 verifiedPayload.getFingerprint());
1973 return verifiedPayload.getPayload();
1974 },
1975 MoreExecutors.directExecutor());
1976 if (Config.REQUIRE_RTP_VERIFICATION) {
1977 return future;
1978 }
1979 return Futures.catching(
1980 future,
1981 CryptoFailedException.class,
1982 e -> {
1983 Log.w(
1984 Config.LOGTAG,
1985 id.account.getJid().asBareJid()
1986 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1987 e);
1988 return rtpContentMap;
1989 },
1990 MoreExecutors.directExecutor());
1991 } else {
1992 return Futures.immediateFuture(rtpContentMap);
1993 }
1994 }
1995
1996 protected void sendSessionTerminate(final Reason reason) {
1997 sendSessionTerminate(reason, null);
1998 }
1999
2000 protected void sendSessionTerminate(final Reason reason, final String text) {
2001 sendSessionTerminate(reason, text, this::writeLogMessage);
2002 sendJingleMessageFinish(reason);
2003 }
2004
2005 private void sendTransportInfo(
2006 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2007 final RtpContentMap transportInfo;
2008 try {
2009 final RtpContentMap rtpContentMap =
2010 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2011 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2012 } catch (final Exception e) {
2013 Log.d(
2014 Config.LOGTAG,
2015 id.account.getJid().asBareJid()
2016 + ": unable to prepare transport-info from candidate for content="
2017 + contentName);
2018 return;
2019 }
2020 final JinglePacket jinglePacket =
2021 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2022 send(jinglePacket);
2023 }
2024
2025 public RtpEndUserState getEndUserState() {
2026 switch (this.state) {
2027 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2028 if (isInitiator()) {
2029 return RtpEndUserState.RINGING;
2030 } else {
2031 return RtpEndUserState.INCOMING_CALL;
2032 }
2033 }
2034 case PROCEED -> {
2035 if (isInitiator()) {
2036 return RtpEndUserState.RINGING;
2037 } else {
2038 return RtpEndUserState.ACCEPTING_CALL;
2039 }
2040 }
2041 case SESSION_INITIALIZED_PRE_APPROVED -> {
2042 if (isInitiator()) {
2043 return RtpEndUserState.RINGING;
2044 } else {
2045 return RtpEndUserState.CONNECTING;
2046 }
2047 }
2048 case SESSION_ACCEPTED -> {
2049 final ContentAddition ca = getPendingContentAddition();
2050 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2051 return RtpEndUserState.INCOMING_CONTENT_ADD;
2052 }
2053 return getPeerConnectionStateAsEndUserState();
2054 }
2055 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2056 if (isInitiator()) {
2057 return RtpEndUserState.DECLINED_OR_BUSY;
2058 } else {
2059 return RtpEndUserState.ENDED;
2060 }
2061 }
2062 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2063 return RtpEndUserState.ENDED;
2064 }
2065 case RETRACTED_RACED -> {
2066 if (isInitiator()) {
2067 return RtpEndUserState.ENDED;
2068 } else {
2069 return RtpEndUserState.RETRACTED;
2070 }
2071 }
2072 case TERMINATED_CONNECTIVITY_ERROR -> {
2073 return zeroDuration()
2074 ? RtpEndUserState.CONNECTIVITY_ERROR
2075 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2076 }
2077 case TERMINATED_APPLICATION_FAILURE -> {
2078 return RtpEndUserState.APPLICATION_ERROR;
2079 }
2080 case TERMINATED_SECURITY_ERROR -> {
2081 return RtpEndUserState.SECURITY_ERROR;
2082 }
2083 }
2084 throw new IllegalStateException(
2085 String.format("%s has no equivalent EndUserState", this.state));
2086 }
2087
2088 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2089 final PeerConnection.PeerConnectionState state;
2090 try {
2091 state = webRTCWrapper.getState();
2092 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2093 // We usually close the WebRTCWrapper *before* transitioning so we might still
2094 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2095 return RtpEndUserState.ENDING_CALL;
2096 }
2097 return switch (state) {
2098 case CONNECTED -> RtpEndUserState.CONNECTED;
2099 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2100 case CLOSED -> RtpEndUserState.ENDING_CALL;
2101 default -> zeroDuration()
2102 ? RtpEndUserState.CONNECTIVITY_ERROR
2103 : RtpEndUserState.RECONNECTING;
2104 };
2105 }
2106
2107 private boolean isPeerConnectionConnected() {
2108 try {
2109 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2110 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2111 return false;
2112 }
2113 }
2114
2115 private void updateCallIntegrationState() {
2116 switch (this.state) {
2117 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2118 if (isInitiator()) {
2119 this.callIntegration.setDialing();
2120 } else {
2121 this.callIntegration.setRinging();
2122 }
2123 }
2124 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2125 if (isInitiator()) {
2126 this.callIntegration.setDialing();
2127 } else {
2128 this.callIntegration.setInitialized();
2129 }
2130 }
2131 case SESSION_ACCEPTED -> {
2132 if (isPeerConnectionConnected()) {
2133 this.callIntegration.setActive();
2134 } else {
2135 this.callIntegration.setInitialized();
2136 }
2137 }
2138 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2139 if (isInitiator()) {
2140 this.callIntegration.busy();
2141 } else {
2142 this.callIntegration.rejected();
2143 }
2144 }
2145 case TERMINATED_SUCCESS -> this.callIntegration.success();
2146 case ACCEPTED -> this.callIntegration.accepted();
2147 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2148 .retracted();
2149 case TERMINATED_CONNECTIVITY_ERROR,
2150 TERMINATED_APPLICATION_FAILURE,
2151 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2152 default -> throw new IllegalStateException(
2153 String.format("%s is not handled", this.state));
2154 }
2155 }
2156
2157 public ContentAddition getPendingContentAddition() {
2158 final RtpContentMap in = this.incomingContentAdd;
2159 final RtpContentMap out = this.outgoingContentAdd;
2160 if (out != null) {
2161 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2162 } else if (in != null) {
2163 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2164 } else {
2165 return null;
2166 }
2167 }
2168
2169 public Set<Media> getMedia() {
2170 final State current = getState();
2171 if (current == State.NULL) {
2172 if (isInitiator()) {
2173 return Preconditions.checkNotNull(
2174 this.proposedMedia, "RTP connection has not been initialized properly");
2175 }
2176 throw new IllegalStateException("RTP connection has not been initialized yet");
2177 }
2178 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2179 return Preconditions.checkNotNull(
2180 this.proposedMedia, "RTP connection has not been initialized properly");
2181 }
2182 final RtpContentMap localContentMap = getLocalContentMap();
2183 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2184 if (localContentMap != null) {
2185 return localContentMap.getMedia();
2186 } else if (initiatorContentMap != null) {
2187 return initiatorContentMap.getMedia();
2188 } else if (isTerminated()) {
2189 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2190 } else {
2191 return Preconditions.checkNotNull(
2192 this.proposedMedia, "RTP connection has not been initialized properly");
2193 }
2194 }
2195
2196 public boolean isVerified() {
2197 final String fingerprint = this.omemoVerification.getFingerprint();
2198 if (fingerprint == null) {
2199 return false;
2200 }
2201 final FingerprintStatus status =
2202 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2203 return status != null && status.isVerified();
2204 }
2205
2206 public boolean addMedia(final Media media) {
2207 final Set<Media> currentMedia = getMedia();
2208 if (currentMedia.contains(media)) {
2209 throw new IllegalStateException(String.format("%s has already been proposed", media));
2210 }
2211 // TODO add state protection - can only add while ACCEPTED or so
2212 Log.d(Config.LOGTAG, "adding media: " + media);
2213 return webRTCWrapper.addTrack(media);
2214 }
2215
2216 public synchronized void acceptCall() {
2217 switch (this.state) {
2218 case PROPOSED -> {
2219 cancelRingingTimeout();
2220 acceptCallFromProposed();
2221 }
2222 case SESSION_INITIALIZED -> {
2223 cancelRingingTimeout();
2224 acceptCallFromSessionInitialized();
2225 }
2226 case ACCEPTED -> Log.w(
2227 Config.LOGTAG,
2228 id.account.getJid().asBareJid()
2229 + ": the call has already been accepted with another client. UI was just lagging behind");
2230 case PROCEED, SESSION_ACCEPTED -> Log.w(
2231 Config.LOGTAG,
2232 id.account.getJid().asBareJid()
2233 + ": the call has already been accepted. user probably double tapped the UI");
2234 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2235 }
2236 }
2237
2238 public synchronized void rejectCall() {
2239 if (isTerminated()) {
2240 Log.w(
2241 Config.LOGTAG,
2242 id.account.getJid().asBareJid()
2243 + ": received rejectCall() when session has already been terminated. nothing to do");
2244 return;
2245 }
2246 switch (this.state) {
2247 case PROPOSED -> rejectCallFromProposed();
2248 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2249 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2250 }
2251 }
2252
2253 public synchronized void integrationFailure() {
2254 final var state = getState();
2255 if (state == State.PROPOSED) {
2256 Log.e(
2257 Config.LOGTAG,
2258 id.account.getJid().asBareJid()
2259 + ": failed call integration in state proposed");
2260 rejectCallFromProposed();
2261 } else if (state == State.SESSION_INITIALIZED) {
2262 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2263 this.webRTCWrapper.close();
2264 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2265 } else {
2266 throw new IllegalStateException(
2267 String.format("Can not fail integration in state %s", state));
2268 }
2269 }
2270
2271 public synchronized void endCall() {
2272 if (isTerminated()) {
2273 Log.w(
2274 Config.LOGTAG,
2275 id.account.getJid().asBareJid()
2276 + ": received endCall() when session has already been terminated. nothing to do");
2277 return;
2278 }
2279 if (isInState(State.PROPOSED) && isResponder()) {
2280 rejectCallFromProposed();
2281 return;
2282 }
2283 if (isInState(State.PROCEED)) {
2284 if (isInitiator()) {
2285 retractFromProceed();
2286 } else {
2287 rejectCallFromProceed();
2288 }
2289 return;
2290 }
2291 if (isInitiator()
2292 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2293 this.webRTCWrapper.close();
2294 sendSessionTerminate(Reason.CANCEL);
2295 return;
2296 }
2297 if (isInState(State.SESSION_INITIALIZED)) {
2298 rejectCallFromSessionInitiate();
2299 return;
2300 }
2301 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2302 this.webRTCWrapper.close();
2303 sendSessionTerminate(Reason.SUCCESS);
2304 return;
2305 }
2306 if (isInState(
2307 State.TERMINATED_APPLICATION_FAILURE,
2308 State.TERMINATED_CONNECTIVITY_ERROR,
2309 State.TERMINATED_DECLINED_OR_BUSY)) {
2310 Log.d(
2311 Config.LOGTAG,
2312 "ignoring request to end call because already in state " + this.state);
2313 return;
2314 }
2315 throw new IllegalStateException(
2316 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2317 }
2318
2319 private void retractFromProceed() {
2320 Log.d(Config.LOGTAG, "retract from proceed");
2321 this.sendJingleMessage("retract");
2322 closeTransitionLogFinish(State.RETRACTED_RACED);
2323 }
2324
2325 private void closeTransitionLogFinish(final State state) {
2326 this.webRTCWrapper.close();
2327 transitionOrThrow(state);
2328 writeLogMessage(state);
2329 finish();
2330 }
2331
2332 private void setupWebRTC(
2333 final Set<Media> media,
2334 final List<PeerConnection.IceServer> iceServers,
2335 final boolean trickle)
2336 throws WebRTCWrapper.InitializationException {
2337 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2338 this.webRTCWrapper.setup(this.xmppConnectionService);
2339 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2340 this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2341 }
2342
2343 private void acceptCallFromProposed() {
2344 transitionOrThrow(State.PROCEED);
2345 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2346 this.callIntegration.startAudioRouting();
2347 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2348 this.sendJingleMessage("proceed");
2349 }
2350
2351 private void rejectCallFromProposed() {
2352 transitionOrThrow(State.REJECTED);
2353 writeLogMessageMissed();
2354 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2355 this.sendJingleMessage("reject");
2356 finish();
2357 }
2358
2359 private void rejectCallFromProceed() {
2360 this.sendJingleMessage("reject");
2361 closeTransitionLogFinish(State.REJECTED_RACED);
2362 }
2363
2364 private void rejectCallFromSessionInitiate() {
2365 webRTCWrapper.close();
2366 sendSessionTerminate(Reason.DECLINE);
2367 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2368 }
2369
2370 private void sendJingleMessage(final String action) {
2371 sendJingleMessage(action, id.with);
2372 }
2373
2374 private void sendJingleMessage(final String action, final Jid to) {
2375 final MessagePacket messagePacket = new MessagePacket();
2376 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2377 messagePacket.setTo(to);
2378 final Element intent =
2379 messagePacket
2380 .addChild(action, Namespace.JINGLE_MESSAGE)
2381 .setAttribute("id", id.sessionId);
2382 if ("proceed".equals(action)) {
2383 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2384 if (isOmemoEnabled()) {
2385 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2386 final Element device =
2387 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2388 device.setAttribute("id", deviceId);
2389 }
2390 }
2391 messagePacket.addChild("store", "urn:xmpp:hints");
2392 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2393 }
2394
2395 private void sendJingleMessageFinish(final Reason reason) {
2396 final var account = id.getAccount();
2397 final MessagePacket messagePacket =
2398 xmppConnectionService
2399 .getMessageGenerator()
2400 .sessionFinish(id.with, id.sessionId, reason);
2401 xmppConnectionService.sendMessagePacket(account, messagePacket);
2402 }
2403
2404 private boolean isOmemoEnabled() {
2405 final Conversational conversational = message.getConversation();
2406 if (conversational instanceof Conversation) {
2407 return ((Conversation) conversational).getNextEncryption()
2408 == Message.ENCRYPTION_AXOLOTL;
2409 }
2410 return false;
2411 }
2412
2413 private void acceptCallFromSessionInitialized() {
2414 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2415 this.callIntegration.startAudioRouting();
2416 sendSessionAccept();
2417 }
2418
2419 @Override
2420 protected synchronized boolean transition(final State target, final Runnable runnable) {
2421 if (super.transition(target, runnable)) {
2422 updateEndUserState();
2423 updateOngoingCallNotification();
2424 return true;
2425 } else {
2426 return false;
2427 }
2428 }
2429
2430 @Override
2431 public void onIceCandidate(final IceCandidate iceCandidate) {
2432 final RtpContentMap rtpContentMap =
2433 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2434 final IceUdpTransportInfo.Credentials credentials;
2435 try {
2436 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2437 } catch (final IllegalArgumentException e) {
2438 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2439 return;
2440 }
2441 final String uFrag = credentials.ufrag;
2442 final IceUdpTransportInfo.Candidate candidate =
2443 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2444 if (candidate == null) {
2445 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2446 return;
2447 }
2448 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2449 sendTransportInfo(iceCandidate.sdpMid, candidate);
2450 }
2451
2452 @Override
2453 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2454 Log.d(
2455 Config.LOGTAG,
2456 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2457 this.stateHistory.add(newState);
2458 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2459 this.sessionDuration.start();
2460 updateOngoingCallNotification();
2461 } else if (this.sessionDuration.isRunning()) {
2462 this.sessionDuration.stop();
2463 updateOngoingCallNotification();
2464 }
2465
2466 final boolean neverConnected =
2467 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2468
2469 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2470 if (neverConnected) {
2471 if (isTerminated()) {
2472 Log.d(
2473 Config.LOGTAG,
2474 id.account.getJid().asBareJid()
2475 + ": not sending session-terminate after connectivity error because session is already in state "
2476 + this.state);
2477 return;
2478 }
2479 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2480 return;
2481 } else {
2482 this.restartIce();
2483 }
2484 }
2485 updateEndUserState();
2486 }
2487
2488 private void restartIce() {
2489 this.stateHistory.clear();
2490 this.webRTCWrapper.restartIceAsync();
2491 }
2492
2493 @Override
2494 public void onRenegotiationNeeded() {
2495 this.webRTCWrapper.execute(this::renegotiate);
2496 }
2497
2498 private synchronized void renegotiate() {
2499 final SessionDescription sessionDescription;
2500 try {
2501 sessionDescription = setLocalSessionDescription();
2502 } catch (final Exception e) {
2503 final Throwable cause = Throwables.getRootCause(e);
2504 webRTCWrapper.close();
2505 if (isTerminated()) {
2506 Log.d(
2507 Config.LOGTAG,
2508 "failed to renegotiate. session was already terminated",
2509 cause);
2510 return;
2511 }
2512 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2513 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2514 return;
2515 }
2516 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2517 final RtpContentMap currentContentMap = getLocalContentMap();
2518 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2519 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2520
2521 Log.d(
2522 Config.LOGTAG,
2523 id.getAccount().getJid().asBareJid()
2524 + ": renegotiate. iceRestart="
2525 + iceRestart
2526 + " content id diff="
2527 + diff);
2528
2529 if (diff.hasModifications() && iceRestart) {
2530 webRTCWrapper.close();
2531 sendSessionTerminate(
2532 Reason.FAILED_APPLICATION,
2533 "WebRTC unexpectedly tried to modify content and transport at once");
2534 return;
2535 }
2536
2537 if (iceRestart) {
2538 initiateIceRestart(rtpContentMap);
2539 return;
2540 } else if (diff.isEmpty()) {
2541 Log.d(
2542 Config.LOGTAG,
2543 "renegotiation. nothing to do. SignalingState="
2544 + this.webRTCWrapper.getSignalingState());
2545 }
2546
2547 if (diff.added.isEmpty()) {
2548 return;
2549 }
2550 modifyLocalContentMap(rtpContentMap);
2551 sendContentAdd(rtpContentMap, diff.added);
2552 }
2553
2554 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2555 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2556 final JinglePacket jinglePacket =
2557 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2558 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2559 jinglePacket.setTo(id.with);
2560 xmppConnectionService.sendIqPacket(
2561 id.account,
2562 jinglePacket,
2563 (account, response) -> {
2564 if (response.getType() == IqPacket.TYPE.RESULT) {
2565 Log.d(Config.LOGTAG, "received success to our ice restart");
2566 setLocalContentMap(rtpContentMap);
2567 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2568 return;
2569 }
2570 if (response.getType() == IqPacket.TYPE.ERROR) {
2571 if (isTieBreak(response)) {
2572 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2573 return;
2574 }
2575 handleIqErrorResponse(response);
2576 }
2577 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2578 handleIqTimeoutResponse(response);
2579 }
2580 });
2581 }
2582
2583 private boolean isTieBreak(final IqPacket response) {
2584 final Element error = response.findChild("error");
2585 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2586 }
2587
2588 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2589 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2590 this.outgoingContentAdd = contentAdd;
2591 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2592 prepareOutgoingContentMap(contentAdd);
2593 Futures.addCallback(
2594 outgoingContentMapFuture,
2595 new FutureCallback<>() {
2596 @Override
2597 public void onSuccess(final RtpContentMap outgoingContentMap) {
2598 sendContentAdd(outgoingContentMap);
2599 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2600 }
2601
2602 @Override
2603 public void onFailure(@NonNull Throwable throwable) {
2604 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2605 }
2606 },
2607 MoreExecutors.directExecutor());
2608 }
2609
2610 private void sendContentAdd(final RtpContentMap contentAdd) {
2611
2612 final JinglePacket jinglePacket =
2613 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2614 jinglePacket.setTo(id.with);
2615 xmppConnectionService.sendIqPacket(
2616 id.account,
2617 jinglePacket,
2618 (connection, response) -> {
2619 if (response.getType() == IqPacket.TYPE.RESULT) {
2620 Log.d(
2621 Config.LOGTAG,
2622 id.getAccount().getJid().asBareJid()
2623 + ": received ACK to our content-add");
2624 return;
2625 }
2626 if (response.getType() == IqPacket.TYPE.ERROR) {
2627 if (isTieBreak(response)) {
2628 this.outgoingContentAdd = null;
2629 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2630 return;
2631 }
2632 handleIqErrorResponse(response);
2633 }
2634 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2635 handleIqTimeoutResponse(response);
2636 }
2637 });
2638 }
2639
2640 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2641 if (isInitiator()) {
2642 this.initiatorRtpContentMap = rtpContentMap;
2643 } else {
2644 this.responderRtpContentMap = rtpContentMap;
2645 }
2646 }
2647
2648 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2649 if (isInitiator()) {
2650 this.responderRtpContentMap = rtpContentMap;
2651 } else {
2652 this.initiatorRtpContentMap = rtpContentMap;
2653 }
2654 }
2655
2656 // this method is to be used for content map modifications that modify media
2657 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2658 final RtpContentMap activeContents = rtpContentMap.activeContents();
2659 setLocalContentMap(activeContents);
2660 this.callIntegration.setAudioDeviceWhenAvailable(
2661 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2662 updateEndUserState();
2663 }
2664
2665 private SessionDescription setLocalSessionDescription()
2666 throws ExecutionException, InterruptedException {
2667 final org.webrtc.SessionDescription sessionDescription =
2668 this.webRTCWrapper.setLocalDescription(false).get();
2669 return SessionDescription.parse(sessionDescription.description);
2670 }
2671
2672 private void closeWebRTCSessionAfterFailedConnection() {
2673 this.webRTCWrapper.close();
2674 synchronized (this) {
2675 if (isTerminated()) {
2676 Log.d(
2677 Config.LOGTAG,
2678 id.account.getJid().asBareJid()
2679 + ": no need to send session-terminate after failed connection. Other party already did");
2680 return;
2681 }
2682 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2683 }
2684 }
2685
2686 public boolean zeroDuration() {
2687 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2688 }
2689
2690 public long getCallDuration() {
2691 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2692 }
2693
2694 @Override
2695 public CallIntegration getCallIntegration() {
2696 return this.callIntegration;
2697 }
2698
2699 public boolean isMicrophoneEnabled() {
2700 return webRTCWrapper.isMicrophoneEnabled();
2701 }
2702
2703 public boolean setMicrophoneEnabled(final boolean enabled) {
2704 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2705 }
2706
2707 public boolean isVideoEnabled() {
2708 return webRTCWrapper.isVideoEnabled();
2709 }
2710
2711 public void setVideoEnabled(final boolean enabled) {
2712 webRTCWrapper.setVideoEnabled(enabled);
2713 }
2714
2715 public boolean isCameraSwitchable() {
2716 return webRTCWrapper.isCameraSwitchable();
2717 }
2718
2719 public boolean isFrontCamera() {
2720 return webRTCWrapper.isFrontCamera();
2721 }
2722
2723 public ListenableFuture<Boolean> switchCamera() {
2724 return webRTCWrapper.switchCamera();
2725 }
2726
2727 @Override
2728 public synchronized void onCallIntegrationShowIncomingCallUi() {
2729 if (isTerminated()) {
2730 // there might be race conditions with the call integration service invoking this
2731 // callback when the rtp session has already ended.
2732 Log.w(
2733 Config.LOGTAG,
2734 "CallIntegration requested incoming call UI but session was already terminated");
2735 return;
2736 }
2737 // TODO apparently this can be called too early as well?
2738 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2739 }
2740
2741 @Override
2742 public void onCallIntegrationDisconnect() {
2743 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2744 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2745 rejectCall();
2746 } else {
2747 endCall();
2748 }
2749 }
2750
2751 @Override
2752 public void onCallIntegrationReject() {
2753 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2754 try {
2755 rejectCall();
2756 } catch (final IllegalStateException e) {
2757 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2758 }
2759 }
2760
2761 @Override
2762 public void onCallIntegrationAnswer() {
2763 // we need to start the UI to a) show it and b) be able to ask for permissions
2764 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2765 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2766 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2767 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2768 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2769 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2770 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2771 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2772 xmppConnectionService.startActivity(intent);
2773 }
2774
2775 @Override
2776 public void onCallIntegrationSilence() {
2777 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2778 }
2779
2780 @Override
2781 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2782 this.webRTCWrapper.setMicrophoneEnabled(enabled);
2783 }
2784
2785 @Override
2786 public void onAudioDeviceChanged(
2787 final CallIntegration.AudioDevice selectedAudioDevice,
2788 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2789 Log.d(
2790 Config.LOGTAG,
2791 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2792 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2793 selectedAudioDevice, availableAudioDevices);
2794 }
2795
2796 private void updateEndUserState() {
2797 final RtpEndUserState endUserState = getEndUserState();
2798 this.updateCallIntegrationState();
2799 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2800 id.account, id.with, id.sessionId, endUserState);
2801 }
2802
2803 private void updateOngoingCallNotification() {
2804 final State state = this.state;
2805 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2806 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(state)
2807 && isResponder()) {
2808 Log.d(Config.LOGTAG, "do not set ongoing call during incoming call notification");
2809 xmppConnectionService.removeOngoingCall();
2810 return;
2811 }
2812 final boolean reconnecting;
2813 if (state == State.SESSION_ACCEPTED) {
2814 reconnecting =
2815 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2816 } else {
2817 reconnecting = false;
2818 }
2819 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2820 } else {
2821 xmppConnectionService.removeOngoingCall();
2822 }
2823 }
2824
2825 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2826 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2827 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2828 request.setTo(id.account.getDomain());
2829 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2830 xmppConnectionService.sendIqPacket(
2831 id.account,
2832 request,
2833 (account, response) -> {
2834 final var iceServers = IceServers.parse(response);
2835 if (iceServers.isEmpty()) {
2836 Log.w(
2837 Config.LOGTAG,
2838 id.account.getJid().asBareJid()
2839 + ": no ICE server found "
2840 + response);
2841 }
2842 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2843 });
2844 } else {
2845 Log.w(
2846 Config.LOGTAG,
2847 id.account.getJid().asBareJid() + ": has no external service discovery");
2848 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2849 }
2850 }
2851
2852 @Override
2853 protected void terminateTransport() {
2854 this.webRTCWrapper.close();
2855 }
2856
2857 @Override
2858 protected void finish() {
2859 if (isTerminated()) {
2860 this.cancelRingingTimeout();
2861 this.callIntegration.verifyDisconnected();
2862 this.webRTCWrapper.verifyClosed();
2863 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2864 super.finish();
2865 try {
2866 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2867 log.getParentFile().mkdirs();
2868 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2869 } catch (final IOException e) { }
2870 } else {
2871 throw new IllegalStateException(
2872 String.format("Unable to call finish from %s", this.state));
2873 }
2874 }
2875
2876 private void writeLogMessage(final State state) {
2877 final long duration = getCallDuration();
2878 if (state == State.TERMINATED_SUCCESS
2879 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2880 writeLogMessageSuccess(duration);
2881 } else {
2882 writeLogMessageMissed();
2883 }
2884 }
2885
2886 private void writeLogMessageSuccess(final long duration) {
2887 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2888 this.writeMessage();
2889 }
2890
2891 private void writeLogMessageMissed() {
2892 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2893 this.writeMessage();
2894 }
2895
2896 private void writeMessage() {
2897 final Conversational conversational = message.getConversation();
2898 if (conversational instanceof Conversation) {
2899 ((Conversation) conversational).add(this.message);
2900 xmppConnectionService.createMessageAsync(message);
2901 xmppConnectionService.updateConversationUi();
2902 } else {
2903 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2904 }
2905 }
2906
2907 public Optional<VideoTrack> getLocalVideoTrack() {
2908 return webRTCWrapper.getLocalVideoTrack();
2909 }
2910
2911 public Optional<VideoTrack> getRemoteVideoTrack() {
2912 return webRTCWrapper.getRemoteVideoTrack();
2913 }
2914
2915 public EglBase.Context getEglBaseContext() {
2916 return webRTCWrapper.getEglBaseContext();
2917 }
2918
2919 void setProposedMedia(final Set<Media> media) {
2920 this.proposedMedia = media;
2921 this.callIntegration.setVideoState(
2922 Media.audioOnly(media)
2923 ? VideoProfile.STATE_AUDIO_ONLY
2924 : VideoProfile.STATE_BIDIRECTIONAL);
2925 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2926 }
2927
2928 public void fireStateUpdate() {
2929 final RtpEndUserState endUserState = getEndUserState();
2930 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2931 id.account, id.with, id.sessionId, endUserState);
2932 }
2933
2934 public boolean isSwitchToVideoAvailable() {
2935 final boolean prerequisite =
2936 Media.audioOnly(getMedia())
2937 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2938 .contains(getEndUserState());
2939 return prerequisite && remoteHasVideoFeature();
2940 }
2941
2942 private boolean remoteHasVideoFeature() {
2943 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2944 }
2945
2946 private boolean remoteHasSdpOfferAnswer() {
2947 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2948 }
2949
2950 @Override
2951 public Account getAccount() {
2952 return id.account;
2953 }
2954
2955 @Override
2956 public Jid getWith() {
2957 return id.with;
2958 }
2959
2960 @Override
2961 public String getSessionId() {
2962 return id.sessionId;
2963 }
2964
2965 private interface OnIceServersDiscovered {
2966 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2967 }
2968}