1package eu.siacs.conversations.xmpp.jingle;
2
3import android.content.Intent;
4import android.telecom.TelecomManager;
5import android.telecom.VideoProfile;
6import android.util.Log;
7import android.os.Environment;
8
9import androidx.annotation.NonNull;
10import androidx.annotation.Nullable;
11
12import com.google.common.base.Joiner;
13import com.google.common.base.Optional;
14import com.google.common.base.Preconditions;
15import com.google.common.base.Stopwatch;
16import com.google.common.base.Strings;
17import com.google.common.base.Throwables;
18import com.google.common.collect.Collections2;
19import com.google.common.collect.ImmutableMultimap;
20import com.google.common.collect.ImmutableSet;
21import com.google.common.collect.Iterables;
22import com.google.common.collect.Maps;
23import com.google.common.collect.Sets;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import eu.siacs.conversations.BuildConfig;
30import eu.siacs.conversations.Config;
31import eu.siacs.conversations.crypto.axolotl.AxolotlService;
32import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
33import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
34import eu.siacs.conversations.entities.Account;
35import eu.siacs.conversations.entities.Conversation;
36import eu.siacs.conversations.entities.Conversational;
37import eu.siacs.conversations.entities.Message;
38import eu.siacs.conversations.entities.RtpSessionStatus;
39import eu.siacs.conversations.services.CallIntegration;
40import eu.siacs.conversations.ui.RtpSessionActivity;
41import eu.siacs.conversations.xml.Element;
42import eu.siacs.conversations.xml.Namespace;
43import eu.siacs.conversations.xmpp.Jid;
44import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
45import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
46import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
47import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
49import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
50import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
51import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
52import eu.siacs.conversations.xmpp.stanzas.IqPacket;
53import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
54
55import org.webrtc.DtmfSender;
56import org.webrtc.EglBase;
57import org.webrtc.IceCandidate;
58import org.webrtc.PeerConnection;
59import org.webrtc.VideoTrack;
60
61import java.io.File;
62import java.io.IOException;
63import java.util.Arrays;
64import java.util.Collection;
65import java.util.Collections;
66import java.util.LinkedList;
67import java.util.List;
68import java.util.Map;
69import java.util.Queue;
70import java.util.Set;
71import java.util.concurrent.ExecutionException;
72import java.util.concurrent.ScheduledFuture;
73import java.util.concurrent.TimeUnit;
74
75public class JingleRtpConnection extends AbstractJingleConnection
76 implements WebRTCWrapper.EventCallback, CallIntegration.Callback, OngoingRtpSession {
77
78 public static final List<State> STATES_SHOWING_ONGOING_CALL =
79 Arrays.asList(
80 State.PROPOSED,
81 State.PROCEED,
82 State.SESSION_INITIALIZED_PRE_APPROVED,
83 State.SESSION_ACCEPTED);
84 private static final long BUSY_TIME_OUT = 30;
85
86 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
87 private final Queue<
88 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
89 pendingIceCandidates = new LinkedList<>();
90 private final OmemoVerification omemoVerification = new OmemoVerification();
91 public final CallIntegration callIntegration;
92 private final Message message;
93
94 private Set<Media> proposedMedia;
95 private RtpContentMap initiatorRtpContentMap;
96 private RtpContentMap responderRtpContentMap;
97 private RtpContentMap incomingContentAdd;
98 private RtpContentMap outgoingContentAdd;
99 private IceUdpTransportInfo.Setup peerDtlsSetup;
100 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
101 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
102 private ScheduledFuture<?> ringingTimeoutFuture;
103 private final long created = System.currentTimeMillis() / 1000L;
104
105 JingleRtpConnection(
106 final JingleConnectionManager jingleConnectionManager,
107 final Id id,
108 final Jid initiator) {
109 this(
110 jingleConnectionManager,
111 id,
112 initiator,
113 new CallIntegration(
114 jingleConnectionManager
115 .getXmppConnectionService()
116 .getApplicationContext()));
117 this.callIntegration.setAddress(
118 CallIntegration.address(id.with.asBareJid()), TelecomManager.PRESENTATION_ALLOWED);
119 this.callIntegration.setInitialized();
120 }
121
122 JingleRtpConnection(
123 final JingleConnectionManager jingleConnectionManager,
124 final Id id,
125 final Jid initiator,
126 final CallIntegration callIntegration) {
127 super(jingleConnectionManager, id, initiator);
128 final Conversation conversation =
129 jingleConnectionManager
130 .getXmppConnectionService()
131 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
132 this.message =
133 new Message(
134 conversation,
135 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
136 Message.TYPE_RTP_SESSION,
137 id.sessionId);
138 this.callIntegration = callIntegration;
139 this.callIntegration.setCallback(this);
140 }
141
142 @Override
143 synchronized void deliverPacket(final JinglePacket jinglePacket) {
144 switch (jinglePacket.getAction()) {
145 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
146 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
147 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
148 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
149 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
150 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
151 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
152 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
153 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
154 default -> {
155 respondOk(jinglePacket);
156 Log.d(
157 Config.LOGTAG,
158 String.format(
159 "%s: received unhandled jingle action %s",
160 id.account.getJid().asBareJid(), jinglePacket.getAction()));
161 }
162 }
163 }
164
165 @Override
166 synchronized void notifyRebound() {
167 if (isTerminated()) {
168 return;
169 }
170 webRTCWrapper.close();
171 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
172 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
173 }
174 if (isInState(
175 State.SESSION_INITIALIZED,
176 State.SESSION_INITIALIZED_PRE_APPROVED,
177 State.SESSION_ACCEPTED)) {
178 // we might have already changed resources (full jid) at this point; so this might not
179 // even reach the other party
180 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
181 } else {
182 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
183 finish();
184 }
185 }
186
187 public boolean applyDtmfTone(final String tone) {
188 return webRTCWrapper.applyDtmfTone(tone);
189 }
190
191 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
192 respondOk(jinglePacket);
193 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
194 final State previous = this.state;
195 Log.d(
196 Config.LOGTAG,
197 id.account.getJid().asBareJid()
198 + ": received session terminate reason="
199 + wrapper.reason
200 + "("
201 + Strings.nullToEmpty(wrapper.text)
202 + ") while in state "
203 + previous);
204 if (TERMINATED.contains(previous)) {
205 Log.d(
206 Config.LOGTAG,
207 id.account.getJid().asBareJid()
208 + ": ignoring session terminate because already in "
209 + previous);
210 return;
211 }
212 webRTCWrapper.close();
213 final State target = reasonToState(wrapper.reason);
214 transitionOrThrow(target);
215 writeLogMessage(target);
216 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
217 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
218 }
219 finish();
220 }
221
222 private void receiveTransportInfo(final JinglePacket jinglePacket) {
223 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
224 // INITIALIZED only after transport-info has been received
225 if (isInState(
226 State.NULL,
227 State.PROCEED,
228 State.SESSION_INITIALIZED,
229 State.SESSION_INITIALIZED_PRE_APPROVED,
230 State.SESSION_ACCEPTED)) {
231 final RtpContentMap contentMap;
232 try {
233 contentMap = RtpContentMap.of(jinglePacket);
234 } catch (final IllegalArgumentException | NullPointerException e) {
235 Log.d(
236 Config.LOGTAG,
237 id.account.getJid().asBareJid()
238 + ": improperly formatted contents; ignoring",
239 e);
240 respondOk(jinglePacket);
241 return;
242 }
243 receiveTransportInfo(jinglePacket, contentMap);
244 } else {
245 if (isTerminated()) {
246 respondOk(jinglePacket);
247 Log.d(
248 Config.LOGTAG,
249 id.account.getJid().asBareJid()
250 + ": ignoring out-of-order transport info; we where already terminated");
251 } else {
252 Log.d(
253 Config.LOGTAG,
254 id.account.getJid().asBareJid()
255 + ": received transport info while in state="
256 + this.state);
257 terminateWithOutOfOrder(jinglePacket);
258 }
259 }
260 }
261
262 private void receiveTransportInfo(
263 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
264 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
265 candidates = contentMap.contents.entrySet();
266 final RtpContentMap remote = getRemoteContentMap();
267 final Set<String> remoteContentIds =
268 remote == null ? Collections.emptySet() : remote.contents.keySet();
269 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
270 Log.d(
271 Config.LOGTAG,
272 "received transport-info for unknown contents "
273 + contentMap.contents.keySet()
274 + " (known: "
275 + remoteContentIds
276 + ")");
277 respondOk(jinglePacket);
278 pendingIceCandidates.addAll(candidates);
279 return;
280 }
281 if (this.state != State.SESSION_ACCEPTED) {
282 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
283 respondOk(jinglePacket);
284 pendingIceCandidates.addAll(candidates);
285 return;
286 }
287 // zero candidates + modified credentials are an ICE restart offer
288 if (checkForIceRestart(jinglePacket, contentMap)) {
289 return;
290 }
291 respondOk(jinglePacket);
292 try {
293 processCandidates(candidates);
294 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
295 Log.w(
296 Config.LOGTAG,
297 id.account.getJid().asBareJid()
298 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
299 }
300 }
301
302 private void receiveContentAdd(final JinglePacket jinglePacket) {
303 final RtpContentMap modification;
304 try {
305 modification = RtpContentMap.of(jinglePacket);
306 modification.requireContentDescriptions();
307 } catch (final RuntimeException e) {
308 Log.d(
309 Config.LOGTAG,
310 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
311 Throwables.getRootCause(e));
312 respondOk(jinglePacket);
313 webRTCWrapper.close();
314 sendSessionTerminate(Reason.of(e), e.getMessage());
315 return;
316 }
317 if (isInState(State.SESSION_ACCEPTED)) {
318 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
319 final ListenableFuture<RtpContentMap> future =
320 receiveRtpContentMap(
321 modification,
322 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
323 Futures.addCallback(
324 future,
325 new FutureCallback<>() {
326 @Override
327 public void onSuccess(final RtpContentMap rtpContentMap) {
328 receiveContentAdd(jinglePacket, rtpContentMap);
329 }
330
331 @Override
332 public void onFailure(@NonNull Throwable throwable) {
333 respondOk(jinglePacket);
334 final Throwable rootCause = Throwables.getRootCause(throwable);
335 Log.d(
336 Config.LOGTAG,
337 id.account.getJid().asBareJid()
338 + ": improperly formatted contents in content-add",
339 throwable);
340 webRTCWrapper.close();
341 sendSessionTerminate(
342 Reason.ofThrowable(rootCause), rootCause.getMessage());
343 }
344 },
345 MoreExecutors.directExecutor());
346 } else {
347 terminateWithOutOfOrder(jinglePacket);
348 }
349 }
350
351 private void receiveContentAdd(
352 final JinglePacket jinglePacket, final RtpContentMap modification) {
353 final RtpContentMap remote = getRemoteContentMap();
354 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
355 respondOk(jinglePacket);
356 this.webRTCWrapper.close();
357 sendSessionTerminate(
358 Reason.FAILED_APPLICATION,
359 String.format(
360 "contents with names %s already exists",
361 Joiner.on(", ").join(modification.getNames())));
362 return;
363 }
364 final ContentAddition contentAddition =
365 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
366
367 final RtpContentMap outgoing = this.outgoingContentAdd;
368 final Set<ContentAddition.Summary> outgoingContentAddSummary =
369 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
370
371 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
372 if (isInitiator()) {
373 Log.d(
374 Config.LOGTAG,
375 id.getAccount().getJid().asBareJid()
376 + ": respond with tie break to matching content-add offer");
377 respondWithTieBreak(jinglePacket);
378 } else {
379 Log.d(
380 Config.LOGTAG,
381 id.getAccount().getJid().asBareJid()
382 + ": automatically accept matching content-add offer");
383 acceptContentAdd(contentAddition.summary, modification);
384 }
385 return;
386 }
387
388 // once we can display multiple video tracks we can be more loose with this condition
389 // theoretically it should also be fine to automatically accept audio only contents
390 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
391 Log.d(
392 Config.LOGTAG,
393 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
394 this.incomingContentAdd = modification;
395 respondOk(jinglePacket);
396 updateEndUserState();
397 } else {
398 respondOk(jinglePacket);
399 // TODO do we want to add a reason?
400 rejectContentAdd(modification);
401 }
402 }
403
404 private void receiveContentAccept(final JinglePacket jinglePacket) {
405 final RtpContentMap receivedContentAccept;
406 try {
407 receivedContentAccept = RtpContentMap.of(jinglePacket);
408 receivedContentAccept.requireContentDescriptions();
409 } catch (final RuntimeException e) {
410 Log.d(
411 Config.LOGTAG,
412 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
413 Throwables.getRootCause(e));
414 respondOk(jinglePacket);
415 webRTCWrapper.close();
416 sendSessionTerminate(Reason.of(e), e.getMessage());
417 return;
418 }
419
420 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
421 if (outgoingContentAdd == null) {
422 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
423 terminateWithOutOfOrder(jinglePacket);
424 return;
425 }
426 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
427 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
428 this.outgoingContentAdd = null;
429 respondOk(jinglePacket);
430 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
431 final ListenableFuture<RtpContentMap> future =
432 receiveRtpContentMap(
433 receivedContentAccept,
434 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
435 Futures.addCallback(
436 future,
437 new FutureCallback<>() {
438 @Override
439 public void onSuccess(final RtpContentMap result) {
440 receiveContentAccept(result);
441 }
442
443 @Override
444 public void onFailure(@NonNull final Throwable throwable) {
445 webRTCWrapper.close();
446 sendSessionTerminate(
447 Reason.ofThrowable(throwable), throwable.getMessage());
448 }
449 },
450 MoreExecutors.directExecutor());
451 } else {
452 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
453 terminateWithOutOfOrder(jinglePacket);
454 }
455 }
456
457 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
458 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
459 final RtpContentMap modifiedContentMap =
460 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
461
462 setRemoteContentMap(modifiedContentMap);
463
464 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
465
466 final org.webrtc.SessionDescription sdp =
467 new org.webrtc.SessionDescription(
468 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
469
470 try {
471 this.webRTCWrapper.setRemoteDescription(sdp).get();
472 } catch (final Exception e) {
473 final Throwable cause = Throwables.getRootCause(e);
474 Log.d(
475 Config.LOGTAG,
476 id.getAccount().getJid().asBareJid()
477 + ": unable to set remote description after receiving content-accept",
478 cause);
479 webRTCWrapper.close();
480 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
481 return;
482 }
483 Log.d(
484 Config.LOGTAG,
485 id.getAccount().getJid().asBareJid()
486 + ": remote has accepted content-add "
487 + ContentAddition.summary(receivedContentAccept));
488 processCandidates(receivedContentAccept.contents.entrySet());
489 updateEndUserState();
490 }
491
492 private void receiveContentModify(final JinglePacket jinglePacket) {
493 if (this.state != State.SESSION_ACCEPTED) {
494 terminateWithOutOfOrder(jinglePacket);
495 return;
496 }
497 final Map<String, Content.Senders> modification =
498 Maps.transformEntries(
499 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
500 final boolean isInitiator = isInitiator();
501 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
502 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
503 final Set<String> currentOutgoingMediaIds =
504 currentOutgoing == null
505 ? Collections.emptySet()
506 : currentOutgoing.contents.keySet();
507 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
508 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
509 respondOk(jinglePacket);
510 final RtpContentMap modifiedContentMap;
511 try {
512 modifiedContentMap =
513 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
514 } catch (final IllegalArgumentException e) {
515 webRTCWrapper.close();
516 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
517 return;
518 }
519 this.outgoingContentAdd = modifiedContentMap;
520 Log.d(
521 Config.LOGTAG,
522 id.account.getJid().asBareJid()
523 + ": processed content-modification for pending content-add");
524 } else if (remoteContentMap != null
525 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
526 respondOk(jinglePacket);
527 final RtpContentMap modifiedRemoteContentMap;
528 try {
529 modifiedRemoteContentMap =
530 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
531 } catch (final IllegalArgumentException e) {
532 webRTCWrapper.close();
533 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
534 return;
535 }
536 final SessionDescription offer;
537 try {
538 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
539 } catch (final IllegalArgumentException | NullPointerException e) {
540 Log.d(
541 Config.LOGTAG,
542 id.getAccount().getJid().asBareJid()
543 + ": unable convert offer from content-modify to SDP",
544 e);
545 webRTCWrapper.close();
546 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
547 return;
548 }
549 Log.d(
550 Config.LOGTAG,
551 id.account.getJid().asBareJid() + ": auto accepting content-modification");
552 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
553 } else {
554 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
555 respondWithItemNotFound(jinglePacket);
556 }
557 }
558
559 private void autoAcceptContentModify(
560 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
561 this.setRemoteContentMap(modifiedRemoteContentMap);
562 final org.webrtc.SessionDescription sdp =
563 new org.webrtc.SessionDescription(
564 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
565 try {
566 this.webRTCWrapper.setRemoteDescription(sdp).get();
567 // auto accept is only done when we already have tracks
568 final SessionDescription answer = setLocalSessionDescription();
569 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
570 modifyLocalContentMap(rtpContentMap);
571 // we do not need to send an answer but do we have to resend the candidates currently in
572 // SDP?
573 // resendCandidatesFromSdp(answer);
574 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
575 } catch (final Exception e) {
576 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
577 webRTCWrapper.close();
578 sendSessionTerminate(Reason.FAILED_APPLICATION);
579 }
580 }
581
582 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
583 final SessionDescription answer) {
584 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
585 new ImmutableMultimap.Builder<>();
586 for (final SessionDescription.Media media : answer.media) {
587 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
588 if (Strings.isNullOrEmpty(mid)) {
589 continue;
590 }
591 for (final String sdpCandidate : media.attributes.get("candidate")) {
592 final IceUdpTransportInfo.Candidate candidate =
593 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
594 if (candidate != null) {
595 candidateBuilder.put(mid, candidate);
596 }
597 }
598 }
599 return candidateBuilder.build();
600 }
601
602 private void receiveContentReject(final JinglePacket jinglePacket) {
603 final RtpContentMap receivedContentReject;
604 try {
605 receivedContentReject = RtpContentMap.of(jinglePacket);
606 } catch (final RuntimeException e) {
607 Log.d(
608 Config.LOGTAG,
609 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
610 Throwables.getRootCause(e));
611 respondOk(jinglePacket);
612 this.webRTCWrapper.close();
613 sendSessionTerminate(Reason.of(e), e.getMessage());
614 return;
615 }
616
617 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
618 if (outgoingContentAdd == null) {
619 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
620 terminateWithOutOfOrder(jinglePacket);
621 return;
622 }
623 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
624 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
625 this.outgoingContentAdd = null;
626 respondOk(jinglePacket);
627 Log.d(Config.LOGTAG, jinglePacket.toString());
628 receiveContentReject(ourSummary);
629 } else {
630 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
631 terminateWithOutOfOrder(jinglePacket);
632 }
633 }
634
635 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
636 try {
637 this.webRTCWrapper.removeTrack(Media.VIDEO);
638 final RtpContentMap localContentMap = customRollback();
639 modifyLocalContentMap(localContentMap);
640 } catch (final Exception e) {
641 final Throwable cause = Throwables.getRootCause(e);
642 Log.d(
643 Config.LOGTAG,
644 id.getAccount().getJid().asBareJid()
645 + ": unable to rollback local description after receiving content-reject",
646 cause);
647 webRTCWrapper.close();
648 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
649 return;
650 }
651 Log.d(
652 Config.LOGTAG,
653 id.getAccount().getJid().asBareJid()
654 + ": remote has rejected our content-add "
655 + summary);
656 }
657
658 private void receiveContentRemove(final JinglePacket jinglePacket) {
659 final RtpContentMap receivedContentRemove;
660 try {
661 receivedContentRemove = RtpContentMap.of(jinglePacket);
662 receivedContentRemove.requireContentDescriptions();
663 } catch (final RuntimeException e) {
664 Log.d(
665 Config.LOGTAG,
666 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
667 Throwables.getRootCause(e));
668 respondOk(jinglePacket);
669 this.webRTCWrapper.close();
670 sendSessionTerminate(Reason.of(e), e.getMessage());
671 return;
672 }
673 respondOk(jinglePacket);
674 receiveContentRemove(receivedContentRemove);
675 }
676
677 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
678 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
679 final Set<ContentAddition.Summary> contentAddSummary =
680 incomingContentAdd == null
681 ? Collections.emptySet()
682 : ContentAddition.summary(incomingContentAdd);
683 final Set<ContentAddition.Summary> removeSummary =
684 ContentAddition.summary(receivedContentRemove);
685 if (contentAddSummary.equals(removeSummary)) {
686 this.incomingContentAdd = null;
687 updateEndUserState();
688 } else {
689 webRTCWrapper.close();
690 sendSessionTerminate(
691 Reason.FAILED_APPLICATION,
692 String.format(
693 "%s only supports %s as a means to retract a not yet accepted %s",
694 BuildConfig.APP_NAME,
695 JinglePacket.Action.CONTENT_REMOVE,
696 JinglePacket.Action.CONTENT_ADD));
697 }
698 }
699
700 public synchronized void retractContentAdd() {
701 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
702 if (outgoingContentAdd == null) {
703 throw new IllegalStateException("Not outgoing content add");
704 }
705 try {
706 webRTCWrapper.removeTrack(Media.VIDEO);
707 final RtpContentMap localContentMap = customRollback();
708 modifyLocalContentMap(localContentMap);
709 } catch (final Exception e) {
710 final Throwable cause = Throwables.getRootCause(e);
711 Log.d(
712 Config.LOGTAG,
713 id.getAccount().getJid().asBareJid()
714 + ": unable to rollback local description after trying to retract content-add",
715 cause);
716 webRTCWrapper.close();
717 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
718 return;
719 }
720 this.outgoingContentAdd = null;
721 final JinglePacket retract =
722 outgoingContentAdd
723 .toStub()
724 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
725 this.send(retract);
726 Log.d(
727 Config.LOGTAG,
728 id.getAccount().getJid()
729 + ": retract content-add "
730 + ContentAddition.summary(outgoingContentAdd));
731 }
732
733 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
734 final SessionDescription sdp = setLocalSessionDescription();
735 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
736 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
737 this.webRTCWrapper
738 .setRemoteDescription(
739 new org.webrtc.SessionDescription(
740 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
741 .get();
742 return localRtpContentMap;
743 }
744
745 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
746 final RtpContentMap currentRemote = getRemoteContentMap();
747 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
748 if (diff.isEmpty()) {
749 throw new IllegalStateException(
750 "Unexpected rollback condition. No difference between local and remote");
751 }
752 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
753 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
754 final RtpContentMap nextRemote =
755 currentRemote.addContent(
756 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
757 return SessionDescription.of(nextRemote, isResponder());
758 }
759 throw new IllegalStateException(
760 "Unexpected rollback condition. Senders were not uniformly none");
761 }
762
763 public synchronized void acceptContentAdd(
764 @NonNull final Set<ContentAddition.Summary> contentAddition) {
765 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
766 if (incomingContentAdd == null) {
767 throw new IllegalStateException("No incoming content add");
768 }
769
770 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
771 this.incomingContentAdd = null;
772 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
773 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
774 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
775 Log.d(
776 Config.LOGTAG,
777 "content addition is receive only. we want to upgrade to 'both'");
778 final RtpContentMap modifiedSenders =
779 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
780 final JinglePacket proposedContentModification =
781 modifiedSenders
782 .toStub()
783 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
784 proposedContentModification.setTo(id.with);
785 xmppConnectionService.sendIqPacket(
786 id.account,
787 proposedContentModification,
788 (account, response) -> {
789 if (response.getType() == IqPacket.TYPE.RESULT) {
790 Log.d(
791 Config.LOGTAG,
792 id.account.getJid().asBareJid()
793 + ": remote has accepted our upgrade to senders=both");
794 acceptContentAdd(
795 ContentAddition.summary(modifiedSenders), modifiedSenders);
796 } else {
797 Log.d(
798 Config.LOGTAG,
799 id.account.getJid().asBareJid()
800 + ": remote has rejected our upgrade to senders=both");
801 acceptContentAdd(contentAddition, incomingContentAdd);
802 }
803 });
804 } else {
805 acceptContentAdd(contentAddition, incomingContentAdd);
806 }
807 } else {
808 throw new IllegalStateException(
809 "Accepted content add does not match pending content-add");
810 }
811 }
812
813 private void acceptContentAdd(
814 @NonNull final Set<ContentAddition.Summary> contentAddition,
815 final RtpContentMap incomingContentAdd) {
816 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
817 final RtpContentMap modifiedContentMap =
818 getRemoteContentMap().addContent(incomingContentAdd, setup);
819 this.setRemoteContentMap(modifiedContentMap);
820
821 final SessionDescription offer;
822 try {
823 offer = SessionDescription.of(modifiedContentMap, isResponder());
824 } catch (final IllegalArgumentException | NullPointerException e) {
825 Log.d(
826 Config.LOGTAG,
827 id.getAccount().getJid().asBareJid()
828 + ": unable convert offer from content-add to SDP",
829 e);
830 webRTCWrapper.close();
831 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
832 return;
833 }
834 this.incomingContentAdd = null;
835 acceptContentAdd(contentAddition, offer);
836 }
837
838 private void acceptContentAdd(
839 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
840 final org.webrtc.SessionDescription sdp =
841 new org.webrtc.SessionDescription(
842 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
843 try {
844 this.webRTCWrapper.setRemoteDescription(sdp).get();
845
846 // TODO add tracks for 'media' where contentAddition.senders matches
847
848 // TODO if senders.sending(isInitiator())
849
850 this.webRTCWrapper.addTrack(Media.VIDEO);
851
852 // TODO add additional transceivers for recv only cases
853
854 final SessionDescription answer = setLocalSessionDescription();
855 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
856
857 final RtpContentMap contentAcceptMap =
858 rtpContentMap.toContentModification(
859 Collections2.transform(contentAddition, ca -> ca.name));
860
861 Log.d(
862 Config.LOGTAG,
863 id.getAccount().getJid().asBareJid()
864 + ": sending content-accept "
865 + ContentAddition.summary(contentAcceptMap));
866
867 addIceCandidatesFromBlackLog();
868
869 modifyLocalContentMap(rtpContentMap);
870 final ListenableFuture<RtpContentMap> future =
871 prepareOutgoingContentMap(contentAcceptMap);
872 Futures.addCallback(
873 future,
874 new FutureCallback<>() {
875 @Override
876 public void onSuccess(final RtpContentMap rtpContentMap) {
877 sendContentAccept(rtpContentMap);
878 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
879 }
880
881 @Override
882 public void onFailure(@NonNull final Throwable throwable) {
883 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
884 }
885 },
886 MoreExecutors.directExecutor());
887 } catch (final Exception e) {
888 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
889 webRTCWrapper.close();
890 sendSessionTerminate(Reason.FAILED_APPLICATION);
891 }
892 }
893
894 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
895 final JinglePacket jinglePacket =
896 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
897 send(jinglePacket);
898 }
899
900 public synchronized void rejectContentAdd() {
901 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
902 if (incomingContentAdd == null) {
903 throw new IllegalStateException("No incoming content add");
904 }
905 this.incomingContentAdd = null;
906 updateEndUserState();
907 rejectContentAdd(incomingContentAdd);
908 }
909
910 private void rejectContentAdd(final RtpContentMap contentMap) {
911 final JinglePacket jinglePacket =
912 contentMap
913 .toStub()
914 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
915 Log.d(
916 Config.LOGTAG,
917 id.getAccount().getJid().asBareJid()
918 + ": rejecting content "
919 + ContentAddition.summary(contentMap));
920 send(jinglePacket);
921 }
922
923 private boolean checkForIceRestart(
924 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
925 final RtpContentMap existing = getRemoteContentMap();
926 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
927 final IceUdpTransportInfo.Credentials newCredentials;
928 try {
929 existingCredentials = existing.getCredentials();
930 newCredentials = rtpContentMap.getDistinctCredentials();
931 } catch (final IllegalStateException e) {
932 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
933 return false;
934 }
935 if (existingCredentials.contains(newCredentials)) {
936 return false;
937 }
938 // TODO an alternative approach is to check if we already got an iq result to our
939 // ICE-restart
940 // and if that's the case we are seeing an answer.
941 // This might be more spec compliant but also more error prone potentially
942 final boolean isSignalStateStable =
943 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
944 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
945 final boolean isOffer = rtpContentMap.emptyCandidates();
946 final RtpContentMap restartContentMap;
947 try {
948 if (isOffer) {
949 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
950 restartContentMap =
951 existing.modifiedCredentials(
952 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
953 } else {
954 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
955 Log.d(
956 Config.LOGTAG,
957 "received confirmation of ICE restart"
958 + newCredentials
959 + " peer_setup="
960 + setup);
961 // DTLS setup attribute needs to be rewritten to reflect current peer state
962 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
963 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
964 }
965 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
966 return isOffer;
967 } else {
968 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
969 respondWithTieBreak(jinglePacket);
970 return true;
971 }
972 } catch (final Exception exception) {
973 respondOk(jinglePacket);
974 final Throwable rootCause = Throwables.getRootCause(exception);
975 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
976 // If this happens a termination is already in progress
977 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
978 return true;
979 }
980 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
981 webRTCWrapper.close();
982 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
983 return true;
984 }
985 }
986
987 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
988 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
989 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
990 throw new IllegalStateException("Invalid peer setup");
991 }
992 return peerSetup;
993 }
994
995 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
996 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
997 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
998 }
999 this.peerDtlsSetup = setup;
1000 }
1001
1002 private boolean applyIceRestart(
1003 final JinglePacket jinglePacket,
1004 final RtpContentMap restartContentMap,
1005 final boolean isOffer)
1006 throws ExecutionException, InterruptedException {
1007 final SessionDescription sessionDescription =
1008 SessionDescription.of(restartContentMap, isResponder());
1009 final org.webrtc.SessionDescription.Type type =
1010 isOffer
1011 ? org.webrtc.SessionDescription.Type.OFFER
1012 : org.webrtc.SessionDescription.Type.ANSWER;
1013 org.webrtc.SessionDescription sdp =
1014 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1015 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1016 if (isInitiator()) {
1017 // We ignore the offer and respond with tie-break. This will clause the responder
1018 // not to apply the content map
1019 return false;
1020 }
1021 }
1022 webRTCWrapper.setRemoteDescription(sdp).get();
1023 setRemoteContentMap(restartContentMap);
1024 if (isOffer) {
1025 final SessionDescription localSessionDescription = setLocalSessionDescription();
1026 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1027 // We need to respond OK before sending any candidates
1028 respondOk(jinglePacket);
1029 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1030 } else {
1031 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1032 }
1033 return true;
1034 }
1035
1036 private void processCandidates(
1037 final Set<Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>>
1038 contents) {
1039 for (final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1040 content : contents) {
1041 processCandidate(content);
1042 }
1043 }
1044
1045 private void processCandidate(
1046 final Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>>
1047 content) {
1048 final RtpContentMap rtpContentMap = getRemoteContentMap();
1049 final List<String> indices = toIdentificationTags(rtpContentMap);
1050 final String sdpMid = content.getKey(); // aka content name
1051 final IceUdpTransportInfo transport = content.getValue().transport;
1052 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1053
1054 // TODO check that credentials remained the same
1055
1056 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1057 final String sdp;
1058 try {
1059 sdp = candidate.toSdpAttribute(credentials.ufrag);
1060 } catch (final IllegalArgumentException e) {
1061 Log.d(
1062 Config.LOGTAG,
1063 id.account.getJid().asBareJid()
1064 + ": ignoring invalid ICE candidate "
1065 + e.getMessage());
1066 continue;
1067 }
1068 final int mLineIndex = indices.indexOf(sdpMid);
1069 if (mLineIndex < 0) {
1070 Log.w(
1071 Config.LOGTAG,
1072 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1073 }
1074 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1075 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1076 this.webRTCWrapper.addIceCandidate(iceCandidate);
1077 }
1078 }
1079
1080 private RtpContentMap getRemoteContentMap() {
1081 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1082 }
1083
1084 private RtpContentMap getLocalContentMap() {
1085 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1086 }
1087
1088 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1089 final Group originalGroup = rtpContentMap.group;
1090 final List<String> identificationTags =
1091 originalGroup == null
1092 ? rtpContentMap.getNames()
1093 : originalGroup.getIdentificationTags();
1094 if (identificationTags.size() == 0) {
1095 Log.w(
1096 Config.LOGTAG,
1097 id.account.getJid().asBareJid()
1098 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1099 }
1100 return identificationTags;
1101 }
1102
1103 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1104 final JinglePacket jinglePacket, final boolean expectVerification) {
1105 try {
1106 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1107 } catch (final Exception e) {
1108 return Futures.immediateFailedFuture(e);
1109 }
1110 }
1111
1112 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1113 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1114 Log.d(
1115 Config.LOGTAG,
1116 "receiveRtpContentMap("
1117 + receivedContentMap.getClass().getSimpleName()
1118 + ",expectVerification="
1119 + expectVerification
1120 + ")");
1121 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1122 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1123 id.account
1124 .getAxolotlService()
1125 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1126 return Futures.transform(
1127 future,
1128 omemoVerifiedPayload -> {
1129 // TODO test if an exception here triggers a correct abort
1130 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1131 Log.d(
1132 Config.LOGTAG,
1133 id.account.getJid().asBareJid()
1134 + ": received verifiable DTLS fingerprint via "
1135 + omemoVerification);
1136 return omemoVerifiedPayload.getPayload();
1137 },
1138 MoreExecutors.directExecutor());
1139 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1140 return Futures.immediateFailedFuture(
1141 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1142 } else {
1143 return Futures.immediateFuture(receivedContentMap);
1144 }
1145 }
1146
1147 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1148 if (isInitiator()) {
1149 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1150 return;
1151 }
1152 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1153 Futures.addCallback(
1154 future,
1155 new FutureCallback<>() {
1156 @Override
1157 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1158 receiveSessionInitiate(jinglePacket, rtpContentMap);
1159 }
1160
1161 @Override
1162 public void onFailure(@NonNull final Throwable throwable) {
1163 respondOk(jinglePacket);
1164 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1165 }
1166 },
1167 MoreExecutors.directExecutor());
1168 }
1169
1170 private void receiveSessionInitiate(
1171 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1172 try {
1173 contentMap.requireContentDescriptions();
1174 contentMap.requireDTLSFingerprint(true);
1175 } catch (final RuntimeException e) {
1176 Log.d(
1177 Config.LOGTAG,
1178 id.account.getJid().asBareJid() + ": improperly formatted contents",
1179 Throwables.getRootCause(e));
1180 respondOk(jinglePacket);
1181 sendSessionTerminate(Reason.of(e), e.getMessage());
1182 return;
1183 }
1184 Log.d(
1185 Config.LOGTAG,
1186 "processing session-init with " + contentMap.contents.size() + " contents");
1187 final State target;
1188 if (this.state == State.PROCEED) {
1189 Preconditions.checkState(
1190 proposedMedia != null && proposedMedia.size() > 0,
1191 "proposed media must be set when processing pre-approved session-initiate");
1192 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1193 sendSessionTerminate(
1194 Reason.SECURITY_ERROR,
1195 String.format(
1196 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1197 this.proposedMedia, contentMap.getMedia()));
1198 return;
1199 }
1200 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1201 } else {
1202 target = State.SESSION_INITIALIZED;
1203 setProposedMedia(contentMap.getMedia());
1204 }
1205 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1206 respondOk(jinglePacket);
1207 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1208 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1209 Log.d(
1210 Config.LOGTAG,
1211 id.account.getJid().asBareJid()
1212 + ": automatically accepting session-initiate");
1213 sendSessionAccept();
1214 } else {
1215 Log.d(
1216 Config.LOGTAG,
1217 id.account.getJid().asBareJid()
1218 + ": received not pre-approved session-initiate. start ringing");
1219 startRinging();
1220 }
1221 } else {
1222 Log.d(
1223 Config.LOGTAG,
1224 String.format(
1225 "%s: received session-initiate while in state %s",
1226 id.account.getJid().asBareJid(), state));
1227 terminateWithOutOfOrder(jinglePacket);
1228 }
1229 }
1230
1231 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1232 if (isResponder()) {
1233 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1234 return;
1235 }
1236 final ListenableFuture<RtpContentMap> future =
1237 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1238 Futures.addCallback(
1239 future,
1240 new FutureCallback<>() {
1241 @Override
1242 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1243 receiveSessionAccept(jinglePacket, rtpContentMap);
1244 }
1245
1246 @Override
1247 public void onFailure(@NonNull final Throwable throwable) {
1248 respondOk(jinglePacket);
1249 Log.d(
1250 Config.LOGTAG,
1251 id.account.getJid().asBareJid()
1252 + ": improperly formatted contents in session-accept",
1253 throwable);
1254 webRTCWrapper.close();
1255 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1256 }
1257 },
1258 MoreExecutors.directExecutor());
1259 }
1260
1261 private void receiveSessionAccept(
1262 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1263 try {
1264 contentMap.requireContentDescriptions();
1265 contentMap.requireDTLSFingerprint();
1266 } catch (final RuntimeException e) {
1267 respondOk(jinglePacket);
1268 Log.d(
1269 Config.LOGTAG,
1270 id.account.getJid().asBareJid()
1271 + ": improperly formatted contents in session-accept",
1272 e);
1273 webRTCWrapper.close();
1274 sendSessionTerminate(Reason.of(e), e.getMessage());
1275 return;
1276 }
1277 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1278 if (!initiatorMedia.equals(contentMap.getMedia())) {
1279 sendSessionTerminate(
1280 Reason.SECURITY_ERROR,
1281 String.format(
1282 "Your session-included included media %s but our session-initiate was %s",
1283 this.proposedMedia, contentMap.getMedia()));
1284 return;
1285 }
1286 Log.d(
1287 Config.LOGTAG,
1288 "processing session-accept with " + contentMap.contents.size() + " contents");
1289 if (transition(State.SESSION_ACCEPTED)) {
1290 respondOk(jinglePacket);
1291 receiveSessionAccept(contentMap);
1292 } else {
1293 Log.d(
1294 Config.LOGTAG,
1295 String.format(
1296 "%s: received session-accept while in state %s",
1297 id.account.getJid().asBareJid(), state));
1298 respondOk(jinglePacket);
1299 }
1300 }
1301
1302 private void receiveSessionAccept(final RtpContentMap contentMap) {
1303 this.responderRtpContentMap = contentMap;
1304 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1305 final SessionDescription sessionDescription;
1306 try {
1307 sessionDescription = SessionDescription.of(contentMap, false);
1308 } catch (final IllegalArgumentException | NullPointerException e) {
1309 Log.d(
1310 Config.LOGTAG,
1311 id.account.getJid().asBareJid()
1312 + ": unable convert offer from session-accept to SDP",
1313 e);
1314 webRTCWrapper.close();
1315 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1316 return;
1317 }
1318 final org.webrtc.SessionDescription answer =
1319 new org.webrtc.SessionDescription(
1320 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1321 try {
1322 this.webRTCWrapper.setRemoteDescription(answer).get();
1323 } catch (final Exception e) {
1324 Log.d(
1325 Config.LOGTAG,
1326 id.account.getJid().asBareJid()
1327 + ": unable to set remote description after receiving session-accept",
1328 Throwables.getRootCause(e));
1329 webRTCWrapper.close();
1330 sendSessionTerminate(
1331 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1332 return;
1333 }
1334 processCandidates(contentMap.contents.entrySet());
1335 }
1336
1337 private void sendSessionAccept() {
1338 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1339 if (rtpContentMap == null) {
1340 throw new IllegalStateException("initiator RTP Content Map has not been set");
1341 }
1342 final SessionDescription offer;
1343 try {
1344 offer = SessionDescription.of(rtpContentMap, true);
1345 } catch (final IllegalArgumentException | NullPointerException e) {
1346 Log.d(
1347 Config.LOGTAG,
1348 id.account.getJid().asBareJid()
1349 + ": unable convert offer from session-initiate to SDP",
1350 e);
1351 webRTCWrapper.close();
1352 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1353 return;
1354 }
1355 sendSessionAccept(rtpContentMap.getMedia(), offer);
1356 }
1357
1358 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1359 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1360 }
1361
1362 private synchronized void sendSessionAccept(
1363 final Set<Media> media,
1364 final SessionDescription offer,
1365 final List<PeerConnection.IceServer> iceServers) {
1366 if (isTerminated()) {
1367 Log.w(
1368 Config.LOGTAG,
1369 id.account.getJid().asBareJid()
1370 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1371 return;
1372 }
1373 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1374 try {
1375 setupWebRTC(media, iceServers, !includeCandidates);
1376 } catch (final WebRTCWrapper.InitializationException e) {
1377 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1378 webRTCWrapper.close();
1379 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1380 return;
1381 }
1382 final org.webrtc.SessionDescription sdp =
1383 new org.webrtc.SessionDescription(
1384 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1385 try {
1386 this.webRTCWrapper.setRemoteDescription(sdp).get();
1387 addIceCandidatesFromBlackLog();
1388 org.webrtc.SessionDescription webRTCSessionDescription =
1389 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1390 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1391 } catch (final Exception e) {
1392 failureToAcceptSession(e);
1393 }
1394 }
1395
1396 private void failureToAcceptSession(final Throwable throwable) {
1397 if (isTerminated()) {
1398 return;
1399 }
1400 final Throwable rootCause = Throwables.getRootCause(throwable);
1401 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1402 webRTCWrapper.close();
1403 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1404 }
1405
1406 private void failureToPerformAction(
1407 final JinglePacket.Action action, final Throwable throwable) {
1408 if (isTerminated()) {
1409 return;
1410 }
1411 final Throwable rootCause = Throwables.getRootCause(throwable);
1412 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1413 webRTCWrapper.close();
1414 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1415 }
1416
1417 private void addIceCandidatesFromBlackLog() {
1418 Map.Entry<String, DescriptionTransport<RtpDescription, IceUdpTransportInfo>> foo;
1419 while ((foo = this.pendingIceCandidates.poll()) != null) {
1420 processCandidate(foo);
1421 Log.d(
1422 Config.LOGTAG,
1423 id.account.getJid().asBareJid() + ": added candidate from back log");
1424 }
1425 }
1426
1427 private void prepareSessionAccept(
1428 final org.webrtc.SessionDescription webRTCSessionDescription,
1429 final boolean includeCandidates) {
1430 final SessionDescription sessionDescription =
1431 SessionDescription.parse(webRTCSessionDescription.description);
1432 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1433 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1434 if (includeCandidates) {
1435 candidates = parseCandidates(sessionDescription);
1436 } else {
1437 candidates = ImmutableMultimap.of();
1438 }
1439 this.responderRtpContentMap = respondingRtpContentMap;
1440 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1441 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1442 prepareOutgoingContentMap(respondingRtpContentMap);
1443 Futures.addCallback(
1444 outgoingContentMapFuture,
1445 new FutureCallback<>() {
1446 @Override
1447 public void onSuccess(final RtpContentMap outgoingContentMap) {
1448 if (includeCandidates) {
1449 Log.d(
1450 Config.LOGTAG,
1451 "including "
1452 + candidates.size()
1453 + " candidates in session accept");
1454 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1455 } else {
1456 sendSessionAccept(outgoingContentMap);
1457 }
1458 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1459 }
1460
1461 @Override
1462 public void onFailure(@NonNull Throwable throwable) {
1463 failureToAcceptSession(throwable);
1464 }
1465 },
1466 MoreExecutors.directExecutor());
1467 }
1468
1469 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1470 if (isTerminated()) {
1471 Log.w(
1472 Config.LOGTAG,
1473 id.account.getJid().asBareJid()
1474 + ": preparing session accept was too slow. already terminated. nothing to do.");
1475 return;
1476 }
1477 transitionOrThrow(State.SESSION_ACCEPTED);
1478 final JinglePacket sessionAccept =
1479 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1480 send(sessionAccept);
1481 }
1482
1483 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1484 final RtpContentMap rtpContentMap) {
1485 if (this.omemoVerification.hasDeviceId()) {
1486 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1487 verifiedPayloadFuture =
1488 id.account
1489 .getAxolotlService()
1490 .encrypt(
1491 rtpContentMap,
1492 id.with,
1493 omemoVerification.getDeviceId());
1494 return Futures.transform(
1495 verifiedPayloadFuture,
1496 verifiedPayload -> {
1497 omemoVerification.setOrEnsureEqual(verifiedPayload);
1498 return verifiedPayload.getPayload();
1499 },
1500 MoreExecutors.directExecutor());
1501 } else {
1502 return Futures.immediateFuture(rtpContentMap);
1503 }
1504 }
1505
1506 synchronized void deliveryMessage(
1507 final Jid from,
1508 final Element message,
1509 final String serverMessageId,
1510 final long timestamp) {
1511 Log.d(
1512 Config.LOGTAG,
1513 id.account.getJid().asBareJid()
1514 + ": delivered message to JingleRtpConnection "
1515 + message);
1516 switch (message.getName()) {
1517 case "propose" -> receivePropose(
1518 from, Propose.upgrade(message), serverMessageId, timestamp);
1519 case "proceed" -> receiveProceed(
1520 from, Proceed.upgrade(message), serverMessageId, timestamp);
1521 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1522 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1523 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1524 }
1525 }
1526
1527 void deliverFailedProceed(final String message) {
1528 Log.d(
1529 Config.LOGTAG,
1530 id.account.getJid().asBareJid()
1531 + ": receive message error for proceed message ("
1532 + Strings.nullToEmpty(message)
1533 + ")");
1534 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1535 webRTCWrapper.close();
1536 Log.d(
1537 Config.LOGTAG,
1538 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1539 this.finish();
1540 }
1541 }
1542
1543 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1544 final boolean originatedFromMyself =
1545 from.asBareJid().equals(id.account.getJid().asBareJid());
1546 if (originatedFromMyself) {
1547 if (transition(State.ACCEPTED)) {
1548 acceptedOnOtherDevice(serverMsgId, timestamp);
1549 } else {
1550 Log.d(
1551 Config.LOGTAG,
1552 id.account.getJid().asBareJid()
1553 + ": unable to transition to accept because already in state="
1554 + this.state);
1555 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1556 }
1557 } else {
1558 Log.d(
1559 Config.LOGTAG,
1560 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1561 }
1562 }
1563
1564 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1565 if (serverMsgId != null) {
1566 this.message.setServerMsgId(serverMsgId);
1567 }
1568 this.message.setTime(timestamp);
1569 this.message.setCarbon(true); // indicate that call was accepted on other device
1570 this.writeLogMessageSuccess(0);
1571 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1572 this.finish();
1573 }
1574
1575 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1576 final boolean originatedFromMyself =
1577 from.asBareJid().equals(id.account.getJid().asBareJid());
1578 // reject from another one of my clients
1579 if (originatedFromMyself) {
1580 receiveRejectFromMyself(serverMsgId, timestamp);
1581 } else if (isInitiator()) {
1582 if (from.equals(id.with)) {
1583 receiveRejectFromResponder();
1584 } else {
1585 Log.d(
1586 Config.LOGTAG,
1587 id.account.getJid()
1588 + ": ignoring reject from "
1589 + from
1590 + " for session with "
1591 + id.with);
1592 }
1593 } else {
1594 Log.d(
1595 Config.LOGTAG,
1596 id.account.getJid()
1597 + ": ignoring reject from "
1598 + from
1599 + " for session with "
1600 + id.with);
1601 }
1602 }
1603
1604 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1605 if (transition(State.REJECTED)) {
1606 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1607 this.finish();
1608 if (serverMsgId != null) {
1609 this.message.setServerMsgId(serverMsgId);
1610 }
1611 this.message.setTime(timestamp);
1612 this.message.setCarbon(true); // indicate that call was rejected on other device
1613 writeLogMessageMissed();
1614 } else {
1615 Log.d(
1616 Config.LOGTAG,
1617 "not able to transition into REJECTED because already in " + this.state);
1618 }
1619 }
1620
1621 private void receiveRejectFromResponder() {
1622 if (isInState(State.PROCEED)) {
1623 Log.d(
1624 Config.LOGTAG,
1625 id.account.getJid()
1626 + ": received reject while still in proceed. callee reconsidered");
1627 closeTransitionLogFinish(State.REJECTED_RACED);
1628 return;
1629 }
1630 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1631 Log.d(
1632 Config.LOGTAG,
1633 id.account.getJid()
1634 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1635 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1636 return;
1637 }
1638 Log.d(
1639 Config.LOGTAG,
1640 id.account.getJid()
1641 + ": ignoring reject from responder because already in state "
1642 + this.state);
1643 }
1644
1645 private void receivePropose(
1646 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1647 final boolean originatedFromMyself =
1648 from.asBareJid().equals(id.account.getJid().asBareJid());
1649 if (originatedFromMyself) {
1650 Log.d(
1651 Config.LOGTAG,
1652 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1653 } else if (transition(
1654 State.PROPOSED,
1655 () -> {
1656 final Collection<RtpDescription> descriptions =
1657 Collections2.transform(
1658 Collections2.filter(
1659 propose.getDescriptions(),
1660 d -> d instanceof RtpDescription),
1661 input -> (RtpDescription) input);
1662 final Collection<Media> media =
1663 Collections2.transform(descriptions, RtpDescription::getMedia);
1664 Preconditions.checkState(
1665 !media.contains(Media.UNKNOWN),
1666 "RTP descriptions contain unknown media");
1667 Log.d(
1668 Config.LOGTAG,
1669 id.account.getJid().asBareJid()
1670 + ": received session proposal from "
1671 + from
1672 + " for "
1673 + media);
1674 this.setProposedMedia(Sets.newHashSet(media));
1675 })) {
1676 if (serverMsgId != null) {
1677 this.message.setServerMsgId(serverMsgId);
1678 }
1679 this.message.setTime(timestamp);
1680 startRinging();
1681 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1682 sendJingleMessage("ringing");
1683 }
1684 } else {
1685 Log.d(
1686 Config.LOGTAG,
1687 id.account.getJid()
1688 + ": ignoring session proposal because already in "
1689 + state);
1690 }
1691 }
1692
1693 private void startRinging() {
1694 this.callIntegration.setRinging();
1695 Log.d(
1696 Config.LOGTAG,
1697 id.account.getJid().asBareJid()
1698 + ": received call from "
1699 + id.with
1700 + ". start ringing");
1701 ringingTimeoutFuture =
1702 jingleConnectionManager.schedule(
1703 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1704 if (CallIntegration.selfManaged(xmppConnectionService)) {
1705 return;
1706 }
1707 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1708 }
1709
1710 private synchronized void ringingTimeout() {
1711 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1712 switch (this.state) {
1713 case PROPOSED -> {
1714 message.markUnread();
1715 rejectCallFromProposed();
1716 }
1717 case SESSION_INITIALIZED -> {
1718 message.markUnread();
1719 rejectCallFromSessionInitiate();
1720 }
1721 }
1722 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1723 }
1724
1725 private void cancelRingingTimeout() {
1726 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1727 if (future != null && !future.isCancelled()) {
1728 future.cancel(false);
1729 }
1730 }
1731
1732 private void receiveProceed(
1733 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1734 final Set<Media> media =
1735 Preconditions.checkNotNull(
1736 this.proposedMedia, "Proposed media has to be set before handling proceed");
1737 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1738 if (from.equals(id.with)) {
1739 if (isInitiator()) {
1740 if (transition(State.PROCEED)) {
1741 if (serverMsgId != null) {
1742 this.message.setServerMsgId(serverMsgId);
1743 }
1744 this.message.setTime(timestamp);
1745 final Integer remoteDeviceId = proceed.getDeviceId();
1746 if (isOmemoEnabled()) {
1747 this.omemoVerification.setDeviceId(remoteDeviceId);
1748 } else {
1749 if (remoteDeviceId != null) {
1750 Log.d(
1751 Config.LOGTAG,
1752 id.account.getJid().asBareJid()
1753 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1754 }
1755 this.omemoVerification.setDeviceId(null);
1756 }
1757 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1758 } else {
1759 Log.d(
1760 Config.LOGTAG,
1761 String.format(
1762 "%s: ignoring proceed because already in %s",
1763 id.account.getJid().asBareJid(), this.state));
1764 }
1765 } else {
1766 Log.d(
1767 Config.LOGTAG,
1768 String.format(
1769 "%s: ignoring proceed because we were not initializing",
1770 id.account.getJid().asBareJid()));
1771 }
1772 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1773 if (transition(State.ACCEPTED)) {
1774 Log.d(
1775 Config.LOGTAG,
1776 id.account.getJid().asBareJid()
1777 + ": moved session with "
1778 + id.with
1779 + " into state accepted after received carbon copied proceed");
1780 acceptedOnOtherDevice(serverMsgId, timestamp);
1781 }
1782 } else {
1783 Log.d(
1784 Config.LOGTAG,
1785 String.format(
1786 "%s: ignoring proceed from %s. was expected from %s",
1787 id.account.getJid().asBareJid(), from, id.with));
1788 }
1789 }
1790
1791 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1792 if (from.equals(id.with)) {
1793 final State target =
1794 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1795 if (transition(target)) {
1796 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1797 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1798 Log.d(
1799 Config.LOGTAG,
1800 id.account.getJid().asBareJid()
1801 + ": session with "
1802 + id.with
1803 + " has been retracted (serverMsgId="
1804 + serverMsgId
1805 + ")");
1806 if (serverMsgId != null) {
1807 this.message.setServerMsgId(serverMsgId);
1808 }
1809 this.message.setTime(timestamp);
1810 if (target == State.RETRACTED) {
1811 this.message.markUnread();
1812 }
1813 writeLogMessageMissed();
1814 finish();
1815 } else {
1816 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1817 }
1818 } else {
1819 // TODO parse retract from self
1820 Log.d(
1821 Config.LOGTAG,
1822 id.account.getJid().asBareJid()
1823 + ": received retract from "
1824 + from
1825 + ". expected retract from"
1826 + id.with
1827 + ". ignoring");
1828 }
1829 }
1830
1831 public void sendSessionInitiate() {
1832 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1833 }
1834
1835 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1836 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1837 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1838 }
1839
1840 private synchronized void sendSessionInitiate(
1841 final Set<Media> media,
1842 final State targetState,
1843 final List<PeerConnection.IceServer> iceServers) {
1844 if (isTerminated()) {
1845 Log.w(
1846 Config.LOGTAG,
1847 id.account.getJid().asBareJid()
1848 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1849 return;
1850 }
1851 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1852 try {
1853 setupWebRTC(media, iceServers, !includeCandidates);
1854 } catch (final WebRTCWrapper.InitializationException e) {
1855 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1856 webRTCWrapper.close();
1857 sendRetract(Reason.ofThrowable(e));
1858 return;
1859 }
1860 try {
1861 org.webrtc.SessionDescription webRTCSessionDescription =
1862 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1863 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1864 } catch (final Exception e) {
1865 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1866 // exceptions
1867 failureToInitiateSession(e, targetState);
1868 }
1869 }
1870
1871 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1872 if (isTerminated()) {
1873 return;
1874 }
1875 Log.d(
1876 Config.LOGTAG,
1877 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1878 Throwables.getRootCause(throwable));
1879 webRTCWrapper.close();
1880 final Reason reason = Reason.ofThrowable(throwable);
1881 if (isInState(targetState)) {
1882 sendSessionTerminate(reason, throwable.getMessage());
1883 } else {
1884 sendRetract(reason);
1885 }
1886 }
1887
1888 private void sendRetract(final Reason reason) {
1889 // TODO embed reason into retract
1890 sendJingleMessage("retract", id.with.asBareJid());
1891 transitionOrThrow(reasonToState(reason));
1892 this.finish();
1893 }
1894
1895 private void prepareSessionInitiate(
1896 final org.webrtc.SessionDescription webRTCSessionDescription,
1897 final boolean includeCandidates,
1898 final State targetState) {
1899 final SessionDescription sessionDescription =
1900 SessionDescription.parse(webRTCSessionDescription.description);
1901 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1902 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1903 if (includeCandidates) {
1904 candidates = parseCandidates(sessionDescription);
1905 } else {
1906 candidates = ImmutableMultimap.of();
1907 }
1908 this.initiatorRtpContentMap = rtpContentMap;
1909 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1910 encryptSessionInitiate(rtpContentMap);
1911 Futures.addCallback(
1912 outgoingContentMapFuture,
1913 new FutureCallback<>() {
1914 @Override
1915 public void onSuccess(final RtpContentMap outgoingContentMap) {
1916 if (includeCandidates) {
1917 Log.d(
1918 Config.LOGTAG,
1919 "including "
1920 + candidates.size()
1921 + " candidates in session initiate");
1922 sendSessionInitiate(
1923 outgoingContentMap.withCandidates(candidates), targetState);
1924 } else {
1925 sendSessionInitiate(outgoingContentMap, targetState);
1926 }
1927 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1928 }
1929
1930 @Override
1931 public void onFailure(@NonNull final Throwable throwable) {
1932 failureToInitiateSession(throwable, targetState);
1933 }
1934 },
1935 MoreExecutors.directExecutor());
1936 }
1937
1938 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1939 if (isTerminated()) {
1940 Log.w(
1941 Config.LOGTAG,
1942 id.account.getJid().asBareJid()
1943 + ": preparing session was too slow. already terminated. nothing to do.");
1944 return;
1945 }
1946 this.transitionOrThrow(targetState);
1947 final JinglePacket sessionInitiate =
1948 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1949 send(sessionInitiate);
1950 }
1951
1952 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1953 final RtpContentMap rtpContentMap) {
1954 if (this.omemoVerification.hasDeviceId()) {
1955 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1956 verifiedPayloadFuture =
1957 id.account
1958 .getAxolotlService()
1959 .encrypt(
1960 rtpContentMap,
1961 id.with,
1962 omemoVerification.getDeviceId());
1963 final ListenableFuture<RtpContentMap> future =
1964 Futures.transform(
1965 verifiedPayloadFuture,
1966 verifiedPayload -> {
1967 omemoVerification.setSessionFingerprint(
1968 verifiedPayload.getFingerprint());
1969 return verifiedPayload.getPayload();
1970 },
1971 MoreExecutors.directExecutor());
1972 if (Config.REQUIRE_RTP_VERIFICATION) {
1973 return future;
1974 }
1975 return Futures.catching(
1976 future,
1977 CryptoFailedException.class,
1978 e -> {
1979 Log.w(
1980 Config.LOGTAG,
1981 id.account.getJid().asBareJid()
1982 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1983 e);
1984 return rtpContentMap;
1985 },
1986 MoreExecutors.directExecutor());
1987 } else {
1988 return Futures.immediateFuture(rtpContentMap);
1989 }
1990 }
1991
1992 protected void sendSessionTerminate(final Reason reason) {
1993 sendSessionTerminate(reason, null);
1994 }
1995
1996 protected void sendSessionTerminate(final Reason reason, final String text) {
1997 sendSessionTerminate(reason, text, this::writeLogMessage);
1998 sendJingleMessageFinish(reason);
1999 }
2000
2001 private void sendTransportInfo(
2002 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2003 final RtpContentMap transportInfo;
2004 try {
2005 final RtpContentMap rtpContentMap =
2006 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2007 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2008 } catch (final Exception e) {
2009 Log.d(
2010 Config.LOGTAG,
2011 id.account.getJid().asBareJid()
2012 + ": unable to prepare transport-info from candidate for content="
2013 + contentName);
2014 return;
2015 }
2016 final JinglePacket jinglePacket =
2017 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2018 send(jinglePacket);
2019 }
2020
2021 public RtpEndUserState getEndUserState() {
2022 switch (this.state) {
2023 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2024 if (isInitiator()) {
2025 return RtpEndUserState.RINGING;
2026 } else {
2027 return RtpEndUserState.INCOMING_CALL;
2028 }
2029 }
2030 case PROCEED -> {
2031 if (isInitiator()) {
2032 return RtpEndUserState.RINGING;
2033 } else {
2034 return RtpEndUserState.ACCEPTING_CALL;
2035 }
2036 }
2037 case SESSION_INITIALIZED_PRE_APPROVED -> {
2038 if (isInitiator()) {
2039 return RtpEndUserState.RINGING;
2040 } else {
2041 return RtpEndUserState.CONNECTING;
2042 }
2043 }
2044 case SESSION_ACCEPTED -> {
2045 final ContentAddition ca = getPendingContentAddition();
2046 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2047 return RtpEndUserState.INCOMING_CONTENT_ADD;
2048 }
2049 return getPeerConnectionStateAsEndUserState();
2050 }
2051 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2052 if (isInitiator()) {
2053 return RtpEndUserState.DECLINED_OR_BUSY;
2054 } else {
2055 return RtpEndUserState.ENDED;
2056 }
2057 }
2058 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2059 return RtpEndUserState.ENDED;
2060 }
2061 case RETRACTED_RACED -> {
2062 if (isInitiator()) {
2063 return RtpEndUserState.ENDED;
2064 } else {
2065 return RtpEndUserState.RETRACTED;
2066 }
2067 }
2068 case TERMINATED_CONNECTIVITY_ERROR -> {
2069 return zeroDuration()
2070 ? RtpEndUserState.CONNECTIVITY_ERROR
2071 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2072 }
2073 case TERMINATED_APPLICATION_FAILURE -> {
2074 return RtpEndUserState.APPLICATION_ERROR;
2075 }
2076 case TERMINATED_SECURITY_ERROR -> {
2077 return RtpEndUserState.SECURITY_ERROR;
2078 }
2079 }
2080 throw new IllegalStateException(
2081 String.format("%s has no equivalent EndUserState", this.state));
2082 }
2083
2084 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2085 final PeerConnection.PeerConnectionState state;
2086 try {
2087 state = webRTCWrapper.getState();
2088 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2089 // We usually close the WebRTCWrapper *before* transitioning so we might still
2090 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2091 return RtpEndUserState.ENDING_CALL;
2092 }
2093 return switch (state) {
2094 case CONNECTED -> RtpEndUserState.CONNECTED;
2095 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2096 case CLOSED -> RtpEndUserState.ENDING_CALL;
2097 default -> zeroDuration()
2098 ? RtpEndUserState.CONNECTIVITY_ERROR
2099 : RtpEndUserState.RECONNECTING;
2100 };
2101 }
2102
2103 private boolean isPeerConnectionConnected() {
2104 try {
2105 return webRTCWrapper.getState() == PeerConnection.PeerConnectionState.CONNECTED;
2106 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2107 return false;
2108 }
2109 }
2110
2111 private void updateCallIntegrationState() {
2112 switch (this.state) {
2113 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2114 if (isInitiator()) {
2115 this.callIntegration.setDialing();
2116 } else {
2117 this.callIntegration.setRinging();
2118 }
2119 }
2120 case PROCEED, SESSION_INITIALIZED_PRE_APPROVED -> {
2121 if (isInitiator()) {
2122 this.callIntegration.setDialing();
2123 } else {
2124 this.callIntegration.setInitialized();
2125 }
2126 }
2127 case SESSION_ACCEPTED -> {
2128 if (isPeerConnectionConnected()) {
2129 this.callIntegration.setActive();
2130 } else {
2131 this.callIntegration.setInitialized();
2132 }
2133 }
2134 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2135 if (isInitiator()) {
2136 this.callIntegration.busy();
2137 } else {
2138 this.callIntegration.rejected();
2139 }
2140 }
2141 case TERMINATED_SUCCESS -> this.callIntegration.success();
2142 case ACCEPTED -> this.callIntegration.accepted();
2143 case RETRACTED, RETRACTED_RACED, TERMINATED_CANCEL_OR_TIMEOUT -> this.callIntegration
2144 .retracted();
2145 case TERMINATED_CONNECTIVITY_ERROR,
2146 TERMINATED_APPLICATION_FAILURE,
2147 TERMINATED_SECURITY_ERROR -> this.callIntegration.error();
2148 default -> throw new IllegalStateException(
2149 String.format("%s is not handled", this.state));
2150 }
2151 }
2152
2153 public ContentAddition getPendingContentAddition() {
2154 final RtpContentMap in = this.incomingContentAdd;
2155 final RtpContentMap out = this.outgoingContentAdd;
2156 if (out != null) {
2157 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2158 } else if (in != null) {
2159 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2160 } else {
2161 return null;
2162 }
2163 }
2164
2165 public Set<Media> getMedia() {
2166 final State current = getState();
2167 if (current == State.NULL) {
2168 if (isInitiator()) {
2169 return Preconditions.checkNotNull(
2170 this.proposedMedia, "RTP connection has not been initialized properly");
2171 }
2172 throw new IllegalStateException("RTP connection has not been initialized yet");
2173 }
2174 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2175 return Preconditions.checkNotNull(
2176 this.proposedMedia, "RTP connection has not been initialized properly");
2177 }
2178 final RtpContentMap localContentMap = getLocalContentMap();
2179 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2180 if (localContentMap != null) {
2181 return localContentMap.getMedia();
2182 } else if (initiatorContentMap != null) {
2183 return initiatorContentMap.getMedia();
2184 } else if (isTerminated()) {
2185 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2186 } else {
2187 return Preconditions.checkNotNull(
2188 this.proposedMedia, "RTP connection has not been initialized properly");
2189 }
2190 }
2191
2192 public boolean isVerified() {
2193 final String fingerprint = this.omemoVerification.getFingerprint();
2194 if (fingerprint == null) {
2195 return false;
2196 }
2197 final FingerprintStatus status =
2198 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2199 return status != null && status.isVerified();
2200 }
2201
2202 public boolean addMedia(final Media media) {
2203 final Set<Media> currentMedia = getMedia();
2204 if (currentMedia.contains(media)) {
2205 throw new IllegalStateException(String.format("%s has already been proposed", media));
2206 }
2207 // TODO add state protection - can only add while ACCEPTED or so
2208 Log.d(Config.LOGTAG, "adding media: " + media);
2209 return webRTCWrapper.addTrack(media);
2210 }
2211
2212 public synchronized void acceptCall() {
2213 switch (this.state) {
2214 case PROPOSED -> {
2215 cancelRingingTimeout();
2216 acceptCallFromProposed();
2217 }
2218 case SESSION_INITIALIZED -> {
2219 cancelRingingTimeout();
2220 acceptCallFromSessionInitialized();
2221 }
2222 case ACCEPTED -> Log.w(
2223 Config.LOGTAG,
2224 id.account.getJid().asBareJid()
2225 + ": the call has already been accepted with another client. UI was just lagging behind");
2226 case PROCEED, SESSION_ACCEPTED -> Log.w(
2227 Config.LOGTAG,
2228 id.account.getJid().asBareJid()
2229 + ": the call has already been accepted. user probably double tapped the UI");
2230 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2231 }
2232 }
2233
2234 public synchronized void rejectCall() {
2235 if (isTerminated()) {
2236 Log.w(
2237 Config.LOGTAG,
2238 id.account.getJid().asBareJid()
2239 + ": received rejectCall() when session has already been terminated. nothing to do");
2240 return;
2241 }
2242 switch (this.state) {
2243 case PROPOSED -> rejectCallFromProposed();
2244 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2245 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2246 }
2247 }
2248
2249 public synchronized void integrationFailure() {
2250 final var state = getState();
2251 if (state == State.PROPOSED) {
2252 Log.e(
2253 Config.LOGTAG,
2254 id.account.getJid().asBareJid()
2255 + ": failed call integration in state proposed");
2256 rejectCallFromProposed();
2257 } else if (state == State.SESSION_INITIALIZED) {
2258 Log.e(Config.LOGTAG, id.account.getJid().asBareJid() + ": failed call integration");
2259 this.webRTCWrapper.close();
2260 sendSessionTerminate(Reason.FAILED_APPLICATION, "CallIntegration failed");
2261 } else {
2262 throw new IllegalStateException(
2263 String.format("Can not fail integration in state %s", state));
2264 }
2265 }
2266
2267 public synchronized void endCall() {
2268 if (isTerminated()) {
2269 Log.w(
2270 Config.LOGTAG,
2271 id.account.getJid().asBareJid()
2272 + ": received endCall() when session has already been terminated. nothing to do");
2273 return;
2274 }
2275 if (isInState(State.PROPOSED) && isResponder()) {
2276 rejectCallFromProposed();
2277 return;
2278 }
2279 if (isInState(State.PROCEED)) {
2280 if (isInitiator()) {
2281 retractFromProceed();
2282 } else {
2283 rejectCallFromProceed();
2284 }
2285 return;
2286 }
2287 if (isInitiator()
2288 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2289 this.webRTCWrapper.close();
2290 sendSessionTerminate(Reason.CANCEL);
2291 return;
2292 }
2293 if (isInState(State.SESSION_INITIALIZED)) {
2294 rejectCallFromSessionInitiate();
2295 return;
2296 }
2297 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2298 this.webRTCWrapper.close();
2299 sendSessionTerminate(Reason.SUCCESS);
2300 return;
2301 }
2302 if (isInState(
2303 State.TERMINATED_APPLICATION_FAILURE,
2304 State.TERMINATED_CONNECTIVITY_ERROR,
2305 State.TERMINATED_DECLINED_OR_BUSY)) {
2306 Log.d(
2307 Config.LOGTAG,
2308 "ignoring request to end call because already in state " + this.state);
2309 return;
2310 }
2311 throw new IllegalStateException(
2312 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2313 }
2314
2315 private void retractFromProceed() {
2316 Log.d(Config.LOGTAG, "retract from proceed");
2317 this.sendJingleMessage("retract");
2318 closeTransitionLogFinish(State.RETRACTED_RACED);
2319 }
2320
2321 private void closeTransitionLogFinish(final State state) {
2322 this.webRTCWrapper.close();
2323 transitionOrThrow(state);
2324 writeLogMessage(state);
2325 finish();
2326 }
2327
2328 private void setupWebRTC(
2329 final Set<Media> media,
2330 final List<PeerConnection.IceServer> iceServers,
2331 final boolean trickle)
2332 throws WebRTCWrapper.InitializationException {
2333 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2334 this.webRTCWrapper.setup(this.xmppConnectionService);
2335 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2336 this.webRTCWrapper.setMicrophoneEnabledOrThrow(callIntegration.isMicrophoneEnabled());
2337 }
2338
2339 private void acceptCallFromProposed() {
2340 transitionOrThrow(State.PROCEED);
2341 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2342 this.callIntegration.startAudioRouting();
2343 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2344 this.sendJingleMessage("proceed");
2345 }
2346
2347 private void rejectCallFromProposed() {
2348 transitionOrThrow(State.REJECTED);
2349 writeLogMessageMissed();
2350 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2351 this.sendJingleMessage("reject");
2352 finish();
2353 }
2354
2355 private void rejectCallFromProceed() {
2356 this.sendJingleMessage("reject");
2357 closeTransitionLogFinish(State.REJECTED_RACED);
2358 }
2359
2360 private void rejectCallFromSessionInitiate() {
2361 webRTCWrapper.close();
2362 sendSessionTerminate(Reason.DECLINE);
2363 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2364 }
2365
2366 private void sendJingleMessage(final String action) {
2367 sendJingleMessage(action, id.with);
2368 }
2369
2370 private void sendJingleMessage(final String action, final Jid to) {
2371 final MessagePacket messagePacket = new MessagePacket();
2372 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2373 messagePacket.setTo(to);
2374 final Element intent =
2375 messagePacket
2376 .addChild(action, Namespace.JINGLE_MESSAGE)
2377 .setAttribute("id", id.sessionId);
2378 if ("proceed".equals(action)) {
2379 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2380 if (isOmemoEnabled()) {
2381 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2382 final Element device =
2383 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2384 device.setAttribute("id", deviceId);
2385 }
2386 }
2387 messagePacket.addChild("store", "urn:xmpp:hints");
2388 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2389 }
2390
2391 private void sendJingleMessageFinish(final Reason reason) {
2392 final var account = id.getAccount();
2393 final MessagePacket messagePacket =
2394 xmppConnectionService
2395 .getMessageGenerator()
2396 .sessionFinish(id.with, id.sessionId, reason);
2397 xmppConnectionService.sendMessagePacket(account, messagePacket);
2398 }
2399
2400 private boolean isOmemoEnabled() {
2401 final Conversational conversational = message.getConversation();
2402 if (conversational instanceof Conversation) {
2403 return ((Conversation) conversational).getNextEncryption()
2404 == Message.ENCRYPTION_AXOLOTL;
2405 }
2406 return false;
2407 }
2408
2409 private void acceptCallFromSessionInitialized() {
2410 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2411 this.callIntegration.startAudioRouting();
2412 sendSessionAccept();
2413 }
2414
2415 @Override
2416 protected synchronized boolean transition(final State target, final Runnable runnable) {
2417 if (super.transition(target, runnable)) {
2418 updateEndUserState();
2419 updateOngoingCallNotification();
2420 return true;
2421 } else {
2422 return false;
2423 }
2424 }
2425
2426 @Override
2427 public void onIceCandidate(final IceCandidate iceCandidate) {
2428 final RtpContentMap rtpContentMap =
2429 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2430 final IceUdpTransportInfo.Credentials credentials;
2431 try {
2432 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2433 } catch (final IllegalArgumentException e) {
2434 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2435 return;
2436 }
2437 final String uFrag = credentials.ufrag;
2438 final IceUdpTransportInfo.Candidate candidate =
2439 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2440 if (candidate == null) {
2441 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2442 return;
2443 }
2444 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2445 sendTransportInfo(iceCandidate.sdpMid, candidate);
2446 }
2447
2448 @Override
2449 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2450 Log.d(
2451 Config.LOGTAG,
2452 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2453 this.stateHistory.add(newState);
2454 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2455 this.sessionDuration.start();
2456 updateOngoingCallNotification();
2457 } else if (this.sessionDuration.isRunning()) {
2458 this.sessionDuration.stop();
2459 updateOngoingCallNotification();
2460 }
2461
2462 final boolean neverConnected =
2463 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2464
2465 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2466 if (neverConnected) {
2467 if (isTerminated()) {
2468 Log.d(
2469 Config.LOGTAG,
2470 id.account.getJid().asBareJid()
2471 + ": not sending session-terminate after connectivity error because session is already in state "
2472 + this.state);
2473 return;
2474 }
2475 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2476 return;
2477 } else {
2478 this.restartIce();
2479 }
2480 }
2481 updateEndUserState();
2482 }
2483
2484 private void restartIce() {
2485 this.stateHistory.clear();
2486 this.webRTCWrapper.restartIceAsync();
2487 }
2488
2489 @Override
2490 public void onRenegotiationNeeded() {
2491 this.webRTCWrapper.execute(this::renegotiate);
2492 }
2493
2494 private synchronized void renegotiate() {
2495 final SessionDescription sessionDescription;
2496 try {
2497 sessionDescription = setLocalSessionDescription();
2498 } catch (final Exception e) {
2499 final Throwable cause = Throwables.getRootCause(e);
2500 webRTCWrapper.close();
2501 if (isTerminated()) {
2502 Log.d(
2503 Config.LOGTAG,
2504 "failed to renegotiate. session was already terminated",
2505 cause);
2506 return;
2507 }
2508 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2509 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2510 return;
2511 }
2512 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2513 final RtpContentMap currentContentMap = getLocalContentMap();
2514 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2515 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2516
2517 Log.d(
2518 Config.LOGTAG,
2519 id.getAccount().getJid().asBareJid()
2520 + ": renegotiate. iceRestart="
2521 + iceRestart
2522 + " content id diff="
2523 + diff);
2524
2525 if (diff.hasModifications() && iceRestart) {
2526 webRTCWrapper.close();
2527 sendSessionTerminate(
2528 Reason.FAILED_APPLICATION,
2529 "WebRTC unexpectedly tried to modify content and transport at once");
2530 return;
2531 }
2532
2533 if (iceRestart) {
2534 initiateIceRestart(rtpContentMap);
2535 return;
2536 } else if (diff.isEmpty()) {
2537 Log.d(
2538 Config.LOGTAG,
2539 "renegotiation. nothing to do. SignalingState="
2540 + this.webRTCWrapper.getSignalingState());
2541 }
2542
2543 if (diff.added.isEmpty()) {
2544 return;
2545 }
2546 modifyLocalContentMap(rtpContentMap);
2547 sendContentAdd(rtpContentMap, diff.added);
2548 }
2549
2550 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2551 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2552 final JinglePacket jinglePacket =
2553 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2554 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2555 jinglePacket.setTo(id.with);
2556 xmppConnectionService.sendIqPacket(
2557 id.account,
2558 jinglePacket,
2559 (account, response) -> {
2560 if (response.getType() == IqPacket.TYPE.RESULT) {
2561 Log.d(Config.LOGTAG, "received success to our ice restart");
2562 setLocalContentMap(rtpContentMap);
2563 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2564 return;
2565 }
2566 if (response.getType() == IqPacket.TYPE.ERROR) {
2567 if (isTieBreak(response)) {
2568 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2569 return;
2570 }
2571 handleIqErrorResponse(response);
2572 }
2573 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2574 handleIqTimeoutResponse(response);
2575 }
2576 });
2577 }
2578
2579 private boolean isTieBreak(final IqPacket response) {
2580 final Element error = response.findChild("error");
2581 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2582 }
2583
2584 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2585 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2586 this.outgoingContentAdd = contentAdd;
2587 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2588 prepareOutgoingContentMap(contentAdd);
2589 Futures.addCallback(
2590 outgoingContentMapFuture,
2591 new FutureCallback<>() {
2592 @Override
2593 public void onSuccess(final RtpContentMap outgoingContentMap) {
2594 sendContentAdd(outgoingContentMap);
2595 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2596 }
2597
2598 @Override
2599 public void onFailure(@NonNull Throwable throwable) {
2600 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2601 }
2602 },
2603 MoreExecutors.directExecutor());
2604 }
2605
2606 private void sendContentAdd(final RtpContentMap contentAdd) {
2607
2608 final JinglePacket jinglePacket =
2609 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2610 jinglePacket.setTo(id.with);
2611 xmppConnectionService.sendIqPacket(
2612 id.account,
2613 jinglePacket,
2614 (connection, response) -> {
2615 if (response.getType() == IqPacket.TYPE.RESULT) {
2616 Log.d(
2617 Config.LOGTAG,
2618 id.getAccount().getJid().asBareJid()
2619 + ": received ACK to our content-add");
2620 return;
2621 }
2622 if (response.getType() == IqPacket.TYPE.ERROR) {
2623 if (isTieBreak(response)) {
2624 this.outgoingContentAdd = null;
2625 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2626 return;
2627 }
2628 handleIqErrorResponse(response);
2629 }
2630 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2631 handleIqTimeoutResponse(response);
2632 }
2633 });
2634 }
2635
2636 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2637 if (isInitiator()) {
2638 this.initiatorRtpContentMap = rtpContentMap;
2639 } else {
2640 this.responderRtpContentMap = rtpContentMap;
2641 }
2642 }
2643
2644 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2645 if (isInitiator()) {
2646 this.responderRtpContentMap = rtpContentMap;
2647 } else {
2648 this.initiatorRtpContentMap = rtpContentMap;
2649 }
2650 }
2651
2652 // this method is to be used for content map modifications that modify media
2653 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2654 final RtpContentMap activeContents = rtpContentMap.activeContents();
2655 setLocalContentMap(activeContents);
2656 this.callIntegration.setAudioDeviceWhenAvailable(
2657 CallIntegration.initialAudioDevice(activeContents.getMedia()));
2658 updateEndUserState();
2659 }
2660
2661 private SessionDescription setLocalSessionDescription()
2662 throws ExecutionException, InterruptedException {
2663 final org.webrtc.SessionDescription sessionDescription =
2664 this.webRTCWrapper.setLocalDescription(false).get();
2665 return SessionDescription.parse(sessionDescription.description);
2666 }
2667
2668 private void closeWebRTCSessionAfterFailedConnection() {
2669 this.webRTCWrapper.close();
2670 synchronized (this) {
2671 if (isTerminated()) {
2672 Log.d(
2673 Config.LOGTAG,
2674 id.account.getJid().asBareJid()
2675 + ": no need to send session-terminate after failed connection. Other party already did");
2676 return;
2677 }
2678 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2679 }
2680 }
2681
2682 public boolean zeroDuration() {
2683 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2684 }
2685
2686 public long getCallDuration() {
2687 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2688 }
2689
2690 @Override
2691 public CallIntegration getCallIntegration() {
2692 return this.callIntegration;
2693 }
2694
2695 public boolean isMicrophoneEnabled() {
2696 return webRTCWrapper.isMicrophoneEnabled();
2697 }
2698
2699 public boolean setMicrophoneEnabled(final boolean enabled) {
2700 return webRTCWrapper.setMicrophoneEnabledOrThrow(enabled);
2701 }
2702
2703 public boolean isVideoEnabled() {
2704 return webRTCWrapper.isVideoEnabled();
2705 }
2706
2707 public void setVideoEnabled(final boolean enabled) {
2708 webRTCWrapper.setVideoEnabled(enabled);
2709 }
2710
2711 public boolean isCameraSwitchable() {
2712 return webRTCWrapper.isCameraSwitchable();
2713 }
2714
2715 public boolean isFrontCamera() {
2716 return webRTCWrapper.isFrontCamera();
2717 }
2718
2719 public ListenableFuture<Boolean> switchCamera() {
2720 return webRTCWrapper.switchCamera();
2721 }
2722
2723 @Override
2724 public synchronized void onCallIntegrationShowIncomingCallUi() {
2725 if (isTerminated()) {
2726 // there might be race conditions with the call integration service invoking this
2727 // callback when the rtp session has already ended.
2728 Log.w(
2729 Config.LOGTAG,
2730 "CallIntegration requested incoming call UI but session was already terminated");
2731 return;
2732 }
2733 // TODO apparently this can be called too early as well?
2734 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
2735 }
2736
2737 @Override
2738 public void onCallIntegrationDisconnect() {
2739 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2740 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2741 rejectCall();
2742 } else {
2743 endCall();
2744 }
2745 }
2746
2747 @Override
2748 public void onCallIntegrationReject() {
2749 Log.d(Config.LOGTAG, "rejecting call from system notification / call integration");
2750 try {
2751 rejectCall();
2752 } catch (final IllegalStateException e) {
2753 Log.w(Config.LOGTAG, "race condition on rejecting call from notification", e);
2754 }
2755 }
2756
2757 @Override
2758 public void onCallIntegrationAnswer() {
2759 // we need to start the UI to a) show it and b) be able to ask for permissions
2760 final Intent intent = new Intent(xmppConnectionService, RtpSessionActivity.class);
2761 intent.setAction(RtpSessionActivity.ACTION_ACCEPT_CALL);
2762 intent.putExtra(RtpSessionActivity.EXTRA_ACCOUNT, id.account.getJid().toEscapedString());
2763 intent.putExtra(RtpSessionActivity.EXTRA_WITH, id.with.toEscapedString());
2764 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
2765 intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK);
2766 intent.putExtra(RtpSessionActivity.EXTRA_SESSION_ID, id.sessionId);
2767 Log.d(Config.LOGTAG, "start activity to accept call from call integration");
2768 xmppConnectionService.startActivity(intent);
2769 }
2770
2771 @Override
2772 public void onCallIntegrationSilence() {
2773 xmppConnectionService.getNotificationService().stopSoundAndVibration();
2774 }
2775
2776 @Override
2777 public void onCallIntegrationMicrophoneEnabled(final boolean enabled) {
2778 this.webRTCWrapper.setMicrophoneEnabled(enabled);
2779 }
2780
2781 @Override
2782 public void onAudioDeviceChanged(
2783 final CallIntegration.AudioDevice selectedAudioDevice,
2784 final Set<CallIntegration.AudioDevice> availableAudioDevices) {
2785 Log.d(
2786 Config.LOGTAG,
2787 "onAudioDeviceChanged(" + selectedAudioDevice + "," + availableAudioDevices + ")");
2788 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2789 selectedAudioDevice, availableAudioDevices);
2790 }
2791
2792 private void updateEndUserState() {
2793 final RtpEndUserState endUserState = getEndUserState();
2794 this.updateCallIntegrationState();
2795 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2796 id.account, id.with, id.sessionId, endUserState);
2797 }
2798
2799 private void updateOngoingCallNotification() {
2800 final State state = this.state;
2801 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2802 final boolean reconnecting;
2803 if (state == State.SESSION_ACCEPTED) {
2804 reconnecting =
2805 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2806 } else {
2807 reconnecting = false;
2808 }
2809 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2810 } else {
2811 xmppConnectionService.removeOngoingCall();
2812 }
2813 }
2814
2815 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2816 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2817 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2818 request.setTo(id.account.getDomain());
2819 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2820 xmppConnectionService.sendIqPacket(
2821 id.account,
2822 request,
2823 (account, response) -> {
2824 final var iceServers = IceServers.parse(response);
2825 if (iceServers.isEmpty()) {
2826 Log.w(
2827 Config.LOGTAG,
2828 id.account.getJid().asBareJid()
2829 + ": no ICE server found "
2830 + response);
2831 }
2832 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2833 });
2834 } else {
2835 Log.w(
2836 Config.LOGTAG,
2837 id.account.getJid().asBareJid() + ": has no external service discovery");
2838 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2839 }
2840 }
2841
2842 @Override
2843 protected void terminateTransport() {
2844 this.webRTCWrapper.close();
2845 }
2846
2847 @Override
2848 protected void finish() {
2849 if (isTerminated()) {
2850 this.cancelRingingTimeout();
2851 this.callIntegration.verifyDisconnected();
2852 this.webRTCWrapper.verifyClosed();
2853 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2854 super.finish();
2855 try {
2856 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2857 log.getParentFile().mkdirs();
2858 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2859 } catch (final IOException e) { }
2860 } else {
2861 throw new IllegalStateException(
2862 String.format("Unable to call finish from %s", this.state));
2863 }
2864 }
2865
2866 private void writeLogMessage(final State state) {
2867 final long duration = getCallDuration();
2868 if (state == State.TERMINATED_SUCCESS
2869 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2870 writeLogMessageSuccess(duration);
2871 } else {
2872 writeLogMessageMissed();
2873 }
2874 }
2875
2876 private void writeLogMessageSuccess(final long duration) {
2877 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2878 this.writeMessage();
2879 }
2880
2881 private void writeLogMessageMissed() {
2882 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2883 this.writeMessage();
2884 }
2885
2886 private void writeMessage() {
2887 final Conversational conversational = message.getConversation();
2888 if (conversational instanceof Conversation) {
2889 ((Conversation) conversational).add(this.message);
2890 xmppConnectionService.createMessageAsync(message);
2891 xmppConnectionService.updateConversationUi();
2892 } else {
2893 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2894 }
2895 }
2896
2897 public Optional<VideoTrack> getLocalVideoTrack() {
2898 return webRTCWrapper.getLocalVideoTrack();
2899 }
2900
2901 public Optional<VideoTrack> getRemoteVideoTrack() {
2902 return webRTCWrapper.getRemoteVideoTrack();
2903 }
2904
2905 public EglBase.Context getEglBaseContext() {
2906 return webRTCWrapper.getEglBaseContext();
2907 }
2908
2909 void setProposedMedia(final Set<Media> media) {
2910 this.proposedMedia = media;
2911 this.callIntegration.setVideoState(
2912 Media.audioOnly(media)
2913 ? VideoProfile.STATE_AUDIO_ONLY
2914 : VideoProfile.STATE_BIDIRECTIONAL);
2915 this.callIntegration.setInitialAudioDevice(CallIntegration.initialAudioDevice(media));
2916 }
2917
2918 public void fireStateUpdate() {
2919 final RtpEndUserState endUserState = getEndUserState();
2920 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2921 id.account, id.with, id.sessionId, endUserState);
2922 }
2923
2924 public boolean isSwitchToVideoAvailable() {
2925 final boolean prerequisite =
2926 Media.audioOnly(getMedia())
2927 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2928 .contains(getEndUserState());
2929 return prerequisite && remoteHasVideoFeature();
2930 }
2931
2932 private boolean remoteHasVideoFeature() {
2933 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2934 }
2935
2936 private boolean remoteHasSdpOfferAnswer() {
2937 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2938 }
2939
2940 @Override
2941 public Account getAccount() {
2942 return id.account;
2943 }
2944
2945 @Override
2946 public Jid getWith() {
2947 return id.with;
2948 }
2949
2950 @Override
2951 public String getSessionId() {
2952 return id.sessionId;
2953 }
2954
2955 private interface OnIceServersDiscovered {
2956 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2957 }
2958}