1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4import android.os.Environment;
5
6import androidx.annotation.NonNull;
7import androidx.annotation.Nullable;
8
9import com.google.common.base.Joiner;
10import com.google.common.base.Optional;
11import com.google.common.base.Preconditions;
12import com.google.common.base.Stopwatch;
13import com.google.common.base.Strings;
14import com.google.common.base.Throwables;
15import com.google.common.collect.Collections2;
16import com.google.common.collect.ImmutableList;
17import com.google.common.collect.ImmutableMultimap;
18import com.google.common.collect.ImmutableSet;
19import com.google.common.collect.Iterables;
20import com.google.common.collect.Maps;
21import com.google.common.collect.Sets;
22import com.google.common.primitives.Ints;
23import com.google.common.util.concurrent.FutureCallback;
24import com.google.common.util.concurrent.Futures;
25import com.google.common.util.concurrent.ListenableFuture;
26import com.google.common.util.concurrent.MoreExecutors;
27
28import eu.siacs.conversations.BuildConfig;
29import eu.siacs.conversations.Config;
30import eu.siacs.conversations.crypto.axolotl.AxolotlService;
31import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
32import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
33import eu.siacs.conversations.entities.Conversation;
34import eu.siacs.conversations.entities.Conversational;
35import eu.siacs.conversations.entities.Message;
36import eu.siacs.conversations.entities.RtpSessionStatus;
37import eu.siacs.conversations.services.AppRTCAudioManager;
38import eu.siacs.conversations.utils.IP;
39import eu.siacs.conversations.xml.Element;
40import eu.siacs.conversations.xml.Namespace;
41import eu.siacs.conversations.xmpp.Jid;
42import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
43import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
44import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
45import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
46import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
49import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
50import eu.siacs.conversations.xmpp.stanzas.IqPacket;
51import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
52
53import org.webrtc.DtmfSender;
54import org.webrtc.EglBase;
55import org.webrtc.IceCandidate;
56import org.webrtc.PeerConnection;
57import org.webrtc.VideoTrack;
58
59import java.io.File;
60import java.io.IOException;
61import java.util.Arrays;
62import java.util.Collection;
63import java.util.Collections;
64import java.util.LinkedList;
65import java.util.List;
66import java.util.Map;
67import java.util.Queue;
68import java.util.Set;
69import java.util.concurrent.ExecutionException;
70import java.util.concurrent.ScheduledFuture;
71import java.util.concurrent.TimeUnit;
72
73public class JingleRtpConnection extends AbstractJingleConnection
74 implements WebRTCWrapper.EventCallback {
75
76 public static final List<State> STATES_SHOWING_ONGOING_CALL =
77 Arrays.asList(
78 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
79 private static final long BUSY_TIME_OUT = 30;
80
81 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
82 private final Queue<Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>>>
83 pendingIceCandidates = new LinkedList<>();
84 private final OmemoVerification omemoVerification = new OmemoVerification();
85 private final Message message;
86
87 private Set<Media> proposedMedia;
88 private RtpContentMap initiatorRtpContentMap;
89 private RtpContentMap responderRtpContentMap;
90 private RtpContentMap incomingContentAdd;
91 private RtpContentMap outgoingContentAdd;
92 private IceUdpTransportInfo.Setup peerDtlsSetup;
93 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
94 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
95 private ScheduledFuture<?> ringingTimeoutFuture;
96 private final long created = System.currentTimeMillis() / 1000L;
97
98 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
99 super(jingleConnectionManager, id, initiator);
100 final Conversation conversation =
101 jingleConnectionManager
102 .getXmppConnectionService()
103 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
104 this.message =
105 new Message(
106 conversation,
107 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
108 Message.TYPE_RTP_SESSION,
109 id.sessionId);
110 }
111
112 @Override
113 synchronized void deliverPacket(final JinglePacket jinglePacket) {
114 switch (jinglePacket.getAction()) {
115 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
116 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
117 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
118 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
119 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
120 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
121 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
122 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
123 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
124 default -> {
125 respondOk(jinglePacket);
126 Log.d(
127 Config.LOGTAG,
128 String.format(
129 "%s: received unhandled jingle action %s",
130 id.account.getJid().asBareJid(), jinglePacket.getAction()));
131 }
132 }
133 }
134
135 @Override
136 synchronized void notifyRebound() {
137 if (isTerminated()) {
138 return;
139 }
140 webRTCWrapper.close();
141 if (isResponder() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
142 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
143 }
144 if (isInState(
145 State.SESSION_INITIALIZED,
146 State.SESSION_INITIALIZED_PRE_APPROVED,
147 State.SESSION_ACCEPTED)) {
148 // we might have already changed resources (full jid) at this point; so this might not
149 // even reach the other party
150 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
151 } else {
152 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
153 finish();
154 }
155 }
156
157 public boolean applyDtmfTone(String tone) {
158 return webRTCWrapper.applyDtmfTone(tone);
159 }
160
161 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
162 respondOk(jinglePacket);
163 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
164 final State previous = this.state;
165 Log.d(
166 Config.LOGTAG,
167 id.account.getJid().asBareJid()
168 + ": received session terminate reason="
169 + wrapper.reason
170 + "("
171 + Strings.nullToEmpty(wrapper.text)
172 + ") while in state "
173 + previous);
174 if (TERMINATED.contains(previous)) {
175 Log.d(
176 Config.LOGTAG,
177 id.account.getJid().asBareJid()
178 + ": ignoring session terminate because already in "
179 + previous);
180 return;
181 }
182 webRTCWrapper.close();
183 final State target = reasonToState(wrapper.reason);
184 transitionOrThrow(target);
185 writeLogMessage(target);
186 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
187 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
188 }
189 finish();
190 }
191
192 private void receiveTransportInfo(final JinglePacket jinglePacket) {
193 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
194 // INITIALIZED only after transport-info has been received
195 if (isInState(
196 State.NULL,
197 State.PROCEED,
198 State.SESSION_INITIALIZED,
199 State.SESSION_INITIALIZED_PRE_APPROVED,
200 State.SESSION_ACCEPTED)) {
201 final RtpContentMap contentMap;
202 try {
203 contentMap = RtpContentMap.of(jinglePacket);
204 } catch (final IllegalArgumentException | NullPointerException e) {
205 Log.d(
206 Config.LOGTAG,
207 id.account.getJid().asBareJid()
208 + ": improperly formatted contents; ignoring",
209 e);
210 respondOk(jinglePacket);
211 return;
212 }
213 receiveTransportInfo(jinglePacket, contentMap);
214 } else {
215 if (isTerminated()) {
216 respondOk(jinglePacket);
217 Log.d(
218 Config.LOGTAG,
219 id.account.getJid().asBareJid()
220 + ": ignoring out-of-order transport info; we where already terminated");
221 } else {
222 Log.d(
223 Config.LOGTAG,
224 id.account.getJid().asBareJid()
225 + ": received transport info while in state="
226 + this.state);
227 terminateWithOutOfOrder(jinglePacket);
228 }
229 }
230 }
231
232 private void receiveTransportInfo(
233 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
234 final Set<Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>>> candidates =
235 contentMap.contents.entrySet();
236 final RtpContentMap remote = getRemoteContentMap();
237 final Set<String> remoteContentIds =
238 remote == null ? Collections.emptySet() : remote.contents.keySet();
239 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
240 Log.d(
241 Config.LOGTAG,
242 "received transport-info for unknown contents "
243 + contentMap.contents.keySet()
244 + " (known: "
245 + remoteContentIds
246 + ")");
247 respondOk(jinglePacket);
248 pendingIceCandidates.addAll(candidates);
249 return;
250 }
251 if (this.state != State.SESSION_ACCEPTED) {
252 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
253 respondOk(jinglePacket);
254 pendingIceCandidates.addAll(candidates);
255 return;
256 }
257 // zero candidates + modified credentials are an ICE restart offer
258 if (checkForIceRestart(jinglePacket, contentMap)) {
259 return;
260 }
261 respondOk(jinglePacket);
262 try {
263 processCandidates(candidates);
264 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
265 Log.w(
266 Config.LOGTAG,
267 id.account.getJid().asBareJid()
268 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
269 }
270 }
271
272 private void receiveContentAdd(final JinglePacket jinglePacket) {
273 final RtpContentMap modification;
274 try {
275 modification = RtpContentMap.of(jinglePacket);
276 modification.requireContentDescriptions();
277 } catch (final RuntimeException e) {
278 Log.d(
279 Config.LOGTAG,
280 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
281 Throwables.getRootCause(e));
282 respondOk(jinglePacket);
283 webRTCWrapper.close();
284 sendSessionTerminate(Reason.of(e), e.getMessage());
285 return;
286 }
287 if (isInState(State.SESSION_ACCEPTED)) {
288 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
289 final ListenableFuture<RtpContentMap> future =
290 receiveRtpContentMap(
291 modification,
292 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
293 Futures.addCallback(
294 future,
295 new FutureCallback<>() {
296 @Override
297 public void onSuccess(final RtpContentMap rtpContentMap) {
298 receiveContentAdd(jinglePacket, rtpContentMap);
299 }
300
301 @Override
302 public void onFailure(@NonNull Throwable throwable) {
303 respondOk(jinglePacket);
304 final Throwable rootCause = Throwables.getRootCause(throwable);
305 Log.d(
306 Config.LOGTAG,
307 id.account.getJid().asBareJid()
308 + ": improperly formatted contents in content-add",
309 throwable);
310 webRTCWrapper.close();
311 sendSessionTerminate(
312 Reason.ofThrowable(rootCause), rootCause.getMessage());
313 }
314 },
315 MoreExecutors.directExecutor());
316 } else {
317 terminateWithOutOfOrder(jinglePacket);
318 }
319 }
320
321 private void receiveContentAdd(
322 final JinglePacket jinglePacket, final RtpContentMap modification) {
323 final RtpContentMap remote = getRemoteContentMap();
324 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
325 respondOk(jinglePacket);
326 this.webRTCWrapper.close();
327 sendSessionTerminate(
328 Reason.FAILED_APPLICATION,
329 String.format(
330 "contents with names %s already exists",
331 Joiner.on(", ").join(modification.getNames())));
332 return;
333 }
334 final ContentAddition contentAddition =
335 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
336
337 final RtpContentMap outgoing = this.outgoingContentAdd;
338 final Set<ContentAddition.Summary> outgoingContentAddSummary =
339 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
340
341 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
342 if (isInitiator()) {
343 Log.d(
344 Config.LOGTAG,
345 id.getAccount().getJid().asBareJid()
346 + ": respond with tie break to matching content-add offer");
347 respondWithTieBreak(jinglePacket);
348 } else {
349 Log.d(
350 Config.LOGTAG,
351 id.getAccount().getJid().asBareJid()
352 + ": automatically accept matching content-add offer");
353 acceptContentAdd(contentAddition.summary, modification);
354 }
355 return;
356 }
357
358 // once we can display multiple video tracks we can be more loose with this condition
359 // theoretically it should also be fine to automatically accept audio only contents
360 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
361 Log.d(
362 Config.LOGTAG,
363 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
364 this.incomingContentAdd = modification;
365 respondOk(jinglePacket);
366 updateEndUserState();
367 } else {
368 respondOk(jinglePacket);
369 // TODO do we want to add a reason?
370 rejectContentAdd(modification);
371 }
372 }
373
374 private void receiveContentAccept(final JinglePacket jinglePacket) {
375 final RtpContentMap receivedContentAccept;
376 try {
377 receivedContentAccept = RtpContentMap.of(jinglePacket);
378 receivedContentAccept.requireContentDescriptions();
379 } catch (final RuntimeException e) {
380 Log.d(
381 Config.LOGTAG,
382 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
383 Throwables.getRootCause(e));
384 respondOk(jinglePacket);
385 webRTCWrapper.close();
386 sendSessionTerminate(Reason.of(e), e.getMessage());
387 return;
388 }
389
390 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
391 if (outgoingContentAdd == null) {
392 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
393 terminateWithOutOfOrder(jinglePacket);
394 return;
395 }
396 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
397 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
398 this.outgoingContentAdd = null;
399 respondOk(jinglePacket);
400 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
401 final ListenableFuture<RtpContentMap> future =
402 receiveRtpContentMap(
403 receivedContentAccept,
404 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
405 Futures.addCallback(
406 future,
407 new FutureCallback<>() {
408 @Override
409 public void onSuccess(final RtpContentMap result) {
410 receiveContentAccept(result);
411 }
412
413 @Override
414 public void onFailure(@NonNull final Throwable throwable) {
415 webRTCWrapper.close();
416 sendSessionTerminate(
417 Reason.ofThrowable(throwable), throwable.getMessage());
418 }
419 },
420 MoreExecutors.directExecutor());
421 } else {
422 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
423 terminateWithOutOfOrder(jinglePacket);
424 }
425 }
426
427 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
428 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
429 final RtpContentMap modifiedContentMap =
430 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
431
432 setRemoteContentMap(modifiedContentMap);
433
434 final SessionDescription answer = SessionDescription.of(modifiedContentMap, isResponder());
435
436 final org.webrtc.SessionDescription sdp =
437 new org.webrtc.SessionDescription(
438 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
439
440 try {
441 this.webRTCWrapper.setRemoteDescription(sdp).get();
442 } catch (final Exception e) {
443 final Throwable cause = Throwables.getRootCause(e);
444 Log.d(
445 Config.LOGTAG,
446 id.getAccount().getJid().asBareJid()
447 + ": unable to set remote description after receiving content-accept",
448 cause);
449 webRTCWrapper.close();
450 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
451 return;
452 }
453 Log.d(
454 Config.LOGTAG,
455 id.getAccount().getJid().asBareJid()
456 + ": remote has accepted content-add "
457 + ContentAddition.summary(receivedContentAccept));
458 processCandidates(receivedContentAccept.contents.entrySet());
459 updateEndUserState();
460 }
461
462 private void receiveContentModify(final JinglePacket jinglePacket) {
463 if (this.state != State.SESSION_ACCEPTED) {
464 terminateWithOutOfOrder(jinglePacket);
465 return;
466 }
467 final Map<String, Content.Senders> modification =
468 Maps.transformEntries(
469 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
470 final boolean isInitiator = isInitiator();
471 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
472 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
473 final Set<String> currentOutgoingMediaIds =
474 currentOutgoing == null
475 ? Collections.emptySet()
476 : currentOutgoing.contents.keySet();
477 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
478 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
479 respondOk(jinglePacket);
480 final RtpContentMap modifiedContentMap;
481 try {
482 modifiedContentMap =
483 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
484 } catch (final IllegalArgumentException e) {
485 webRTCWrapper.close();
486 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
487 return;
488 }
489 this.outgoingContentAdd = modifiedContentMap;
490 Log.d(
491 Config.LOGTAG,
492 id.account.getJid().asBareJid()
493 + ": processed content-modification for pending content-add");
494 } else if (remoteContentMap != null
495 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
496 respondOk(jinglePacket);
497 final RtpContentMap modifiedRemoteContentMap;
498 try {
499 modifiedRemoteContentMap =
500 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
501 } catch (final IllegalArgumentException e) {
502 webRTCWrapper.close();
503 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
504 return;
505 }
506 final SessionDescription offer;
507 try {
508 offer = SessionDescription.of(modifiedRemoteContentMap, isResponder());
509 } catch (final IllegalArgumentException | NullPointerException e) {
510 Log.d(
511 Config.LOGTAG,
512 id.getAccount().getJid().asBareJid()
513 + ": unable convert offer from content-modify to SDP",
514 e);
515 webRTCWrapper.close();
516 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
517 return;
518 }
519 Log.d(
520 Config.LOGTAG,
521 id.account.getJid().asBareJid() + ": auto accepting content-modification");
522 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
523 } else {
524 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
525 respondWithItemNotFound(jinglePacket);
526 }
527 }
528
529 private void autoAcceptContentModify(
530 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
531 this.setRemoteContentMap(modifiedRemoteContentMap);
532 final org.webrtc.SessionDescription sdp =
533 new org.webrtc.SessionDescription(
534 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
535 try {
536 this.webRTCWrapper.setRemoteDescription(sdp).get();
537 // auto accept is only done when we already have tracks
538 final SessionDescription answer = setLocalSessionDescription();
539 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
540 modifyLocalContentMap(rtpContentMap);
541 // we do not need to send an answer but do we have to resend the candidates currently in
542 // SDP?
543 // resendCandidatesFromSdp(answer);
544 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
545 } catch (final Exception e) {
546 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
547 webRTCWrapper.close();
548 sendSessionTerminate(Reason.FAILED_APPLICATION);
549 }
550 }
551
552 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
553 final SessionDescription answer) {
554 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
555 new ImmutableMultimap.Builder<>();
556 for (final SessionDescription.Media media : answer.media) {
557 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
558 if (Strings.isNullOrEmpty(mid)) {
559 continue;
560 }
561 for (final String sdpCandidate : media.attributes.get("candidate")) {
562 final IceUdpTransportInfo.Candidate candidate =
563 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
564 if (candidate != null) {
565 candidateBuilder.put(mid, candidate);
566 }
567 }
568 }
569 return candidateBuilder.build();
570 }
571
572 private void receiveContentReject(final JinglePacket jinglePacket) {
573 final RtpContentMap receivedContentReject;
574 try {
575 receivedContentReject = RtpContentMap.of(jinglePacket);
576 } catch (final RuntimeException e) {
577 Log.d(
578 Config.LOGTAG,
579 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
580 Throwables.getRootCause(e));
581 respondOk(jinglePacket);
582 this.webRTCWrapper.close();
583 sendSessionTerminate(Reason.of(e), e.getMessage());
584 return;
585 }
586
587 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
588 if (outgoingContentAdd == null) {
589 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
590 terminateWithOutOfOrder(jinglePacket);
591 return;
592 }
593 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
594 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
595 this.outgoingContentAdd = null;
596 respondOk(jinglePacket);
597 Log.d(Config.LOGTAG, jinglePacket.toString());
598 receiveContentReject(ourSummary);
599 } else {
600 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
601 terminateWithOutOfOrder(jinglePacket);
602 }
603 }
604
605 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
606 try {
607 this.webRTCWrapper.removeTrack(Media.VIDEO);
608 final RtpContentMap localContentMap = customRollback();
609 modifyLocalContentMap(localContentMap);
610 } catch (final Exception e) {
611 final Throwable cause = Throwables.getRootCause(e);
612 Log.d(
613 Config.LOGTAG,
614 id.getAccount().getJid().asBareJid()
615 + ": unable to rollback local description after receiving content-reject",
616 cause);
617 webRTCWrapper.close();
618 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
619 return;
620 }
621 Log.d(
622 Config.LOGTAG,
623 id.getAccount().getJid().asBareJid()
624 + ": remote has rejected our content-add "
625 + summary);
626 }
627
628 private void receiveContentRemove(final JinglePacket jinglePacket) {
629 final RtpContentMap receivedContentRemove;
630 try {
631 receivedContentRemove = RtpContentMap.of(jinglePacket);
632 receivedContentRemove.requireContentDescriptions();
633 } catch (final RuntimeException e) {
634 Log.d(
635 Config.LOGTAG,
636 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
637 Throwables.getRootCause(e));
638 respondOk(jinglePacket);
639 this.webRTCWrapper.close();
640 sendSessionTerminate(Reason.of(e), e.getMessage());
641 return;
642 }
643 respondOk(jinglePacket);
644 receiveContentRemove(receivedContentRemove);
645 }
646
647 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
648 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
649 final Set<ContentAddition.Summary> contentAddSummary =
650 incomingContentAdd == null
651 ? Collections.emptySet()
652 : ContentAddition.summary(incomingContentAdd);
653 final Set<ContentAddition.Summary> removeSummary =
654 ContentAddition.summary(receivedContentRemove);
655 if (contentAddSummary.equals(removeSummary)) {
656 this.incomingContentAdd = null;
657 updateEndUserState();
658 } else {
659 webRTCWrapper.close();
660 sendSessionTerminate(
661 Reason.FAILED_APPLICATION,
662 String.format(
663 "%s only supports %s as a means to retract a not yet accepted %s",
664 BuildConfig.APP_NAME,
665 JinglePacket.Action.CONTENT_REMOVE,
666 JinglePacket.Action.CONTENT_ADD));
667 }
668 }
669
670 public synchronized void retractContentAdd() {
671 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
672 if (outgoingContentAdd == null) {
673 throw new IllegalStateException("Not outgoing content add");
674 }
675 try {
676 webRTCWrapper.removeTrack(Media.VIDEO);
677 final RtpContentMap localContentMap = customRollback();
678 modifyLocalContentMap(localContentMap);
679 } catch (final Exception e) {
680 final Throwable cause = Throwables.getRootCause(e);
681 Log.d(
682 Config.LOGTAG,
683 id.getAccount().getJid().asBareJid()
684 + ": unable to rollback local description after trying to retract content-add",
685 cause);
686 webRTCWrapper.close();
687 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
688 return;
689 }
690 this.outgoingContentAdd = null;
691 final JinglePacket retract =
692 outgoingContentAdd
693 .toStub()
694 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
695 this.send(retract);
696 Log.d(
697 Config.LOGTAG,
698 id.getAccount().getJid()
699 + ": retract content-add "
700 + ContentAddition.summary(outgoingContentAdd));
701 }
702
703 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
704 final SessionDescription sdp = setLocalSessionDescription();
705 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
706 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
707 this.webRTCWrapper
708 .setRemoteDescription(
709 new org.webrtc.SessionDescription(
710 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
711 .get();
712 return localRtpContentMap;
713 }
714
715 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
716 final RtpContentMap currentRemote = getRemoteContentMap();
717 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
718 if (diff.isEmpty()) {
719 throw new IllegalStateException(
720 "Unexpected rollback condition. No difference between local and remote");
721 }
722 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
723 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
724 final RtpContentMap nextRemote =
725 currentRemote.addContent(
726 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
727 return SessionDescription.of(nextRemote, isResponder());
728 }
729 throw new IllegalStateException(
730 "Unexpected rollback condition. Senders were not uniformly none");
731 }
732
733 public synchronized void acceptContentAdd(
734 @NonNull final Set<ContentAddition.Summary> contentAddition) {
735 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
736 if (incomingContentAdd == null) {
737 throw new IllegalStateException("No incoming content add");
738 }
739
740 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
741 this.incomingContentAdd = null;
742 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
743 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
744 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
745 Log.d(
746 Config.LOGTAG,
747 "content addition is receive only. we want to upgrade to 'both'");
748 final RtpContentMap modifiedSenders =
749 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
750 final JinglePacket proposedContentModification =
751 modifiedSenders
752 .toStub()
753 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
754 proposedContentModification.setTo(id.with);
755 xmppConnectionService.sendIqPacket(
756 id.account,
757 proposedContentModification,
758 (account, response) -> {
759 if (response.getType() == IqPacket.TYPE.RESULT) {
760 Log.d(
761 Config.LOGTAG,
762 id.account.getJid().asBareJid()
763 + ": remote has accepted our upgrade to senders=both");
764 acceptContentAdd(
765 ContentAddition.summary(modifiedSenders), modifiedSenders);
766 } else {
767 Log.d(
768 Config.LOGTAG,
769 id.account.getJid().asBareJid()
770 + ": remote has rejected our upgrade to senders=both");
771 acceptContentAdd(contentAddition, incomingContentAdd);
772 }
773 });
774 } else {
775 acceptContentAdd(contentAddition, incomingContentAdd);
776 }
777 } else {
778 throw new IllegalStateException(
779 "Accepted content add does not match pending content-add");
780 }
781 }
782
783 private void acceptContentAdd(
784 @NonNull final Set<ContentAddition.Summary> contentAddition,
785 final RtpContentMap incomingContentAdd) {
786 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
787 final RtpContentMap modifiedContentMap =
788 getRemoteContentMap().addContent(incomingContentAdd, setup);
789 this.setRemoteContentMap(modifiedContentMap);
790
791 final SessionDescription offer;
792 try {
793 offer = SessionDescription.of(modifiedContentMap, isResponder());
794 } catch (final IllegalArgumentException | NullPointerException e) {
795 Log.d(
796 Config.LOGTAG,
797 id.getAccount().getJid().asBareJid()
798 + ": unable convert offer from content-add to SDP",
799 e);
800 webRTCWrapper.close();
801 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
802 return;
803 }
804 this.incomingContentAdd = null;
805 acceptContentAdd(contentAddition, offer);
806 }
807
808 private void acceptContentAdd(
809 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
810 final org.webrtc.SessionDescription sdp =
811 new org.webrtc.SessionDescription(
812 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
813 try {
814 this.webRTCWrapper.setRemoteDescription(sdp).get();
815
816 // TODO add tracks for 'media' where contentAddition.senders matches
817
818 // TODO if senders.sending(isInitiator())
819
820 this.webRTCWrapper.addTrack(Media.VIDEO);
821
822 // TODO add additional transceivers for recv only cases
823
824 final SessionDescription answer = setLocalSessionDescription();
825 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
826
827 final RtpContentMap contentAcceptMap =
828 rtpContentMap.toContentModification(
829 Collections2.transform(contentAddition, ca -> ca.name));
830
831 Log.d(
832 Config.LOGTAG,
833 id.getAccount().getJid().asBareJid()
834 + ": sending content-accept "
835 + ContentAddition.summary(contentAcceptMap));
836
837 addIceCandidatesFromBlackLog();
838
839 modifyLocalContentMap(rtpContentMap);
840 final ListenableFuture<RtpContentMap> future =
841 prepareOutgoingContentMap(contentAcceptMap);
842 Futures.addCallback(
843 future,
844 new FutureCallback<>() {
845 @Override
846 public void onSuccess(final RtpContentMap rtpContentMap) {
847 sendContentAccept(rtpContentMap);
848 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
849 }
850
851 @Override
852 public void onFailure(@NonNull final Throwable throwable) {
853 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
854 }
855 },
856 MoreExecutors.directExecutor());
857 } catch (final Exception e) {
858 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
859 webRTCWrapper.close();
860 sendSessionTerminate(Reason.FAILED_APPLICATION);
861 }
862 }
863
864 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
865 final JinglePacket jinglePacket =
866 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
867 send(jinglePacket);
868 }
869
870 public synchronized void rejectContentAdd() {
871 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
872 if (incomingContentAdd == null) {
873 throw new IllegalStateException("No incoming content add");
874 }
875 this.incomingContentAdd = null;
876 updateEndUserState();
877 rejectContentAdd(incomingContentAdd);
878 }
879
880 private void rejectContentAdd(final RtpContentMap contentMap) {
881 final JinglePacket jinglePacket =
882 contentMap
883 .toStub()
884 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
885 Log.d(
886 Config.LOGTAG,
887 id.getAccount().getJid().asBareJid()
888 + ": rejecting content "
889 + ContentAddition.summary(contentMap));
890 send(jinglePacket);
891 }
892
893 private boolean checkForIceRestart(
894 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
895 final RtpContentMap existing = getRemoteContentMap();
896 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
897 final IceUdpTransportInfo.Credentials newCredentials;
898 try {
899 existingCredentials = existing.getCredentials();
900 newCredentials = rtpContentMap.getDistinctCredentials();
901 } catch (final IllegalStateException e) {
902 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
903 return false;
904 }
905 if (existingCredentials.contains(newCredentials)) {
906 return false;
907 }
908 // TODO an alternative approach is to check if we already got an iq result to our
909 // ICE-restart
910 // and if that's the case we are seeing an answer.
911 // This might be more spec compliant but also more error prone potentially
912 final boolean isSignalStateStable =
913 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
914 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
915 final boolean isOffer = rtpContentMap.emptyCandidates();
916 final RtpContentMap restartContentMap;
917 try {
918 if (isOffer) {
919 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
920 restartContentMap =
921 existing.modifiedCredentials(
922 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
923 } else {
924 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
925 Log.d(
926 Config.LOGTAG,
927 "received confirmation of ICE restart"
928 + newCredentials
929 + " peer_setup="
930 + setup);
931 // DTLS setup attribute needs to be rewritten to reflect current peer state
932 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
933 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
934 }
935 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
936 return isOffer;
937 } else {
938 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
939 respondWithTieBreak(jinglePacket);
940 return true;
941 }
942 } catch (final Exception exception) {
943 respondOk(jinglePacket);
944 final Throwable rootCause = Throwables.getRootCause(exception);
945 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
946 // If this happens a termination is already in progress
947 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
948 return true;
949 }
950 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
951 webRTCWrapper.close();
952 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
953 return true;
954 }
955 }
956
957 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
958 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
959 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
960 throw new IllegalStateException("Invalid peer setup");
961 }
962 return peerSetup;
963 }
964
965 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
966 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
967 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
968 }
969 this.peerDtlsSetup = setup;
970 }
971
972 private boolean applyIceRestart(
973 final JinglePacket jinglePacket,
974 final RtpContentMap restartContentMap,
975 final boolean isOffer)
976 throws ExecutionException, InterruptedException {
977 final SessionDescription sessionDescription =
978 SessionDescription.of(restartContentMap, isResponder());
979 final org.webrtc.SessionDescription.Type type =
980 isOffer
981 ? org.webrtc.SessionDescription.Type.OFFER
982 : org.webrtc.SessionDescription.Type.ANSWER;
983 org.webrtc.SessionDescription sdp =
984 new org.webrtc.SessionDescription(type, sessionDescription.toString());
985 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
986 if (isInitiator()) {
987 // We ignore the offer and respond with tie-break. This will clause the responder
988 // not to apply the content map
989 return false;
990 }
991 }
992 webRTCWrapper.setRemoteDescription(sdp).get();
993 setRemoteContentMap(restartContentMap);
994 if (isOffer) {
995 final SessionDescription localSessionDescription = setLocalSessionDescription();
996 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
997 // We need to respond OK before sending any candidates
998 respondOk(jinglePacket);
999 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1000 } else {
1001 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1002 }
1003 return true;
1004 }
1005
1006 private void processCandidates(
1007 final Set<Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>>> contents) {
1008 for (final Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>> content : contents) {
1009 processCandidate(content);
1010 }
1011 }
1012
1013 private void processCandidate(
1014 final Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>> content) {
1015 final RtpContentMap rtpContentMap = getRemoteContentMap();
1016 final List<String> indices = toIdentificationTags(rtpContentMap);
1017 final String sdpMid = content.getKey(); // aka content name
1018 final IceUdpTransportInfo transport = content.getValue().transport;
1019 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1020
1021 // TODO check that credentials remained the same
1022
1023 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1024 final String sdp;
1025 try {
1026 sdp = candidate.toSdpAttribute(credentials.ufrag);
1027 } catch (final IllegalArgumentException e) {
1028 Log.d(
1029 Config.LOGTAG,
1030 id.account.getJid().asBareJid()
1031 + ": ignoring invalid ICE candidate "
1032 + e.getMessage());
1033 continue;
1034 }
1035 final int mLineIndex = indices.indexOf(sdpMid);
1036 if (mLineIndex < 0) {
1037 Log.w(
1038 Config.LOGTAG,
1039 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1040 }
1041 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1042 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1043 this.webRTCWrapper.addIceCandidate(iceCandidate);
1044 }
1045 }
1046
1047 private RtpContentMap getRemoteContentMap() {
1048 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1049 }
1050
1051 private RtpContentMap getLocalContentMap() {
1052 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1053 }
1054
1055 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1056 final Group originalGroup = rtpContentMap.group;
1057 final List<String> identificationTags =
1058 originalGroup == null
1059 ? rtpContentMap.getNames()
1060 : originalGroup.getIdentificationTags();
1061 if (identificationTags.size() == 0) {
1062 Log.w(
1063 Config.LOGTAG,
1064 id.account.getJid().asBareJid()
1065 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1066 }
1067 return identificationTags;
1068 }
1069
1070 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1071 final JinglePacket jinglePacket, final boolean expectVerification) {
1072 try {
1073 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1074 } catch (final Exception e) {
1075 return Futures.immediateFailedFuture(e);
1076 }
1077 }
1078
1079 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1080 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1081 Log.d(
1082 Config.LOGTAG,
1083 "receiveRtpContentMap("
1084 + receivedContentMap.getClass().getSimpleName()
1085 + ",expectVerification="
1086 + expectVerification
1087 + ")");
1088 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1089 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1090 id.account
1091 .getAxolotlService()
1092 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1093 return Futures.transform(
1094 future,
1095 omemoVerifiedPayload -> {
1096 // TODO test if an exception here triggers a correct abort
1097 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1098 Log.d(
1099 Config.LOGTAG,
1100 id.account.getJid().asBareJid()
1101 + ": received verifiable DTLS fingerprint via "
1102 + omemoVerification);
1103 return omemoVerifiedPayload.getPayload();
1104 },
1105 MoreExecutors.directExecutor());
1106 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1107 return Futures.immediateFailedFuture(
1108 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1109 } else {
1110 return Futures.immediateFuture(receivedContentMap);
1111 }
1112 }
1113
1114 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1115 if (isInitiator()) {
1116 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_INITIATE);
1117 return;
1118 }
1119 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1120 Futures.addCallback(
1121 future,
1122 new FutureCallback<>() {
1123 @Override
1124 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1125 receiveSessionInitiate(jinglePacket, rtpContentMap);
1126 }
1127
1128 @Override
1129 public void onFailure(@NonNull final Throwable throwable) {
1130 respondOk(jinglePacket);
1131 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1132 }
1133 },
1134 MoreExecutors.directExecutor());
1135 }
1136
1137 private void receiveSessionInitiate(
1138 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1139 try {
1140 contentMap.requireContentDescriptions();
1141 contentMap.requireDTLSFingerprint(true);
1142 } catch (final RuntimeException e) {
1143 Log.d(
1144 Config.LOGTAG,
1145 id.account.getJid().asBareJid() + ": improperly formatted contents",
1146 Throwables.getRootCause(e));
1147 respondOk(jinglePacket);
1148 sendSessionTerminate(Reason.of(e), e.getMessage());
1149 return;
1150 }
1151 Log.d(
1152 Config.LOGTAG,
1153 "processing session-init with " + contentMap.contents.size() + " contents");
1154 final State target;
1155 if (this.state == State.PROCEED) {
1156 Preconditions.checkState(
1157 proposedMedia != null && proposedMedia.size() > 0,
1158 "proposed media must be set when processing pre-approved session-initiate");
1159 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1160 sendSessionTerminate(
1161 Reason.SECURITY_ERROR,
1162 String.format(
1163 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1164 this.proposedMedia, contentMap.getMedia()));
1165 return;
1166 }
1167 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1168 } else {
1169 target = State.SESSION_INITIALIZED;
1170 }
1171 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1172 respondOk(jinglePacket);
1173 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1174 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1175 Log.d(
1176 Config.LOGTAG,
1177 id.account.getJid().asBareJid()
1178 + ": automatically accepting session-initiate");
1179 sendSessionAccept();
1180 } else {
1181 Log.d(
1182 Config.LOGTAG,
1183 id.account.getJid().asBareJid()
1184 + ": received not pre-approved session-initiate. start ringing");
1185 startRinging();
1186 }
1187 } else {
1188 Log.d(
1189 Config.LOGTAG,
1190 String.format(
1191 "%s: received session-initiate while in state %s",
1192 id.account.getJid().asBareJid(), state));
1193 terminateWithOutOfOrder(jinglePacket);
1194 }
1195 }
1196
1197 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1198 if (isResponder()) {
1199 receiveOutOfOrderAction(jinglePacket, JinglePacket.Action.SESSION_ACCEPT);
1200 return;
1201 }
1202 final ListenableFuture<RtpContentMap> future =
1203 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1204 Futures.addCallback(
1205 future,
1206 new FutureCallback<>() {
1207 @Override
1208 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1209 receiveSessionAccept(jinglePacket, rtpContentMap);
1210 }
1211
1212 @Override
1213 public void onFailure(@NonNull final Throwable throwable) {
1214 respondOk(jinglePacket);
1215 Log.d(
1216 Config.LOGTAG,
1217 id.account.getJid().asBareJid()
1218 + ": improperly formatted contents in session-accept",
1219 throwable);
1220 webRTCWrapper.close();
1221 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1222 }
1223 },
1224 MoreExecutors.directExecutor());
1225 }
1226
1227 private void receiveSessionAccept(
1228 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1229 try {
1230 contentMap.requireContentDescriptions();
1231 contentMap.requireDTLSFingerprint();
1232 } catch (final RuntimeException e) {
1233 respondOk(jinglePacket);
1234 Log.d(
1235 Config.LOGTAG,
1236 id.account.getJid().asBareJid()
1237 + ": improperly formatted contents in session-accept",
1238 e);
1239 webRTCWrapper.close();
1240 sendSessionTerminate(Reason.of(e), e.getMessage());
1241 return;
1242 }
1243 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1244 if (!initiatorMedia.equals(contentMap.getMedia())) {
1245 sendSessionTerminate(
1246 Reason.SECURITY_ERROR,
1247 String.format(
1248 "Your session-included included media %s but our session-initiate was %s",
1249 this.proposedMedia, contentMap.getMedia()));
1250 return;
1251 }
1252 Log.d(
1253 Config.LOGTAG,
1254 "processing session-accept with " + contentMap.contents.size() + " contents");
1255 if (transition(State.SESSION_ACCEPTED)) {
1256 respondOk(jinglePacket);
1257 receiveSessionAccept(contentMap);
1258 } else {
1259 Log.d(
1260 Config.LOGTAG,
1261 String.format(
1262 "%s: received session-accept while in state %s",
1263 id.account.getJid().asBareJid(), state));
1264 respondOk(jinglePacket);
1265 }
1266 }
1267
1268 private void receiveSessionAccept(final RtpContentMap contentMap) {
1269 this.responderRtpContentMap = contentMap;
1270 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1271 final SessionDescription sessionDescription;
1272 try {
1273 sessionDescription = SessionDescription.of(contentMap, false);
1274 } catch (final IllegalArgumentException | NullPointerException e) {
1275 Log.d(
1276 Config.LOGTAG,
1277 id.account.getJid().asBareJid()
1278 + ": unable convert offer from session-accept to SDP",
1279 e);
1280 webRTCWrapper.close();
1281 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1282 return;
1283 }
1284 final org.webrtc.SessionDescription answer =
1285 new org.webrtc.SessionDescription(
1286 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1287 try {
1288 this.webRTCWrapper.setRemoteDescription(answer).get();
1289 } catch (final Exception e) {
1290 Log.d(
1291 Config.LOGTAG,
1292 id.account.getJid().asBareJid()
1293 + ": unable to set remote description after receiving session-accept",
1294 Throwables.getRootCause(e));
1295 webRTCWrapper.close();
1296 sendSessionTerminate(
1297 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1298 return;
1299 }
1300 processCandidates(contentMap.contents.entrySet());
1301 }
1302
1303 private void sendSessionAccept() {
1304 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1305 if (rtpContentMap == null) {
1306 throw new IllegalStateException("initiator RTP Content Map has not been set");
1307 }
1308 final SessionDescription offer;
1309 try {
1310 offer = SessionDescription.of(rtpContentMap, true);
1311 } catch (final IllegalArgumentException | NullPointerException e) {
1312 Log.d(
1313 Config.LOGTAG,
1314 id.account.getJid().asBareJid()
1315 + ": unable convert offer from session-initiate to SDP",
1316 e);
1317 webRTCWrapper.close();
1318 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1319 return;
1320 }
1321 sendSessionAccept(rtpContentMap.getMedia(), offer);
1322 }
1323
1324 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1325 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1326 }
1327
1328 private synchronized void sendSessionAccept(
1329 final Set<Media> media,
1330 final SessionDescription offer,
1331 final List<PeerConnection.IceServer> iceServers) {
1332 if (isTerminated()) {
1333 Log.w(
1334 Config.LOGTAG,
1335 id.account.getJid().asBareJid()
1336 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1337 return;
1338 }
1339 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1340 try {
1341 setupWebRTC(media, iceServers, !includeCandidates);
1342 } catch (final WebRTCWrapper.InitializationException e) {
1343 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1344 webRTCWrapper.close();
1345 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1346 return;
1347 }
1348 final org.webrtc.SessionDescription sdp =
1349 new org.webrtc.SessionDescription(
1350 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1351 try {
1352 this.webRTCWrapper.setRemoteDescription(sdp).get();
1353 addIceCandidatesFromBlackLog();
1354 org.webrtc.SessionDescription webRTCSessionDescription =
1355 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1356 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1357 } catch (final Exception e) {
1358 failureToAcceptSession(e);
1359 }
1360 }
1361
1362 private void failureToAcceptSession(final Throwable throwable) {
1363 if (isTerminated()) {
1364 return;
1365 }
1366 final Throwable rootCause = Throwables.getRootCause(throwable);
1367 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1368 webRTCWrapper.close();
1369 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1370 }
1371
1372 private void failureToPerformAction(
1373 final JinglePacket.Action action, final Throwable throwable) {
1374 if (isTerminated()) {
1375 return;
1376 }
1377 final Throwable rootCause = Throwables.getRootCause(throwable);
1378 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1379 webRTCWrapper.close();
1380 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1381 }
1382
1383 private void addIceCandidatesFromBlackLog() {
1384 Map.Entry<String, DescriptionTransport<RtpDescription,IceUdpTransportInfo>> foo;
1385 while ((foo = this.pendingIceCandidates.poll()) != null) {
1386 processCandidate(foo);
1387 Log.d(
1388 Config.LOGTAG,
1389 id.account.getJid().asBareJid() + ": added candidate from back log");
1390 }
1391 }
1392
1393 private void prepareSessionAccept(
1394 final org.webrtc.SessionDescription webRTCSessionDescription,
1395 final boolean includeCandidates) {
1396 final SessionDescription sessionDescription =
1397 SessionDescription.parse(webRTCSessionDescription.description);
1398 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1399 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1400 if (includeCandidates) {
1401 candidates = parseCandidates(sessionDescription);
1402 } else {
1403 candidates = ImmutableMultimap.of();
1404 }
1405 this.responderRtpContentMap = respondingRtpContentMap;
1406 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1407 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1408 prepareOutgoingContentMap(respondingRtpContentMap);
1409 Futures.addCallback(
1410 outgoingContentMapFuture,
1411 new FutureCallback<>() {
1412 @Override
1413 public void onSuccess(final RtpContentMap outgoingContentMap) {
1414 if (includeCandidates) {
1415 Log.d(
1416 Config.LOGTAG,
1417 "including "
1418 + candidates.size()
1419 + " candidates in session accept");
1420 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1421 } else {
1422 sendSessionAccept(outgoingContentMap);
1423 }
1424 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1425 }
1426
1427 @Override
1428 public void onFailure(@NonNull Throwable throwable) {
1429 failureToAcceptSession(throwable);
1430 }
1431 },
1432 MoreExecutors.directExecutor());
1433 }
1434
1435 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1436 if (isTerminated()) {
1437 Log.w(
1438 Config.LOGTAG,
1439 id.account.getJid().asBareJid()
1440 + ": preparing session accept was too slow. already terminated. nothing to do.");
1441 return;
1442 }
1443 transitionOrThrow(State.SESSION_ACCEPTED);
1444 final JinglePacket sessionAccept =
1445 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1446 send(sessionAccept);
1447 }
1448
1449 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1450 final RtpContentMap rtpContentMap) {
1451 if (this.omemoVerification.hasDeviceId()) {
1452 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1453 verifiedPayloadFuture =
1454 id.account
1455 .getAxolotlService()
1456 .encrypt(
1457 rtpContentMap,
1458 id.with,
1459 omemoVerification.getDeviceId());
1460 return Futures.transform(
1461 verifiedPayloadFuture,
1462 verifiedPayload -> {
1463 omemoVerification.setOrEnsureEqual(verifiedPayload);
1464 return verifiedPayload.getPayload();
1465 },
1466 MoreExecutors.directExecutor());
1467 } else {
1468 return Futures.immediateFuture(rtpContentMap);
1469 }
1470 }
1471
1472 synchronized void deliveryMessage(
1473 final Jid from,
1474 final Element message,
1475 final String serverMessageId,
1476 final long timestamp) {
1477 Log.d(
1478 Config.LOGTAG,
1479 id.account.getJid().asBareJid()
1480 + ": delivered message to JingleRtpConnection "
1481 + message);
1482 switch (message.getName()) {
1483 case "propose" -> receivePropose(
1484 from, Propose.upgrade(message), serverMessageId, timestamp);
1485 case "proceed" -> receiveProceed(
1486 from, Proceed.upgrade(message), serverMessageId, timestamp);
1487 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1488 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1489 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1490 }
1491 }
1492
1493 void deliverFailedProceed(final String message) {
1494 Log.d(
1495 Config.LOGTAG,
1496 id.account.getJid().asBareJid()
1497 + ": receive message error for proceed message ("
1498 + Strings.nullToEmpty(message)
1499 + ")");
1500 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1501 webRTCWrapper.close();
1502 Log.d(
1503 Config.LOGTAG,
1504 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1505 this.finish();
1506 }
1507 }
1508
1509 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1510 final boolean originatedFromMyself =
1511 from.asBareJid().equals(id.account.getJid().asBareJid());
1512 if (originatedFromMyself) {
1513 if (transition(State.ACCEPTED)) {
1514 acceptedOnOtherDevice(serverMsgId, timestamp);
1515 } else {
1516 Log.d(
1517 Config.LOGTAG,
1518 id.account.getJid().asBareJid()
1519 + ": unable to transition to accept because already in state="
1520 + this.state);
1521 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1522 }
1523 } else {
1524 Log.d(
1525 Config.LOGTAG,
1526 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1527 }
1528 }
1529
1530 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1531 if (serverMsgId != null) {
1532 this.message.setServerMsgId(serverMsgId);
1533 }
1534 this.message.setTime(timestamp);
1535 this.message.setCarbon(true); // indicate that call was accepted on other device
1536 this.writeLogMessageSuccess(0);
1537 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1538 this.finish();
1539 }
1540
1541 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1542 final boolean originatedFromMyself =
1543 from.asBareJid().equals(id.account.getJid().asBareJid());
1544 // reject from another one of my clients
1545 if (originatedFromMyself) {
1546 receiveRejectFromMyself(serverMsgId, timestamp);
1547 } else if (isInitiator()) {
1548 if (from.equals(id.with)) {
1549 receiveRejectFromResponder();
1550 } else {
1551 Log.d(
1552 Config.LOGTAG,
1553 id.account.getJid()
1554 + ": ignoring reject from "
1555 + from
1556 + " for session with "
1557 + id.with);
1558 }
1559 } else {
1560 Log.d(
1561 Config.LOGTAG,
1562 id.account.getJid()
1563 + ": ignoring reject from "
1564 + from
1565 + " for session with "
1566 + id.with);
1567 }
1568 }
1569
1570 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1571 if (transition(State.REJECTED)) {
1572 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1573 this.finish();
1574 if (serverMsgId != null) {
1575 this.message.setServerMsgId(serverMsgId);
1576 }
1577 this.message.setTime(timestamp);
1578 this.message.setCarbon(true); // indicate that call was rejected on other device
1579 writeLogMessageMissed();
1580 } else {
1581 Log.d(
1582 Config.LOGTAG,
1583 "not able to transition into REJECTED because already in " + this.state);
1584 }
1585 }
1586
1587 private void receiveRejectFromResponder() {
1588 if (isInState(State.PROCEED)) {
1589 Log.d(
1590 Config.LOGTAG,
1591 id.account.getJid()
1592 + ": received reject while still in proceed. callee reconsidered");
1593 closeTransitionLogFinish(State.REJECTED_RACED);
1594 return;
1595 }
1596 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1597 Log.d(
1598 Config.LOGTAG,
1599 id.account.getJid()
1600 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1601 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1602 return;
1603 }
1604 Log.d(
1605 Config.LOGTAG,
1606 id.account.getJid()
1607 + ": ignoring reject from responder because already in state "
1608 + this.state);
1609 }
1610
1611 private void receivePropose(
1612 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1613 final boolean originatedFromMyself =
1614 from.asBareJid().equals(id.account.getJid().asBareJid());
1615 if (originatedFromMyself) {
1616 Log.d(
1617 Config.LOGTAG,
1618 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1619 } else if (transition(
1620 State.PROPOSED,
1621 () -> {
1622 final Collection<RtpDescription> descriptions =
1623 Collections2.transform(
1624 Collections2.filter(
1625 propose.getDescriptions(),
1626 d -> d instanceof RtpDescription),
1627 input -> (RtpDescription) input);
1628 final Collection<Media> media =
1629 Collections2.transform(descriptions, RtpDescription::getMedia);
1630 Preconditions.checkState(
1631 !media.contains(Media.UNKNOWN),
1632 "RTP descriptions contain unknown media");
1633 Log.d(
1634 Config.LOGTAG,
1635 id.account.getJid().asBareJid()
1636 + ": received session proposal from "
1637 + from
1638 + " for "
1639 + media);
1640 this.proposedMedia = Sets.newHashSet(media);
1641 })) {
1642 if (serverMsgId != null) {
1643 this.message.setServerMsgId(serverMsgId);
1644 }
1645 this.message.setTime(timestamp);
1646 startRinging();
1647 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1648 sendJingleMessage("ringing");
1649 }
1650 } else {
1651 Log.d(
1652 Config.LOGTAG,
1653 id.account.getJid()
1654 + ": ignoring session proposal because already in "
1655 + state);
1656 }
1657 }
1658
1659 private void startRinging() {
1660 Log.d(
1661 Config.LOGTAG,
1662 id.account.getJid().asBareJid()
1663 + ": received call from "
1664 + id.with
1665 + ". start ringing");
1666 ringingTimeoutFuture =
1667 jingleConnectionManager.schedule(
1668 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1669 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1670 }
1671
1672 private synchronized void ringingTimeout() {
1673 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1674 switch (this.state) {
1675 case PROPOSED -> {
1676 message.markUnread();
1677 rejectCallFromProposed();
1678 }
1679 case SESSION_INITIALIZED -> {
1680 message.markUnread();
1681 rejectCallFromSessionInitiate();
1682 }
1683 }
1684 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1685 }
1686
1687 private void cancelRingingTimeout() {
1688 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1689 if (future != null && !future.isCancelled()) {
1690 future.cancel(false);
1691 }
1692 }
1693
1694 private void receiveProceed(
1695 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1696 final Set<Media> media =
1697 Preconditions.checkNotNull(
1698 this.proposedMedia, "Proposed media has to be set before handling proceed");
1699 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1700 if (from.equals(id.with)) {
1701 if (isInitiator()) {
1702 if (transition(State.PROCEED)) {
1703 if (serverMsgId != null) {
1704 this.message.setServerMsgId(serverMsgId);
1705 }
1706 this.message.setTime(timestamp);
1707 final Integer remoteDeviceId = proceed.getDeviceId();
1708 if (isOmemoEnabled()) {
1709 this.omemoVerification.setDeviceId(remoteDeviceId);
1710 } else {
1711 if (remoteDeviceId != null) {
1712 Log.d(
1713 Config.LOGTAG,
1714 id.account.getJid().asBareJid()
1715 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1716 }
1717 this.omemoVerification.setDeviceId(null);
1718 }
1719 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1720 } else {
1721 Log.d(
1722 Config.LOGTAG,
1723 String.format(
1724 "%s: ignoring proceed because already in %s",
1725 id.account.getJid().asBareJid(), this.state));
1726 }
1727 } else {
1728 Log.d(
1729 Config.LOGTAG,
1730 String.format(
1731 "%s: ignoring proceed because we were not initializing",
1732 id.account.getJid().asBareJid()));
1733 }
1734 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1735 if (transition(State.ACCEPTED)) {
1736 Log.d(
1737 Config.LOGTAG,
1738 id.account.getJid().asBareJid()
1739 + ": moved session with "
1740 + id.with
1741 + " into state accepted after received carbon copied proceed");
1742 acceptedOnOtherDevice(serverMsgId, timestamp);
1743 }
1744 } else {
1745 Log.d(
1746 Config.LOGTAG,
1747 String.format(
1748 "%s: ignoring proceed from %s. was expected from %s",
1749 id.account.getJid().asBareJid(), from, id.with));
1750 }
1751 }
1752
1753 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1754 if (from.equals(id.with)) {
1755 final State target =
1756 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1757 if (transition(target)) {
1758 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1759 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1760 Log.d(
1761 Config.LOGTAG,
1762 id.account.getJid().asBareJid()
1763 + ": session with "
1764 + id.with
1765 + " has been retracted (serverMsgId="
1766 + serverMsgId
1767 + ")");
1768 if (serverMsgId != null) {
1769 this.message.setServerMsgId(serverMsgId);
1770 }
1771 this.message.setTime(timestamp);
1772 if (target == State.RETRACTED) {
1773 this.message.markUnread();
1774 }
1775 writeLogMessageMissed();
1776 finish();
1777 } else {
1778 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1779 }
1780 } else {
1781 // TODO parse retract from self
1782 Log.d(
1783 Config.LOGTAG,
1784 id.account.getJid().asBareJid()
1785 + ": received retract from "
1786 + from
1787 + ". expected retract from"
1788 + id.with
1789 + ". ignoring");
1790 }
1791 }
1792
1793 public void sendSessionInitiate() {
1794 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1795 }
1796
1797 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1798 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1799 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1800 }
1801
1802 private synchronized void sendSessionInitiate(
1803 final Set<Media> media,
1804 final State targetState,
1805 final List<PeerConnection.IceServer> iceServers) {
1806 if (isTerminated()) {
1807 Log.w(
1808 Config.LOGTAG,
1809 id.account.getJid().asBareJid()
1810 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1811 return;
1812 }
1813 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1814 try {
1815 setupWebRTC(media, iceServers, !includeCandidates);
1816 } catch (final WebRTCWrapper.InitializationException e) {
1817 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1818 webRTCWrapper.close();
1819 sendRetract(Reason.ofThrowable(e));
1820 return;
1821 }
1822 try {
1823 org.webrtc.SessionDescription webRTCSessionDescription =
1824 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1825 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1826 } catch (final Exception e) {
1827 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1828 // exceptions
1829 failureToInitiateSession(e, targetState);
1830 }
1831 }
1832
1833 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1834 if (isTerminated()) {
1835 return;
1836 }
1837 Log.d(
1838 Config.LOGTAG,
1839 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1840 Throwables.getRootCause(throwable));
1841 webRTCWrapper.close();
1842 final Reason reason = Reason.ofThrowable(throwable);
1843 if (isInState(targetState)) {
1844 sendSessionTerminate(reason, throwable.getMessage());
1845 } else {
1846 sendRetract(reason);
1847 }
1848 }
1849
1850 private void sendRetract(final Reason reason) {
1851 // TODO embed reason into retract
1852 sendJingleMessage("retract", id.with.asBareJid());
1853 transitionOrThrow(reasonToState(reason));
1854 this.finish();
1855 }
1856
1857 private void prepareSessionInitiate(
1858 final org.webrtc.SessionDescription webRTCSessionDescription,
1859 final boolean includeCandidates,
1860 final State targetState) {
1861 final SessionDescription sessionDescription =
1862 SessionDescription.parse(webRTCSessionDescription.description);
1863 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1864 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1865 if (includeCandidates) {
1866 candidates = parseCandidates(sessionDescription);
1867 } else {
1868 candidates = ImmutableMultimap.of();
1869 }
1870 this.initiatorRtpContentMap = rtpContentMap;
1871 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1872 encryptSessionInitiate(rtpContentMap);
1873 Futures.addCallback(
1874 outgoingContentMapFuture,
1875 new FutureCallback<>() {
1876 @Override
1877 public void onSuccess(final RtpContentMap outgoingContentMap) {
1878 if (includeCandidates) {
1879 Log.d(
1880 Config.LOGTAG,
1881 "including "
1882 + candidates.size()
1883 + " candidates in session initiate");
1884 sendSessionInitiate(
1885 outgoingContentMap.withCandidates(candidates), targetState);
1886 } else {
1887 sendSessionInitiate(outgoingContentMap, targetState);
1888 }
1889 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1890 }
1891
1892 @Override
1893 public void onFailure(@NonNull final Throwable throwable) {
1894 failureToInitiateSession(throwable, targetState);
1895 }
1896 },
1897 MoreExecutors.directExecutor());
1898 }
1899
1900 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1901 if (isTerminated()) {
1902 Log.w(
1903 Config.LOGTAG,
1904 id.account.getJid().asBareJid()
1905 + ": preparing session was too slow. already terminated. nothing to do.");
1906 return;
1907 }
1908 this.transitionOrThrow(targetState);
1909 final JinglePacket sessionInitiate =
1910 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1911 send(sessionInitiate);
1912 }
1913
1914 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1915 final RtpContentMap rtpContentMap) {
1916 if (this.omemoVerification.hasDeviceId()) {
1917 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1918 verifiedPayloadFuture =
1919 id.account
1920 .getAxolotlService()
1921 .encrypt(
1922 rtpContentMap,
1923 id.with,
1924 omemoVerification.getDeviceId());
1925 final ListenableFuture<RtpContentMap> future =
1926 Futures.transform(
1927 verifiedPayloadFuture,
1928 verifiedPayload -> {
1929 omemoVerification.setSessionFingerprint(
1930 verifiedPayload.getFingerprint());
1931 return verifiedPayload.getPayload();
1932 },
1933 MoreExecutors.directExecutor());
1934 if (Config.REQUIRE_RTP_VERIFICATION) {
1935 return future;
1936 }
1937 return Futures.catching(
1938 future,
1939 CryptoFailedException.class,
1940 e -> {
1941 Log.w(
1942 Config.LOGTAG,
1943 id.account.getJid().asBareJid()
1944 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1945 e);
1946 return rtpContentMap;
1947 },
1948 MoreExecutors.directExecutor());
1949 } else {
1950 return Futures.immediateFuture(rtpContentMap);
1951 }
1952 }
1953
1954 protected void sendSessionTerminate(final Reason reason) {
1955 sendSessionTerminate(reason, null);
1956 }
1957
1958
1959 protected void sendSessionTerminate(final Reason reason, final String text) {
1960 sendSessionTerminate(reason,text, this::writeLogMessage);
1961 }
1962
1963
1964 private void sendTransportInfo(
1965 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1966 final RtpContentMap transportInfo;
1967 try {
1968 final RtpContentMap rtpContentMap =
1969 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1970 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1971 } catch (final Exception e) {
1972 Log.d(
1973 Config.LOGTAG,
1974 id.account.getJid().asBareJid()
1975 + ": unable to prepare transport-info from candidate for content="
1976 + contentName);
1977 return;
1978 }
1979 final JinglePacket jinglePacket =
1980 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1981 send(jinglePacket);
1982 }
1983
1984 public RtpEndUserState getEndUserState() {
1985 switch (this.state) {
1986 case NULL, PROPOSED, SESSION_INITIALIZED -> {
1987 if (isInitiator()) {
1988 return RtpEndUserState.RINGING;
1989 } else {
1990 return RtpEndUserState.INCOMING_CALL;
1991 }
1992 }
1993 case PROCEED -> {
1994 if (isInitiator()) {
1995 return RtpEndUserState.RINGING;
1996 } else {
1997 return RtpEndUserState.ACCEPTING_CALL;
1998 }
1999 }
2000 case SESSION_INITIALIZED_PRE_APPROVED -> {
2001 if (isInitiator()) {
2002 return RtpEndUserState.RINGING;
2003 } else {
2004 return RtpEndUserState.CONNECTING;
2005 }
2006 }
2007 case SESSION_ACCEPTED -> {
2008 final ContentAddition ca = getPendingContentAddition();
2009 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2010 return RtpEndUserState.INCOMING_CONTENT_ADD;
2011 }
2012 return getPeerConnectionStateAsEndUserState();
2013 }
2014 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2015 if (isInitiator()) {
2016 return RtpEndUserState.DECLINED_OR_BUSY;
2017 } else {
2018 return RtpEndUserState.ENDED;
2019 }
2020 }
2021 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2022 return RtpEndUserState.ENDED;
2023 }
2024 case RETRACTED_RACED -> {
2025 if (isInitiator()) {
2026 return RtpEndUserState.ENDED;
2027 } else {
2028 return RtpEndUserState.RETRACTED;
2029 }
2030 }
2031 case TERMINATED_CONNECTIVITY_ERROR -> {
2032 return zeroDuration()
2033 ? RtpEndUserState.CONNECTIVITY_ERROR
2034 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2035 }
2036 case TERMINATED_APPLICATION_FAILURE -> {
2037 return RtpEndUserState.APPLICATION_ERROR;
2038 }
2039 case TERMINATED_SECURITY_ERROR -> {
2040 return RtpEndUserState.SECURITY_ERROR;
2041 }
2042 }
2043 throw new IllegalStateException(
2044 String.format("%s has no equivalent EndUserState", this.state));
2045 }
2046
2047 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2048 final PeerConnection.PeerConnectionState state;
2049 try {
2050 state = webRTCWrapper.getState();
2051 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2052 // We usually close the WebRTCWrapper *before* transitioning so we might still
2053 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2054 return RtpEndUserState.ENDING_CALL;
2055 }
2056 return switch (state) {
2057 case CONNECTED -> RtpEndUserState.CONNECTED;
2058 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2059 case CLOSED -> RtpEndUserState.ENDING_CALL;
2060 default -> zeroDuration()
2061 ? RtpEndUserState.CONNECTIVITY_ERROR
2062 : RtpEndUserState.RECONNECTING;
2063 };
2064 }
2065
2066 public ContentAddition getPendingContentAddition() {
2067 final RtpContentMap in = this.incomingContentAdd;
2068 final RtpContentMap out = this.outgoingContentAdd;
2069 if (out != null) {
2070 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2071 } else if (in != null) {
2072 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2073 } else {
2074 return null;
2075 }
2076 }
2077
2078 public Set<Media> getMedia() {
2079 final State current = getState();
2080 if (current == State.NULL) {
2081 if (isInitiator()) {
2082 return Preconditions.checkNotNull(
2083 this.proposedMedia, "RTP connection has not been initialized properly");
2084 }
2085 throw new IllegalStateException("RTP connection has not been initialized yet");
2086 }
2087 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2088 return Preconditions.checkNotNull(
2089 this.proposedMedia, "RTP connection has not been initialized properly");
2090 }
2091 final RtpContentMap localContentMap = getLocalContentMap();
2092 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2093 if (localContentMap != null) {
2094 return localContentMap.getMedia();
2095 } else if (initiatorContentMap != null) {
2096 return initiatorContentMap.getMedia();
2097 } else if (isTerminated()) {
2098 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2099 } else {
2100 return Preconditions.checkNotNull(
2101 this.proposedMedia, "RTP connection has not been initialized properly");
2102 }
2103 }
2104
2105 public boolean isVerified() {
2106 final String fingerprint = this.omemoVerification.getFingerprint();
2107 if (fingerprint == null) {
2108 return false;
2109 }
2110 final FingerprintStatus status =
2111 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2112 return status != null && status.isVerified();
2113 }
2114
2115 public boolean addMedia(final Media media) {
2116 final Set<Media> currentMedia = getMedia();
2117 if (currentMedia.contains(media)) {
2118 throw new IllegalStateException(String.format("%s has already been proposed", media));
2119 }
2120 // TODO add state protection - can only add while ACCEPTED or so
2121 Log.d(Config.LOGTAG, "adding media: " + media);
2122 return webRTCWrapper.addTrack(media);
2123 }
2124
2125 public synchronized void acceptCall() {
2126 switch (this.state) {
2127 case PROPOSED -> {
2128 cancelRingingTimeout();
2129 acceptCallFromProposed();
2130 }
2131 case SESSION_INITIALIZED -> {
2132 cancelRingingTimeout();
2133 acceptCallFromSessionInitialized();
2134 }
2135 case ACCEPTED -> Log.w(
2136 Config.LOGTAG,
2137 id.account.getJid().asBareJid()
2138 + ": the call has already been accepted with another client. UI was just lagging behind");
2139 case PROCEED, SESSION_ACCEPTED -> Log.w(
2140 Config.LOGTAG,
2141 id.account.getJid().asBareJid()
2142 + ": the call has already been accepted. user probably double tapped the UI");
2143 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2144 }
2145 }
2146
2147 public void notifyPhoneCall() {
2148 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2149 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2150 rejectCall();
2151 } else {
2152 endCall();
2153 }
2154 }
2155
2156 public synchronized void rejectCall() {
2157 if (isTerminated()) {
2158 Log.w(
2159 Config.LOGTAG,
2160 id.account.getJid().asBareJid()
2161 + ": received rejectCall() when session has already been terminated. nothing to do");
2162 return;
2163 }
2164 switch (this.state) {
2165 case PROPOSED -> rejectCallFromProposed();
2166 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2167 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2168 }
2169 }
2170
2171 public synchronized void endCall() {
2172 if (isTerminated()) {
2173 Log.w(
2174 Config.LOGTAG,
2175 id.account.getJid().asBareJid()
2176 + ": received endCall() when session has already been terminated. nothing to do");
2177 return;
2178 }
2179 if (isInState(State.PROPOSED) && isResponder()) {
2180 rejectCallFromProposed();
2181 return;
2182 }
2183 if (isInState(State.PROCEED)) {
2184 if (isInitiator()) {
2185 retractFromProceed();
2186 } else {
2187 rejectCallFromProceed();
2188 }
2189 return;
2190 }
2191 if (isInitiator()
2192 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2193 this.webRTCWrapper.close();
2194 sendSessionTerminate(Reason.CANCEL);
2195 return;
2196 }
2197 if (isInState(State.SESSION_INITIALIZED)) {
2198 rejectCallFromSessionInitiate();
2199 return;
2200 }
2201 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2202 this.webRTCWrapper.close();
2203 sendSessionTerminate(Reason.SUCCESS);
2204 return;
2205 }
2206 if (isInState(
2207 State.TERMINATED_APPLICATION_FAILURE,
2208 State.TERMINATED_CONNECTIVITY_ERROR,
2209 State.TERMINATED_DECLINED_OR_BUSY)) {
2210 Log.d(
2211 Config.LOGTAG,
2212 "ignoring request to end call because already in state " + this.state);
2213 return;
2214 }
2215 throw new IllegalStateException(
2216 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2217 }
2218
2219 private void retractFromProceed() {
2220 Log.d(Config.LOGTAG, "retract from proceed");
2221 this.sendJingleMessage("retract");
2222 closeTransitionLogFinish(State.RETRACTED_RACED);
2223 }
2224
2225 private void closeTransitionLogFinish(final State state) {
2226 this.webRTCWrapper.close();
2227 transitionOrThrow(state);
2228 writeLogMessage(state);
2229 finish();
2230 }
2231
2232 private void setupWebRTC(
2233 final Set<Media> media,
2234 final List<PeerConnection.IceServer> iceServers,
2235 final boolean trickle)
2236 throws WebRTCWrapper.InitializationException {
2237 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2238 this.webRTCWrapper.setup(
2239 this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2240 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2241 }
2242
2243 private void acceptCallFromProposed() {
2244 transitionOrThrow(State.PROCEED);
2245 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2246 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2247 this.sendJingleMessage("proceed");
2248 }
2249
2250 private void rejectCallFromProposed() {
2251 transitionOrThrow(State.REJECTED);
2252 writeLogMessageMissed();
2253 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2254 this.sendJingleMessage("reject");
2255 finish();
2256 }
2257
2258 private void rejectCallFromProceed() {
2259 this.sendJingleMessage("reject");
2260 closeTransitionLogFinish(State.REJECTED_RACED);
2261 }
2262
2263 private void rejectCallFromSessionInitiate() {
2264 webRTCWrapper.close();
2265 sendSessionTerminate(Reason.DECLINE);
2266 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2267 }
2268
2269 private void sendJingleMessage(final String action) {
2270 sendJingleMessage(action, id.with);
2271 }
2272
2273 private void sendJingleMessage(final String action, final Jid to) {
2274 final MessagePacket messagePacket = new MessagePacket();
2275 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2276 messagePacket.setTo(to);
2277 final Element intent =
2278 messagePacket
2279 .addChild(action, Namespace.JINGLE_MESSAGE)
2280 .setAttribute("id", id.sessionId);
2281 if ("proceed".equals(action)) {
2282 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2283 if (isOmemoEnabled()) {
2284 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2285 final Element device =
2286 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2287 device.setAttribute("id", deviceId);
2288 }
2289 }
2290 messagePacket.addChild("store", "urn:xmpp:hints");
2291 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2292 }
2293
2294 private boolean isOmemoEnabled() {
2295 final Conversational conversational = message.getConversation();
2296 if (conversational instanceof Conversation) {
2297 return ((Conversation) conversational).getNextEncryption()
2298 == Message.ENCRYPTION_AXOLOTL;
2299 }
2300 return false;
2301 }
2302
2303 private void acceptCallFromSessionInitialized() {
2304 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2305 sendSessionAccept();
2306 }
2307
2308
2309 @Override
2310 protected synchronized boolean transition(final State target, final Runnable runnable) {
2311 if (super.transition(target, runnable)) {
2312 updateEndUserState();
2313 updateOngoingCallNotification();
2314 return true;
2315 } else {
2316 return false;
2317 }
2318 }
2319
2320 @Override
2321 public void onIceCandidate(final IceCandidate iceCandidate) {
2322 final RtpContentMap rtpContentMap =
2323 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2324 final IceUdpTransportInfo.Credentials credentials;
2325 try {
2326 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2327 } catch (final IllegalArgumentException e) {
2328 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2329 return;
2330 }
2331 final String uFrag = credentials.ufrag;
2332 final IceUdpTransportInfo.Candidate candidate =
2333 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2334 if (candidate == null) {
2335 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2336 return;
2337 }
2338 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2339 sendTransportInfo(iceCandidate.sdpMid, candidate);
2340 }
2341
2342 @Override
2343 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2344 Log.d(
2345 Config.LOGTAG,
2346 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2347 this.stateHistory.add(newState);
2348 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2349 this.sessionDuration.start();
2350 updateOngoingCallNotification();
2351 } else if (this.sessionDuration.isRunning()) {
2352 this.sessionDuration.stop();
2353 updateOngoingCallNotification();
2354 }
2355
2356 final boolean neverConnected =
2357 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2358
2359 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2360 if (neverConnected) {
2361 if (isTerminated()) {
2362 Log.d(
2363 Config.LOGTAG,
2364 id.account.getJid().asBareJid()
2365 + ": not sending session-terminate after connectivity error because session is already in state "
2366 + this.state);
2367 return;
2368 }
2369 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2370 return;
2371 } else {
2372 this.restartIce();
2373 }
2374 }
2375 updateEndUserState();
2376 }
2377
2378 private void restartIce() {
2379 this.stateHistory.clear();
2380 this.webRTCWrapper.restartIceAsync();
2381 }
2382
2383 @Override
2384 public void onRenegotiationNeeded() {
2385 this.webRTCWrapper.execute(this::renegotiate);
2386 }
2387
2388 private void renegotiate() {
2389 final SessionDescription sessionDescription;
2390 try {
2391 sessionDescription = setLocalSessionDescription();
2392 } catch (final Exception e) {
2393 final Throwable cause = Throwables.getRootCause(e);
2394 webRTCWrapper.close();
2395 if (isTerminated()) {
2396 Log.d(
2397 Config.LOGTAG,
2398 "failed to renegotiate. session was already terminated",
2399 cause);
2400 return;
2401 }
2402 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2403 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2404 return;
2405 }
2406 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2407 final RtpContentMap currentContentMap = getLocalContentMap();
2408 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2409 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2410
2411 Log.d(
2412 Config.LOGTAG,
2413 id.getAccount().getJid().asBareJid()
2414 + ": renegotiate. iceRestart="
2415 + iceRestart
2416 + " content id diff="
2417 + diff);
2418
2419 if (diff.hasModifications() && iceRestart) {
2420 webRTCWrapper.close();
2421 sendSessionTerminate(
2422 Reason.FAILED_APPLICATION,
2423 "WebRTC unexpectedly tried to modify content and transport at once");
2424 return;
2425 }
2426
2427 if (iceRestart) {
2428 initiateIceRestart(rtpContentMap);
2429 return;
2430 } else if (diff.isEmpty()) {
2431 Log.d(
2432 Config.LOGTAG,
2433 "renegotiation. nothing to do. SignalingState="
2434 + this.webRTCWrapper.getSignalingState());
2435 }
2436
2437 if (diff.added.size() > 0) {
2438 modifyLocalContentMap(rtpContentMap);
2439 sendContentAdd(rtpContentMap, diff.added);
2440 }
2441 }
2442
2443 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2444 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2445 final JinglePacket jinglePacket =
2446 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2447 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2448 jinglePacket.setTo(id.with);
2449 xmppConnectionService.sendIqPacket(
2450 id.account,
2451 jinglePacket,
2452 (account, response) -> {
2453 if (response.getType() == IqPacket.TYPE.RESULT) {
2454 Log.d(Config.LOGTAG, "received success to our ice restart");
2455 setLocalContentMap(rtpContentMap);
2456 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2457 return;
2458 }
2459 if (response.getType() == IqPacket.TYPE.ERROR) {
2460 if (isTieBreak(response)) {
2461 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2462 return;
2463 }
2464 handleIqErrorResponse(response);
2465 }
2466 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2467 handleIqTimeoutResponse(response);
2468 }
2469 });
2470 }
2471
2472 private boolean isTieBreak(final IqPacket response) {
2473 final Element error = response.findChild("error");
2474 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2475 }
2476
2477 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2478 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2479 this.outgoingContentAdd = contentAdd;
2480 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2481 prepareOutgoingContentMap(contentAdd);
2482 Futures.addCallback(
2483 outgoingContentMapFuture,
2484 new FutureCallback<>() {
2485 @Override
2486 public void onSuccess(final RtpContentMap outgoingContentMap) {
2487 sendContentAdd(outgoingContentMap);
2488 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2489 }
2490
2491 @Override
2492 public void onFailure(@NonNull Throwable throwable) {
2493 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2494 }
2495 },
2496 MoreExecutors.directExecutor());
2497 }
2498
2499 private void sendContentAdd(final RtpContentMap contentAdd) {
2500
2501 final JinglePacket jinglePacket =
2502 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2503 jinglePacket.setTo(id.with);
2504 xmppConnectionService.sendIqPacket(
2505 id.account,
2506 jinglePacket,
2507 (connection, response) -> {
2508 if (response.getType() == IqPacket.TYPE.RESULT) {
2509 Log.d(
2510 Config.LOGTAG,
2511 id.getAccount().getJid().asBareJid()
2512 + ": received ACK to our content-add");
2513 return;
2514 }
2515 if (response.getType() == IqPacket.TYPE.ERROR) {
2516 if (isTieBreak(response)) {
2517 this.outgoingContentAdd = null;
2518 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2519 return;
2520 }
2521 handleIqErrorResponse(response);
2522 }
2523 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2524 handleIqTimeoutResponse(response);
2525 }
2526 });
2527 }
2528
2529 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2530 if (isInitiator()) {
2531 this.initiatorRtpContentMap = rtpContentMap;
2532 } else {
2533 this.responderRtpContentMap = rtpContentMap;
2534 }
2535 }
2536
2537 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2538 if (isInitiator()) {
2539 this.responderRtpContentMap = rtpContentMap;
2540 } else {
2541 this.initiatorRtpContentMap = rtpContentMap;
2542 }
2543 }
2544
2545 // this method is to be used for content map modifications that modify media
2546 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2547 final RtpContentMap activeContents = rtpContentMap.activeContents();
2548 setLocalContentMap(activeContents);
2549 this.webRTCWrapper.switchSpeakerPhonePreference(
2550 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2551 updateEndUserState();
2552 }
2553
2554 private SessionDescription setLocalSessionDescription()
2555 throws ExecutionException, InterruptedException {
2556 final org.webrtc.SessionDescription sessionDescription =
2557 this.webRTCWrapper.setLocalDescription(false).get();
2558 return SessionDescription.parse(sessionDescription.description);
2559 }
2560
2561 private void closeWebRTCSessionAfterFailedConnection() {
2562 this.webRTCWrapper.close();
2563 synchronized (this) {
2564 if (isTerminated()) {
2565 Log.d(
2566 Config.LOGTAG,
2567 id.account.getJid().asBareJid()
2568 + ": no need to send session-terminate after failed connection. Other party already did");
2569 return;
2570 }
2571 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2572 }
2573 }
2574
2575 public boolean zeroDuration() {
2576 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2577 }
2578
2579 public long getCallDuration() {
2580 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2581 }
2582
2583 public AppRTCAudioManager getAudioManager() {
2584 return webRTCWrapper.getAudioManager();
2585 }
2586
2587 public boolean isMicrophoneEnabled() {
2588 return webRTCWrapper.isMicrophoneEnabled();
2589 }
2590
2591 public boolean setMicrophoneEnabled(final boolean enabled) {
2592 return webRTCWrapper.setMicrophoneEnabled(enabled);
2593 }
2594
2595 public boolean isVideoEnabled() {
2596 return webRTCWrapper.isVideoEnabled();
2597 }
2598
2599 public void setVideoEnabled(final boolean enabled) {
2600 webRTCWrapper.setVideoEnabled(enabled);
2601 }
2602
2603 public boolean isCameraSwitchable() {
2604 return webRTCWrapper.isCameraSwitchable();
2605 }
2606
2607 public boolean isFrontCamera() {
2608 return webRTCWrapper.isFrontCamera();
2609 }
2610
2611 public ListenableFuture<Boolean> switchCamera() {
2612 return webRTCWrapper.switchCamera();
2613 }
2614
2615 @Override
2616 public void onAudioDeviceChanged(
2617 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2618 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2619 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2620 selectedAudioDevice, availableAudioDevices);
2621 }
2622
2623 private void updateEndUserState() {
2624 final RtpEndUserState endUserState = getEndUserState();
2625 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2626 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2627 id.account, id.with, id.sessionId, endUserState);
2628 }
2629
2630 private void updateOngoingCallNotification() {
2631 final State state = this.state;
2632 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2633 final boolean reconnecting;
2634 if (state == State.SESSION_ACCEPTED) {
2635 reconnecting =
2636 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2637 } else {
2638 reconnecting = false;
2639 }
2640 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2641 } else {
2642 xmppConnectionService.removeOngoingCall();
2643 }
2644 }
2645
2646 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2647 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2648 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2649 request.setTo(id.account.getDomain());
2650 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2651 xmppConnectionService.sendIqPacket(
2652 id.account,
2653 request,
2654 (account, response) -> {
2655 final var iceServers = IceServers.parse(response);
2656 if (iceServers.size() == 0) {
2657 Log.w(
2658 Config.LOGTAG,
2659 id.account.getJid().asBareJid()
2660 + ": no ICE server found "
2661 + response);
2662 }
2663 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2664 });
2665 } else {
2666 Log.w(
2667 Config.LOGTAG,
2668 id.account.getJid().asBareJid() + ": has no external service discovery");
2669 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2670 }
2671 }
2672
2673 @Override
2674 protected void terminateTransport() {
2675 this.webRTCWrapper.close();
2676 }
2677
2678 @Override
2679 protected void finish() {
2680 if (isTerminated()) {
2681 this.cancelRingingTimeout();
2682 this.webRTCWrapper.verifyClosed();
2683 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2684 super.finish();
2685 try {
2686 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2687 log.getParentFile().mkdirs();
2688 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2689 } catch (final IOException e) { }
2690 } else {
2691 throw new IllegalStateException(
2692 String.format("Unable to call finish from %s", this.state));
2693 }
2694 }
2695
2696 private void writeLogMessage(final State state) {
2697 final long duration = getCallDuration();
2698 if (state == State.TERMINATED_SUCCESS
2699 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2700 writeLogMessageSuccess(duration);
2701 } else {
2702 writeLogMessageMissed();
2703 }
2704 }
2705
2706 private void writeLogMessageSuccess(final long duration) {
2707 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2708 this.writeMessage();
2709 }
2710
2711 private void writeLogMessageMissed() {
2712 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2713 this.writeMessage();
2714 }
2715
2716 private void writeMessage() {
2717 final Conversational conversational = message.getConversation();
2718 if (conversational instanceof Conversation) {
2719 ((Conversation) conversational).add(this.message);
2720 xmppConnectionService.createMessageAsync(message);
2721 xmppConnectionService.updateConversationUi();
2722 } else {
2723 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2724 }
2725 }
2726
2727 public Optional<VideoTrack> getLocalVideoTrack() {
2728 return webRTCWrapper.getLocalVideoTrack();
2729 }
2730
2731 public Optional<VideoTrack> getRemoteVideoTrack() {
2732 return webRTCWrapper.getRemoteVideoTrack();
2733 }
2734
2735 public EglBase.Context getEglBaseContext() {
2736 return webRTCWrapper.getEglBaseContext();
2737 }
2738
2739 void setProposedMedia(final Set<Media> media) {
2740 this.proposedMedia = media;
2741 }
2742
2743 public void fireStateUpdate() {
2744 final RtpEndUserState endUserState = getEndUserState();
2745 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2746 id.account, id.with, id.sessionId, endUserState);
2747 }
2748
2749 public boolean isSwitchToVideoAvailable() {
2750 final boolean prerequisite =
2751 Media.audioOnly(getMedia())
2752 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2753 .contains(getEndUserState());
2754 return prerequisite && remoteHasVideoFeature();
2755 }
2756
2757 private boolean remoteHasVideoFeature() {
2758 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
2759 }
2760
2761 private boolean remoteHasSdpOfferAnswer() {
2762 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
2763 }
2764
2765 private interface OnIceServersDiscovered {
2766 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2767 }
2768}