1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4import android.os.Environment;
5
6import androidx.annotation.NonNull;
7import androidx.annotation.Nullable;
8
9import com.google.common.base.Joiner;
10import com.google.common.base.Optional;
11import com.google.common.base.Preconditions;
12import com.google.common.base.Stopwatch;
13import com.google.common.base.Strings;
14import com.google.common.base.Throwables;
15import com.google.common.collect.Collections2;
16import com.google.common.collect.ImmutableList;
17import com.google.common.collect.ImmutableMap;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Sets;
23import com.google.common.primitives.Ints;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import eu.siacs.conversations.BuildConfig;
30import eu.siacs.conversations.Config;
31import eu.siacs.conversations.crypto.axolotl.AxolotlService;
32import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
33import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
34import eu.siacs.conversations.entities.Account;
35import eu.siacs.conversations.entities.Contact;
36import eu.siacs.conversations.entities.Conversation;
37import eu.siacs.conversations.entities.Conversational;
38import eu.siacs.conversations.entities.Message;
39import eu.siacs.conversations.entities.Presence;
40import eu.siacs.conversations.entities.RtpSessionStatus;
41import eu.siacs.conversations.entities.ServiceDiscoveryResult;
42import eu.siacs.conversations.services.AppRTCAudioManager;
43import eu.siacs.conversations.utils.IP;
44import eu.siacs.conversations.xml.Element;
45import eu.siacs.conversations.xml.Namespace;
46import eu.siacs.conversations.xmpp.Jid;
47import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
48import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
49import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
50import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
51import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
52import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
53import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
54import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
55import eu.siacs.conversations.xmpp.stanzas.IqPacket;
56import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
57
58import org.webrtc.DtmfSender;
59import org.webrtc.EglBase;
60import org.webrtc.IceCandidate;
61import org.webrtc.PeerConnection;
62import org.webrtc.VideoTrack;
63
64import java.io.File;
65import java.io.IOException;
66import java.util.Arrays;
67import java.util.Collection;
68import java.util.Collections;
69import java.util.LinkedList;
70import java.util.List;
71import java.util.Map;
72import java.util.Queue;
73import java.util.Set;
74import java.util.concurrent.ExecutionException;
75import java.util.concurrent.ScheduledFuture;
76import java.util.concurrent.TimeUnit;
77
78public class JingleRtpConnection extends AbstractJingleConnection
79 implements WebRTCWrapper.EventCallback {
80
81 public static final List<State> STATES_SHOWING_ONGOING_CALL =
82 Arrays.asList(
83 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
84 private static final long BUSY_TIME_OUT = 30;
85 private static final List<State> TERMINATED =
86 Arrays.asList(
87 State.ACCEPTED,
88 State.REJECTED,
89 State.REJECTED_RACED,
90 State.RETRACTED,
91 State.RETRACTED_RACED,
92 State.TERMINATED_SUCCESS,
93 State.TERMINATED_DECLINED_OR_BUSY,
94 State.TERMINATED_CONNECTIVITY_ERROR,
95 State.TERMINATED_CANCEL_OR_TIMEOUT,
96 State.TERMINATED_APPLICATION_FAILURE,
97 State.TERMINATED_SECURITY_ERROR);
98
99 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
100
101 static {
102 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
103 new ImmutableMap.Builder<>();
104 transitionBuilder.put(
105 State.NULL,
106 ImmutableList.of(
107 State.PROPOSED,
108 State.SESSION_INITIALIZED,
109 State.TERMINATED_APPLICATION_FAILURE,
110 State.TERMINATED_SECURITY_ERROR));
111 transitionBuilder.put(
112 State.PROPOSED,
113 ImmutableList.of(
114 State.ACCEPTED,
115 State.PROCEED,
116 State.REJECTED,
117 State.RETRACTED,
118 State.TERMINATED_APPLICATION_FAILURE,
119 State.TERMINATED_SECURITY_ERROR,
120 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
121 // rebinds
122 ));
123 transitionBuilder.put(
124 State.PROCEED,
125 ImmutableList.of(
126 State.REJECTED_RACED,
127 State.RETRACTED_RACED,
128 State.SESSION_INITIALIZED_PRE_APPROVED,
129 State.TERMINATED_SUCCESS,
130 State.TERMINATED_APPLICATION_FAILURE,
131 State.TERMINATED_SECURITY_ERROR,
132 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
133 // bounces of the proceed message
134 ));
135 transitionBuilder.put(
136 State.SESSION_INITIALIZED,
137 ImmutableList.of(
138 State.SESSION_ACCEPTED,
139 State.TERMINATED_SUCCESS,
140 State.TERMINATED_DECLINED_OR_BUSY,
141 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
142 // and IQ timeouts
143 State.TERMINATED_CANCEL_OR_TIMEOUT,
144 State.TERMINATED_APPLICATION_FAILURE,
145 State.TERMINATED_SECURITY_ERROR));
146 transitionBuilder.put(
147 State.SESSION_INITIALIZED_PRE_APPROVED,
148 ImmutableList.of(
149 State.SESSION_ACCEPTED,
150 State.TERMINATED_SUCCESS,
151 State.TERMINATED_DECLINED_OR_BUSY,
152 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
153 // and IQ timeouts
154 State.TERMINATED_CANCEL_OR_TIMEOUT,
155 State.TERMINATED_APPLICATION_FAILURE,
156 State.TERMINATED_SECURITY_ERROR));
157 transitionBuilder.put(
158 State.SESSION_ACCEPTED,
159 ImmutableList.of(
160 State.TERMINATED_SUCCESS,
161 State.TERMINATED_DECLINED_OR_BUSY,
162 State.TERMINATED_CONNECTIVITY_ERROR,
163 State.TERMINATED_CANCEL_OR_TIMEOUT,
164 State.TERMINATED_APPLICATION_FAILURE,
165 State.TERMINATED_SECURITY_ERROR));
166 VALID_TRANSITIONS = transitionBuilder.build();
167 }
168
169 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
170 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
171 pendingIceCandidates = new LinkedList<>();
172 private final OmemoVerification omemoVerification = new OmemoVerification();
173 private final Message message;
174 private State state = State.NULL;
175 private Set<Media> proposedMedia;
176 private RtpContentMap initiatorRtpContentMap;
177 private RtpContentMap responderRtpContentMap;
178 private RtpContentMap incomingContentAdd;
179 private RtpContentMap outgoingContentAdd;
180 private IceUdpTransportInfo.Setup peerDtlsSetup;
181 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
182 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
183 private ScheduledFuture<?> ringingTimeoutFuture;
184 private final long created = System.currentTimeMillis() / 1000L;
185
186 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
187 super(jingleConnectionManager, id, initiator);
188 final Conversation conversation =
189 jingleConnectionManager
190 .getXmppConnectionService()
191 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
192 this.message =
193 new Message(
194 conversation,
195 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
196 Message.TYPE_RTP_SESSION,
197 id.sessionId);
198 }
199
200 private static State reasonToState(Reason reason) {
201 return switch (reason) {
202 case SUCCESS -> State.TERMINATED_SUCCESS;
203 case DECLINE, BUSY -> State.TERMINATED_DECLINED_OR_BUSY;
204 case CANCEL, TIMEOUT -> State.TERMINATED_CANCEL_OR_TIMEOUT;
205 case SECURITY_ERROR -> State.TERMINATED_SECURITY_ERROR;
206 case FAILED_APPLICATION, UNSUPPORTED_TRANSPORTS, UNSUPPORTED_APPLICATIONS -> State
207 .TERMINATED_APPLICATION_FAILURE;
208 default -> State.TERMINATED_CONNECTIVITY_ERROR;
209 };
210 }
211
212 @Override
213 synchronized void deliverPacket(final JinglePacket jinglePacket) {
214 switch (jinglePacket.getAction()) {
215 case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket);
216 case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket);
217 case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket);
218 case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket);
219 case CONTENT_ADD -> receiveContentAdd(jinglePacket);
220 case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket);
221 case CONTENT_REJECT -> receiveContentReject(jinglePacket);
222 case CONTENT_REMOVE -> receiveContentRemove(jinglePacket);
223 case CONTENT_MODIFY -> receiveContentModify(jinglePacket);
224 default -> {
225 respondOk(jinglePacket);
226 Log.d(
227 Config.LOGTAG,
228 String.format(
229 "%s: received unhandled jingle action %s",
230 id.account.getJid().asBareJid(), jinglePacket.getAction()));
231 }
232 }
233 }
234
235 @Override
236 synchronized void notifyRebound() {
237 if (isTerminated()) {
238 return;
239 }
240 webRTCWrapper.close();
241 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
242 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
243 }
244 if (isInState(
245 State.SESSION_INITIALIZED,
246 State.SESSION_INITIALIZED_PRE_APPROVED,
247 State.SESSION_ACCEPTED)) {
248 // we might have already changed resources (full jid) at this point; so this might not
249 // even reach the other party
250 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
251 } else {
252 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
253 finish();
254 }
255 }
256
257 public boolean applyDtmfTone(String tone) {
258 return webRTCWrapper.applyDtmfTone(tone);
259 }
260
261 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
262 respondOk(jinglePacket);
263 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
264 final State previous = this.state;
265 Log.d(
266 Config.LOGTAG,
267 id.account.getJid().asBareJid()
268 + ": received session terminate reason="
269 + wrapper.reason
270 + "("
271 + Strings.nullToEmpty(wrapper.text)
272 + ") while in state "
273 + previous);
274 if (TERMINATED.contains(previous)) {
275 Log.d(
276 Config.LOGTAG,
277 id.account.getJid().asBareJid()
278 + ": ignoring session terminate because already in "
279 + previous);
280 return;
281 }
282 webRTCWrapper.close();
283 final State target = reasonToState(wrapper.reason);
284 transitionOrThrow(target);
285 writeLogMessage(target);
286 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
287 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
288 }
289 finish();
290 }
291
292 private void receiveTransportInfo(final JinglePacket jinglePacket) {
293 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
294 // INITIALIZED only after transport-info has been received
295 if (isInState(
296 State.NULL,
297 State.PROCEED,
298 State.SESSION_INITIALIZED,
299 State.SESSION_INITIALIZED_PRE_APPROVED,
300 State.SESSION_ACCEPTED)) {
301 final RtpContentMap contentMap;
302 try {
303 contentMap = RtpContentMap.of(jinglePacket);
304 } catch (final IllegalArgumentException | NullPointerException e) {
305 Log.d(
306 Config.LOGTAG,
307 id.account.getJid().asBareJid()
308 + ": improperly formatted contents; ignoring",
309 e);
310 respondOk(jinglePacket);
311 return;
312 }
313 receiveTransportInfo(jinglePacket, contentMap);
314 } else {
315 if (isTerminated()) {
316 respondOk(jinglePacket);
317 Log.d(
318 Config.LOGTAG,
319 id.account.getJid().asBareJid()
320 + ": ignoring out-of-order transport info; we where already terminated");
321 } else {
322 Log.d(
323 Config.LOGTAG,
324 id.account.getJid().asBareJid()
325 + ": received transport info while in state="
326 + this.state);
327 terminateWithOutOfOrder(jinglePacket);
328 }
329 }
330 }
331
332 private void receiveTransportInfo(
333 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
334 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
335 contentMap.contents.entrySet();
336 final RtpContentMap remote = getRemoteContentMap();
337 final Set<String> remoteContentIds =
338 remote == null ? Collections.emptySet() : remote.contents.keySet();
339 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
340 Log.d(
341 Config.LOGTAG,
342 "received transport-info for unknown contents "
343 + contentMap.contents.keySet()
344 + " (known: "
345 + remoteContentIds
346 + ")");
347 respondOk(jinglePacket);
348 pendingIceCandidates.addAll(candidates);
349 return;
350 }
351 if (this.state != State.SESSION_ACCEPTED) {
352 Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog");
353 respondOk(jinglePacket);
354 pendingIceCandidates.addAll(candidates);
355 return;
356 }
357 // zero candidates + modified credentials are an ICE restart offer
358 if (checkForIceRestart(jinglePacket, contentMap)) {
359 return;
360 }
361 respondOk(jinglePacket);
362 try {
363 processCandidates(candidates);
364 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
365 Log.w(
366 Config.LOGTAG,
367 id.account.getJid().asBareJid()
368 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
369 }
370 }
371
372 private void receiveContentAdd(final JinglePacket jinglePacket) {
373 final RtpContentMap modification;
374 try {
375 modification = RtpContentMap.of(jinglePacket);
376 modification.requireContentDescriptions();
377 } catch (final RuntimeException e) {
378 Log.d(
379 Config.LOGTAG,
380 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
381 Throwables.getRootCause(e));
382 respondOk(jinglePacket);
383 webRTCWrapper.close();
384 sendSessionTerminate(Reason.of(e), e.getMessage());
385 return;
386 }
387 if (isInState(State.SESSION_ACCEPTED)) {
388 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
389 final ListenableFuture<RtpContentMap> future =
390 receiveRtpContentMap(
391 modification,
392 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
393 Futures.addCallback(
394 future,
395 new FutureCallback<>() {
396 @Override
397 public void onSuccess(final RtpContentMap rtpContentMap) {
398 receiveContentAdd(jinglePacket, rtpContentMap);
399 }
400
401 @Override
402 public void onFailure(@NonNull Throwable throwable) {
403 respondOk(jinglePacket);
404 final Throwable rootCause = Throwables.getRootCause(throwable);
405 Log.d(
406 Config.LOGTAG,
407 id.account.getJid().asBareJid()
408 + ": improperly formatted contents in content-add",
409 throwable);
410 webRTCWrapper.close();
411 sendSessionTerminate(
412 Reason.ofThrowable(rootCause), rootCause.getMessage());
413 }
414 },
415 MoreExecutors.directExecutor());
416 } else {
417 terminateWithOutOfOrder(jinglePacket);
418 }
419 }
420
421 private void receiveContentAdd(
422 final JinglePacket jinglePacket, final RtpContentMap modification) {
423 final RtpContentMap remote = getRemoteContentMap();
424 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
425 respondOk(jinglePacket);
426 this.webRTCWrapper.close();
427 sendSessionTerminate(
428 Reason.FAILED_APPLICATION,
429 String.format(
430 "contents with names %s already exists",
431 Joiner.on(", ").join(modification.getNames())));
432 return;
433 }
434 final ContentAddition contentAddition =
435 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
436
437 final RtpContentMap outgoing = this.outgoingContentAdd;
438 final Set<ContentAddition.Summary> outgoingContentAddSummary =
439 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
440
441 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
442 if (isInitiator()) {
443 Log.d(
444 Config.LOGTAG,
445 id.getAccount().getJid().asBareJid()
446 + ": respond with tie break to matching content-add offer");
447 respondWithTieBreak(jinglePacket);
448 } else {
449 Log.d(
450 Config.LOGTAG,
451 id.getAccount().getJid().asBareJid()
452 + ": automatically accept matching content-add offer");
453 acceptContentAdd(contentAddition.summary, modification);
454 }
455 return;
456 }
457
458 // once we can display multiple video tracks we can be more loose with this condition
459 // theoretically it should also be fine to automatically accept audio only contents
460 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
461 Log.d(
462 Config.LOGTAG,
463 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
464 this.incomingContentAdd = modification;
465 respondOk(jinglePacket);
466 updateEndUserState();
467 } else {
468 respondOk(jinglePacket);
469 // TODO do we want to add a reason?
470 rejectContentAdd(modification);
471 }
472 }
473
474 private void receiveContentAccept(final JinglePacket jinglePacket) {
475 final RtpContentMap receivedContentAccept;
476 try {
477 receivedContentAccept = RtpContentMap.of(jinglePacket);
478 receivedContentAccept.requireContentDescriptions();
479 } catch (final RuntimeException e) {
480 Log.d(
481 Config.LOGTAG,
482 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
483 Throwables.getRootCause(e));
484 respondOk(jinglePacket);
485 webRTCWrapper.close();
486 sendSessionTerminate(Reason.of(e), e.getMessage());
487 return;
488 }
489
490 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
491 if (outgoingContentAdd == null) {
492 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
493 terminateWithOutOfOrder(jinglePacket);
494 return;
495 }
496 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
497 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
498 this.outgoingContentAdd = null;
499 respondOk(jinglePacket);
500 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
501 final ListenableFuture<RtpContentMap> future =
502 receiveRtpContentMap(
503 receivedContentAccept,
504 this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
505 Futures.addCallback(
506 future,
507 new FutureCallback<>() {
508 @Override
509 public void onSuccess(final RtpContentMap result) {
510 receiveContentAccept(result);
511 }
512
513 @Override
514 public void onFailure(@NonNull final Throwable throwable) {
515 webRTCWrapper.close();
516 sendSessionTerminate(
517 Reason.ofThrowable(throwable), throwable.getMessage());
518 }
519 },
520 MoreExecutors.directExecutor());
521 } else {
522 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
523 terminateWithOutOfOrder(jinglePacket);
524 }
525 }
526
527 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
528 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
529 final RtpContentMap modifiedContentMap =
530 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
531
532 setRemoteContentMap(modifiedContentMap);
533
534 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
535
536 final org.webrtc.SessionDescription sdp =
537 new org.webrtc.SessionDescription(
538 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
539
540 try {
541 this.webRTCWrapper.setRemoteDescription(sdp).get();
542 } catch (final Exception e) {
543 final Throwable cause = Throwables.getRootCause(e);
544 Log.d(
545 Config.LOGTAG,
546 id.getAccount().getJid().asBareJid()
547 + ": unable to set remote description after receiving content-accept",
548 cause);
549 webRTCWrapper.close();
550 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
551 return;
552 }
553 Log.d(
554 Config.LOGTAG,
555 id.getAccount().getJid().asBareJid()
556 + ": remote has accepted content-add "
557 + ContentAddition.summary(receivedContentAccept));
558 processCandidates(receivedContentAccept.contents.entrySet());
559 updateEndUserState();
560 }
561
562 private void receiveContentModify(final JinglePacket jinglePacket) {
563 if (this.state != State.SESSION_ACCEPTED) {
564 terminateWithOutOfOrder(jinglePacket);
565 return;
566 }
567 final Map<String, Content.Senders> modification =
568 Maps.transformEntries(
569 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
570 final boolean isInitiator = isInitiator();
571 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
572 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
573 final Set<String> currentOutgoingMediaIds =
574 currentOutgoing == null
575 ? Collections.emptySet()
576 : currentOutgoing.contents.keySet();
577 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
578 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
579 respondOk(jinglePacket);
580 final RtpContentMap modifiedContentMap;
581 try {
582 modifiedContentMap =
583 currentOutgoing.modifiedSendersChecked(isInitiator, modification);
584 } catch (final IllegalArgumentException e) {
585 webRTCWrapper.close();
586 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
587 return;
588 }
589 this.outgoingContentAdd = modifiedContentMap;
590 Log.d(
591 Config.LOGTAG,
592 id.account.getJid().asBareJid()
593 + ": processed content-modification for pending content-add");
594 } else if (remoteContentMap != null
595 && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
596 respondOk(jinglePacket);
597 final RtpContentMap modifiedRemoteContentMap;
598 try {
599 modifiedRemoteContentMap =
600 remoteContentMap.modifiedSendersChecked(isInitiator, modification);
601 } catch (final IllegalArgumentException e) {
602 webRTCWrapper.close();
603 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
604 return;
605 }
606 final SessionDescription offer;
607 try {
608 offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator());
609 } catch (final IllegalArgumentException | NullPointerException e) {
610 Log.d(
611 Config.LOGTAG,
612 id.getAccount().getJid().asBareJid()
613 + ": unable convert offer from content-modify to SDP",
614 e);
615 webRTCWrapper.close();
616 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
617 return;
618 }
619 Log.d(
620 Config.LOGTAG,
621 id.account.getJid().asBareJid() + ": auto accepting content-modification");
622 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
623 } else {
624 Log.d(Config.LOGTAG, "received unsupported content modification " + modification);
625 respondWithItemNotFound(jinglePacket);
626 }
627 }
628
629 private void autoAcceptContentModify(
630 final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
631 this.setRemoteContentMap(modifiedRemoteContentMap);
632 final org.webrtc.SessionDescription sdp =
633 new org.webrtc.SessionDescription(
634 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
635 try {
636 this.webRTCWrapper.setRemoteDescription(sdp).get();
637 // auto accept is only done when we already have tracks
638 final SessionDescription answer = setLocalSessionDescription();
639 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
640 modifyLocalContentMap(rtpContentMap);
641 // we do not need to send an answer but do we have to resend the candidates currently in
642 // SDP?
643 // resendCandidatesFromSdp(answer);
644 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
645 } catch (final Exception e) {
646 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
647 webRTCWrapper.close();
648 sendSessionTerminate(Reason.FAILED_APPLICATION);
649 }
650 }
651
652 private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(
653 final SessionDescription answer) {
654 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder =
655 new ImmutableMultimap.Builder<>();
656 for (final SessionDescription.Media media : answer.media) {
657 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
658 if (Strings.isNullOrEmpty(mid)) {
659 continue;
660 }
661 for (final String sdpCandidate : media.attributes.get("candidate")) {
662 final IceUdpTransportInfo.Candidate candidate =
663 IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
664 if (candidate != null) {
665 candidateBuilder.put(mid, candidate);
666 }
667 }
668 }
669 return candidateBuilder.build();
670 }
671
672 private void receiveContentReject(final JinglePacket jinglePacket) {
673 final RtpContentMap receivedContentReject;
674 try {
675 receivedContentReject = RtpContentMap.of(jinglePacket);
676 } catch (final RuntimeException e) {
677 Log.d(
678 Config.LOGTAG,
679 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
680 Throwables.getRootCause(e));
681 respondOk(jinglePacket);
682 this.webRTCWrapper.close();
683 sendSessionTerminate(Reason.of(e), e.getMessage());
684 return;
685 }
686
687 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
688 if (outgoingContentAdd == null) {
689 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
690 terminateWithOutOfOrder(jinglePacket);
691 return;
692 }
693 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
694 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
695 this.outgoingContentAdd = null;
696 respondOk(jinglePacket);
697 Log.d(Config.LOGTAG, jinglePacket.toString());
698 receiveContentReject(ourSummary);
699 } else {
700 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
701 terminateWithOutOfOrder(jinglePacket);
702 }
703 }
704
705 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
706 try {
707 this.webRTCWrapper.removeTrack(Media.VIDEO);
708 final RtpContentMap localContentMap = customRollback();
709 modifyLocalContentMap(localContentMap);
710 } catch (final Exception e) {
711 final Throwable cause = Throwables.getRootCause(e);
712 Log.d(
713 Config.LOGTAG,
714 id.getAccount().getJid().asBareJid()
715 + ": unable to rollback local description after receiving content-reject",
716 cause);
717 webRTCWrapper.close();
718 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
719 return;
720 }
721 Log.d(
722 Config.LOGTAG,
723 id.getAccount().getJid().asBareJid()
724 + ": remote has rejected our content-add "
725 + summary);
726 }
727
728 private void receiveContentRemove(final JinglePacket jinglePacket) {
729 final RtpContentMap receivedContentRemove;
730 try {
731 receivedContentRemove = RtpContentMap.of(jinglePacket);
732 receivedContentRemove.requireContentDescriptions();
733 } catch (final RuntimeException e) {
734 Log.d(
735 Config.LOGTAG,
736 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
737 Throwables.getRootCause(e));
738 respondOk(jinglePacket);
739 this.webRTCWrapper.close();
740 sendSessionTerminate(Reason.of(e), e.getMessage());
741 return;
742 }
743 respondOk(jinglePacket);
744 receiveContentRemove(receivedContentRemove);
745 }
746
747 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
748 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
749 final Set<ContentAddition.Summary> contentAddSummary =
750 incomingContentAdd == null
751 ? Collections.emptySet()
752 : ContentAddition.summary(incomingContentAdd);
753 final Set<ContentAddition.Summary> removeSummary =
754 ContentAddition.summary(receivedContentRemove);
755 if (contentAddSummary.equals(removeSummary)) {
756 this.incomingContentAdd = null;
757 updateEndUserState();
758 } else {
759 webRTCWrapper.close();
760 sendSessionTerminate(
761 Reason.FAILED_APPLICATION,
762 String.format(
763 "%s only supports %s as a means to retract a not yet accepted %s",
764 BuildConfig.APP_NAME,
765 JinglePacket.Action.CONTENT_REMOVE,
766 JinglePacket.Action.CONTENT_ADD));
767 }
768 }
769
770 public synchronized void retractContentAdd() {
771 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
772 if (outgoingContentAdd == null) {
773 throw new IllegalStateException("Not outgoing content add");
774 }
775 try {
776 webRTCWrapper.removeTrack(Media.VIDEO);
777 final RtpContentMap localContentMap = customRollback();
778 modifyLocalContentMap(localContentMap);
779 } catch (final Exception e) {
780 final Throwable cause = Throwables.getRootCause(e);
781 Log.d(
782 Config.LOGTAG,
783 id.getAccount().getJid().asBareJid()
784 + ": unable to rollback local description after trying to retract content-add",
785 cause);
786 webRTCWrapper.close();
787 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
788 return;
789 }
790 this.outgoingContentAdd = null;
791 final JinglePacket retract =
792 outgoingContentAdd
793 .toStub()
794 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
795 this.send(retract);
796 Log.d(
797 Config.LOGTAG,
798 id.getAccount().getJid()
799 + ": retract content-add "
800 + ContentAddition.summary(outgoingContentAdd));
801 }
802
803 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
804 final SessionDescription sdp = setLocalSessionDescription();
805 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
806 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
807 this.webRTCWrapper
808 .setRemoteDescription(
809 new org.webrtc.SessionDescription(
810 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
811 .get();
812 return localRtpContentMap;
813 }
814
815 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
816 final RtpContentMap currentRemote = getRemoteContentMap();
817 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
818 if (diff.isEmpty()) {
819 throw new IllegalStateException(
820 "Unexpected rollback condition. No difference between local and remote");
821 }
822 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
823 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
824 final RtpContentMap nextRemote =
825 currentRemote.addContent(
826 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
827 return SessionDescription.of(nextRemote, !isInitiator());
828 }
829 throw new IllegalStateException(
830 "Unexpected rollback condition. Senders were not uniformly none");
831 }
832
833 public synchronized void acceptContentAdd(
834 @NonNull final Set<ContentAddition.Summary> contentAddition) {
835 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
836 if (incomingContentAdd == null) {
837 throw new IllegalStateException("No incoming content add");
838 }
839
840 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
841 this.incomingContentAdd = null;
842 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
843 Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders);
844 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
845 Log.d(
846 Config.LOGTAG,
847 "content addition is receive only. we want to upgrade to 'both'");
848 final RtpContentMap modifiedSenders =
849 incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
850 final JinglePacket proposedContentModification =
851 modifiedSenders
852 .toStub()
853 .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
854 proposedContentModification.setTo(id.with);
855 xmppConnectionService.sendIqPacket(
856 id.account,
857 proposedContentModification,
858 (account, response) -> {
859 if (response.getType() == IqPacket.TYPE.RESULT) {
860 Log.d(
861 Config.LOGTAG,
862 id.account.getJid().asBareJid()
863 + ": remote has accepted our upgrade to senders=both");
864 acceptContentAdd(
865 ContentAddition.summary(modifiedSenders), modifiedSenders);
866 } else {
867 Log.d(
868 Config.LOGTAG,
869 id.account.getJid().asBareJid()
870 + ": remote has rejected our upgrade to senders=both");
871 acceptContentAdd(contentAddition, incomingContentAdd);
872 }
873 });
874 } else {
875 acceptContentAdd(contentAddition, incomingContentAdd);
876 }
877 } else {
878 throw new IllegalStateException(
879 "Accepted content add does not match pending content-add");
880 }
881 }
882
883 private void acceptContentAdd(
884 @NonNull final Set<ContentAddition.Summary> contentAddition,
885 final RtpContentMap incomingContentAdd) {
886 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
887 final RtpContentMap modifiedContentMap =
888 getRemoteContentMap().addContent(incomingContentAdd, setup);
889 this.setRemoteContentMap(modifiedContentMap);
890
891 final SessionDescription offer;
892 try {
893 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
894 } catch (final IllegalArgumentException | NullPointerException e) {
895 Log.d(
896 Config.LOGTAG,
897 id.getAccount().getJid().asBareJid()
898 + ": unable convert offer from content-add to SDP",
899 e);
900 webRTCWrapper.close();
901 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
902 return;
903 }
904 this.incomingContentAdd = null;
905 acceptContentAdd(contentAddition, offer);
906 }
907
908 private void acceptContentAdd(
909 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
910 final org.webrtc.SessionDescription sdp =
911 new org.webrtc.SessionDescription(
912 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
913 try {
914 this.webRTCWrapper.setRemoteDescription(sdp).get();
915
916 // TODO add tracks for 'media' where contentAddition.senders matches
917
918 // TODO if senders.sending(isInitiator())
919
920 this.webRTCWrapper.addTrack(Media.VIDEO);
921
922 // TODO add additional transceivers for recv only cases
923
924 final SessionDescription answer = setLocalSessionDescription();
925 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
926
927 final RtpContentMap contentAcceptMap =
928 rtpContentMap.toContentModification(
929 Collections2.transform(contentAddition, ca -> ca.name));
930
931 Log.d(
932 Config.LOGTAG,
933 id.getAccount().getJid().asBareJid()
934 + ": sending content-accept "
935 + ContentAddition.summary(contentAcceptMap));
936
937 addIceCandidatesFromBlackLog();
938
939 modifyLocalContentMap(rtpContentMap);
940 final ListenableFuture<RtpContentMap> future =
941 prepareOutgoingContentMap(contentAcceptMap);
942 Futures.addCallback(
943 future,
944 new FutureCallback<>() {
945 @Override
946 public void onSuccess(final RtpContentMap rtpContentMap) {
947 sendContentAccept(rtpContentMap);
948 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
949 }
950
951 @Override
952 public void onFailure(@NonNull final Throwable throwable) {
953 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
954 }
955 },
956 MoreExecutors.directExecutor());
957 } catch (final Exception e) {
958 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
959 webRTCWrapper.close();
960 sendSessionTerminate(Reason.FAILED_APPLICATION);
961 }
962 }
963
964 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
965 final JinglePacket jinglePacket =
966 contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
967 send(jinglePacket);
968 }
969
970 public synchronized void rejectContentAdd() {
971 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
972 if (incomingContentAdd == null) {
973 throw new IllegalStateException("No incoming content add");
974 }
975 this.incomingContentAdd = null;
976 updateEndUserState();
977 rejectContentAdd(incomingContentAdd);
978 }
979
980 private void rejectContentAdd(final RtpContentMap contentMap) {
981 final JinglePacket jinglePacket =
982 contentMap
983 .toStub()
984 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
985 Log.d(
986 Config.LOGTAG,
987 id.getAccount().getJid().asBareJid()
988 + ": rejecting content "
989 + ContentAddition.summary(contentMap));
990 send(jinglePacket);
991 }
992
993 private boolean checkForIceRestart(
994 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
995 final RtpContentMap existing = getRemoteContentMap();
996 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
997 final IceUdpTransportInfo.Credentials newCredentials;
998 try {
999 existingCredentials = existing.getCredentials();
1000 newCredentials = rtpContentMap.getDistinctCredentials();
1001 } catch (final IllegalStateException e) {
1002 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
1003 return false;
1004 }
1005 if (existingCredentials.contains(newCredentials)) {
1006 return false;
1007 }
1008 // TODO an alternative approach is to check if we already got an iq result to our
1009 // ICE-restart
1010 // and if that's the case we are seeing an answer.
1011 // This might be more spec compliant but also more error prone potentially
1012 final boolean isSignalStateStable =
1013 this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
1014 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
1015 final boolean isOffer = rtpContentMap.emptyCandidates();
1016 final RtpContentMap restartContentMap;
1017 try {
1018 if (isOffer) {
1019 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
1020 restartContentMap =
1021 existing.modifiedCredentials(
1022 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
1023 } else {
1024 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
1025 Log.d(
1026 Config.LOGTAG,
1027 "received confirmation of ICE restart"
1028 + newCredentials
1029 + " peer_setup="
1030 + setup);
1031 // DTLS setup attribute needs to be rewritten to reflect current peer state
1032 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
1033 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
1034 }
1035 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
1036 return isOffer;
1037 } else {
1038 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
1039 respondWithTieBreak(jinglePacket);
1040 return true;
1041 }
1042 } catch (final Exception exception) {
1043 respondOk(jinglePacket);
1044 final Throwable rootCause = Throwables.getRootCause(exception);
1045 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
1046 // If this happens a termination is already in progress
1047 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
1048 return true;
1049 }
1050 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
1051 webRTCWrapper.close();
1052 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1053 return true;
1054 }
1055 }
1056
1057 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
1058 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
1059 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
1060 throw new IllegalStateException("Invalid peer setup");
1061 }
1062 return peerSetup;
1063 }
1064
1065 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
1066 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
1067 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
1068 }
1069 this.peerDtlsSetup = setup;
1070 }
1071
1072 private boolean applyIceRestart(
1073 final JinglePacket jinglePacket,
1074 final RtpContentMap restartContentMap,
1075 final boolean isOffer)
1076 throws ExecutionException, InterruptedException {
1077 final SessionDescription sessionDescription =
1078 SessionDescription.of(restartContentMap, !isInitiator());
1079 final org.webrtc.SessionDescription.Type type =
1080 isOffer
1081 ? org.webrtc.SessionDescription.Type.OFFER
1082 : org.webrtc.SessionDescription.Type.ANSWER;
1083 org.webrtc.SessionDescription sdp =
1084 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1085 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1086 if (isInitiator()) {
1087 // We ignore the offer and respond with tie-break. This will clause the responder
1088 // not to apply the content map
1089 return false;
1090 }
1091 }
1092 webRTCWrapper.setRemoteDescription(sdp).get();
1093 setRemoteContentMap(restartContentMap);
1094 if (isOffer) {
1095 final SessionDescription localSessionDescription = setLocalSessionDescription();
1096 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1097 // We need to respond OK before sending any candidates
1098 respondOk(jinglePacket);
1099 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1100 } else {
1101 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1102 }
1103 return true;
1104 }
1105
1106 private void processCandidates(
1107 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
1108 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
1109 processCandidate(content);
1110 }
1111 }
1112
1113 private void processCandidate(
1114 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
1115 final RtpContentMap rtpContentMap = getRemoteContentMap();
1116 final List<String> indices = toIdentificationTags(rtpContentMap);
1117 final String sdpMid = content.getKey(); // aka content name
1118 final IceUdpTransportInfo transport = content.getValue().transport;
1119 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1120
1121 // TODO check that credentials remained the same
1122
1123 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1124 final String sdp;
1125 try {
1126 sdp = candidate.toSdpAttribute(credentials.ufrag);
1127 } catch (final IllegalArgumentException e) {
1128 Log.d(
1129 Config.LOGTAG,
1130 id.account.getJid().asBareJid()
1131 + ": ignoring invalid ICE candidate "
1132 + e.getMessage());
1133 continue;
1134 }
1135 final int mLineIndex = indices.indexOf(sdpMid);
1136 if (mLineIndex < 0) {
1137 Log.w(
1138 Config.LOGTAG,
1139 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1140 }
1141 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1142 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1143 this.webRTCWrapper.addIceCandidate(iceCandidate);
1144 }
1145 }
1146
1147 private RtpContentMap getRemoteContentMap() {
1148 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1149 }
1150
1151 private RtpContentMap getLocalContentMap() {
1152 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1153 }
1154
1155 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1156 final Group originalGroup = rtpContentMap.group;
1157 final List<String> identificationTags =
1158 originalGroup == null
1159 ? rtpContentMap.getNames()
1160 : originalGroup.getIdentificationTags();
1161 if (identificationTags.size() == 0) {
1162 Log.w(
1163 Config.LOGTAG,
1164 id.account.getJid().asBareJid()
1165 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1166 }
1167 return identificationTags;
1168 }
1169
1170 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1171 final JinglePacket jinglePacket, final boolean expectVerification) {
1172 try {
1173 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1174 } catch (final Exception e) {
1175 return Futures.immediateFailedFuture(e);
1176 }
1177 }
1178
1179 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1180 final RtpContentMap receivedContentMap, final boolean expectVerification) {
1181 Log.d(
1182 Config.LOGTAG,
1183 "receiveRtpContentMap("
1184 + receivedContentMap.getClass().getSimpleName()
1185 + ",expectVerification="
1186 + expectVerification
1187 + ")");
1188 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1189 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1190 id.account
1191 .getAxolotlService()
1192 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1193 return Futures.transform(
1194 future,
1195 omemoVerifiedPayload -> {
1196 // TODO test if an exception here triggers a correct abort
1197 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1198 Log.d(
1199 Config.LOGTAG,
1200 id.account.getJid().asBareJid()
1201 + ": received verifiable DTLS fingerprint via "
1202 + omemoVerification);
1203 return omemoVerifiedPayload.getPayload();
1204 },
1205 MoreExecutors.directExecutor());
1206 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1207 return Futures.immediateFailedFuture(
1208 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1209 } else {
1210 return Futures.immediateFuture(receivedContentMap);
1211 }
1212 }
1213
1214 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1215 if (isInitiator()) {
1216 Log.d(
1217 Config.LOGTAG,
1218 String.format(
1219 "%s: received session-initiate even though we were initiating",
1220 id.account.getJid().asBareJid()));
1221 if (isTerminated()) {
1222 Log.d(
1223 Config.LOGTAG,
1224 String.format(
1225 "%s: got a reason to terminate with out-of-order. but already in state %s",
1226 id.account.getJid().asBareJid(), getState()));
1227 respondWithOutOfOrder(jinglePacket);
1228 } else {
1229 terminateWithOutOfOrder(jinglePacket);
1230 }
1231 return;
1232 }
1233 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1234 Futures.addCallback(
1235 future,
1236 new FutureCallback<>() {
1237 @Override
1238 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1239 receiveSessionInitiate(jinglePacket, rtpContentMap);
1240 }
1241
1242 @Override
1243 public void onFailure(@NonNull final Throwable throwable) {
1244 respondOk(jinglePacket);
1245 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1246 }
1247 },
1248 MoreExecutors.directExecutor());
1249 }
1250
1251 private void receiveSessionInitiate(
1252 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1253 try {
1254 contentMap.requireContentDescriptions();
1255 contentMap.requireDTLSFingerprint(true);
1256 } catch (final RuntimeException e) {
1257 Log.d(
1258 Config.LOGTAG,
1259 id.account.getJid().asBareJid() + ": improperly formatted contents",
1260 Throwables.getRootCause(e));
1261 respondOk(jinglePacket);
1262 sendSessionTerminate(Reason.of(e), e.getMessage());
1263 return;
1264 }
1265 Log.d(
1266 Config.LOGTAG,
1267 "processing session-init with " + contentMap.contents.size() + " contents");
1268 final State target;
1269 if (this.state == State.PROCEED) {
1270 Preconditions.checkState(
1271 proposedMedia != null && proposedMedia.size() > 0,
1272 "proposed media must be set when processing pre-approved session-initiate");
1273 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1274 sendSessionTerminate(
1275 Reason.SECURITY_ERROR,
1276 String.format(
1277 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1278 this.proposedMedia, contentMap.getMedia()));
1279 return;
1280 }
1281 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1282 } else {
1283 target = State.SESSION_INITIALIZED;
1284 }
1285 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1286 respondOk(jinglePacket);
1287 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1288 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1289 Log.d(
1290 Config.LOGTAG,
1291 id.account.getJid().asBareJid()
1292 + ": automatically accepting session-initiate");
1293 sendSessionAccept();
1294 } else {
1295 Log.d(
1296 Config.LOGTAG,
1297 id.account.getJid().asBareJid()
1298 + ": received not pre-approved session-initiate. start ringing");
1299 startRinging();
1300 }
1301 } else {
1302 Log.d(
1303 Config.LOGTAG,
1304 String.format(
1305 "%s: received session-initiate while in state %s",
1306 id.account.getJid().asBareJid(), state));
1307 terminateWithOutOfOrder(jinglePacket);
1308 }
1309 }
1310
1311 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1312 if (!isInitiator()) {
1313 Log.d(
1314 Config.LOGTAG,
1315 String.format(
1316 "%s: received session-accept even though we were responding",
1317 id.account.getJid().asBareJid()));
1318 terminateWithOutOfOrder(jinglePacket);
1319 return;
1320 }
1321 final ListenableFuture<RtpContentMap> future =
1322 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1323 Futures.addCallback(
1324 future,
1325 new FutureCallback<>() {
1326 @Override
1327 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1328 receiveSessionAccept(jinglePacket, rtpContentMap);
1329 }
1330
1331 @Override
1332 public void onFailure(@NonNull final Throwable throwable) {
1333 respondOk(jinglePacket);
1334 Log.d(
1335 Config.LOGTAG,
1336 id.account.getJid().asBareJid()
1337 + ": improperly formatted contents in session-accept",
1338 throwable);
1339 webRTCWrapper.close();
1340 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1341 }
1342 },
1343 MoreExecutors.directExecutor());
1344 }
1345
1346 private void receiveSessionAccept(
1347 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1348 try {
1349 contentMap.requireContentDescriptions();
1350 contentMap.requireDTLSFingerprint();
1351 } catch (final RuntimeException e) {
1352 respondOk(jinglePacket);
1353 Log.d(
1354 Config.LOGTAG,
1355 id.account.getJid().asBareJid()
1356 + ": improperly formatted contents in session-accept",
1357 e);
1358 webRTCWrapper.close();
1359 sendSessionTerminate(Reason.of(e), e.getMessage());
1360 return;
1361 }
1362 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1363 if (!initiatorMedia.equals(contentMap.getMedia())) {
1364 sendSessionTerminate(
1365 Reason.SECURITY_ERROR,
1366 String.format(
1367 "Your session-included included media %s but our session-initiate was %s",
1368 this.proposedMedia, contentMap.getMedia()));
1369 return;
1370 }
1371 Log.d(
1372 Config.LOGTAG,
1373 "processing session-accept with " + contentMap.contents.size() + " contents");
1374 if (transition(State.SESSION_ACCEPTED)) {
1375 respondOk(jinglePacket);
1376 receiveSessionAccept(contentMap);
1377 } else {
1378 Log.d(
1379 Config.LOGTAG,
1380 String.format(
1381 "%s: received session-accept while in state %s",
1382 id.account.getJid().asBareJid(), state));
1383 respondOk(jinglePacket);
1384 }
1385 }
1386
1387 private void receiveSessionAccept(final RtpContentMap contentMap) {
1388 this.responderRtpContentMap = contentMap;
1389 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1390 final SessionDescription sessionDescription;
1391 try {
1392 sessionDescription = SessionDescription.of(contentMap, false);
1393 } catch (final IllegalArgumentException | NullPointerException e) {
1394 Log.d(
1395 Config.LOGTAG,
1396 id.account.getJid().asBareJid()
1397 + ": unable convert offer from session-accept to SDP",
1398 e);
1399 webRTCWrapper.close();
1400 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1401 return;
1402 }
1403 final org.webrtc.SessionDescription answer =
1404 new org.webrtc.SessionDescription(
1405 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1406 try {
1407 this.webRTCWrapper.setRemoteDescription(answer).get();
1408 } catch (final Exception e) {
1409 Log.d(
1410 Config.LOGTAG,
1411 id.account.getJid().asBareJid()
1412 + ": unable to set remote description after receiving session-accept",
1413 Throwables.getRootCause(e));
1414 webRTCWrapper.close();
1415 sendSessionTerminate(
1416 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1417 return;
1418 }
1419 processCandidates(contentMap.contents.entrySet());
1420 }
1421
1422 private void sendSessionAccept() {
1423 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1424 if (rtpContentMap == null) {
1425 throw new IllegalStateException("initiator RTP Content Map has not been set");
1426 }
1427 final SessionDescription offer;
1428 try {
1429 offer = SessionDescription.of(rtpContentMap, true);
1430 } catch (final IllegalArgumentException | NullPointerException e) {
1431 Log.d(
1432 Config.LOGTAG,
1433 id.account.getJid().asBareJid()
1434 + ": unable convert offer from session-initiate to SDP",
1435 e);
1436 webRTCWrapper.close();
1437 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1438 return;
1439 }
1440 sendSessionAccept(rtpContentMap.getMedia(), offer);
1441 }
1442
1443 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1444 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1445 }
1446
1447 private synchronized void sendSessionAccept(
1448 final Set<Media> media,
1449 final SessionDescription offer,
1450 final List<PeerConnection.IceServer> iceServers) {
1451 if (isTerminated()) {
1452 Log.w(
1453 Config.LOGTAG,
1454 id.account.getJid().asBareJid()
1455 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1456 return;
1457 }
1458 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1459 try {
1460 setupWebRTC(media, iceServers, !includeCandidates);
1461 } catch (final WebRTCWrapper.InitializationException e) {
1462 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1463 webRTCWrapper.close();
1464 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1465 return;
1466 }
1467 final org.webrtc.SessionDescription sdp =
1468 new org.webrtc.SessionDescription(
1469 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1470 try {
1471 this.webRTCWrapper.setRemoteDescription(sdp).get();
1472 addIceCandidatesFromBlackLog();
1473 org.webrtc.SessionDescription webRTCSessionDescription =
1474 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1475 prepareSessionAccept(webRTCSessionDescription, includeCandidates);
1476 } catch (final Exception e) {
1477 failureToAcceptSession(e);
1478 }
1479 }
1480
1481 private void failureToAcceptSession(final Throwable throwable) {
1482 if (isTerminated()) {
1483 return;
1484 }
1485 final Throwable rootCause = Throwables.getRootCause(throwable);
1486 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1487 webRTCWrapper.close();
1488 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1489 }
1490
1491 private void failureToPerformAction(
1492 final JinglePacket.Action action, final Throwable throwable) {
1493 if (isTerminated()) {
1494 return;
1495 }
1496 final Throwable rootCause = Throwables.getRootCause(throwable);
1497 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1498 webRTCWrapper.close();
1499 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1500 }
1501
1502 private void addIceCandidatesFromBlackLog() {
1503 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1504 while ((foo = this.pendingIceCandidates.poll()) != null) {
1505 processCandidate(foo);
1506 Log.d(
1507 Config.LOGTAG,
1508 id.account.getJid().asBareJid() + ": added candidate from back log");
1509 }
1510 }
1511
1512 private void prepareSessionAccept(
1513 final org.webrtc.SessionDescription webRTCSessionDescription,
1514 final boolean includeCandidates) {
1515 final SessionDescription sessionDescription =
1516 SessionDescription.parse(webRTCSessionDescription.description);
1517 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1518 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1519 if (includeCandidates) {
1520 candidates = parseCandidates(sessionDescription);
1521 } else {
1522 candidates = ImmutableMultimap.of();
1523 }
1524 this.responderRtpContentMap = respondingRtpContentMap;
1525 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1526 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1527 prepareOutgoingContentMap(respondingRtpContentMap);
1528 Futures.addCallback(
1529 outgoingContentMapFuture,
1530 new FutureCallback<>() {
1531 @Override
1532 public void onSuccess(final RtpContentMap outgoingContentMap) {
1533 if (includeCandidates) {
1534 Log.d(
1535 Config.LOGTAG,
1536 "including "
1537 + candidates.size()
1538 + " candidates in session accept");
1539 sendSessionAccept(outgoingContentMap.withCandidates(candidates));
1540 webRTCWrapper.resetPendingCandidates();
1541 } else {
1542 sendSessionAccept(outgoingContentMap);
1543 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1544 }
1545 }
1546
1547 @Override
1548 public void onFailure(@NonNull Throwable throwable) {
1549 failureToAcceptSession(throwable);
1550 }
1551 },
1552 MoreExecutors.directExecutor());
1553 }
1554
1555 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1556 if (isTerminated()) {
1557 Log.w(
1558 Config.LOGTAG,
1559 id.account.getJid().asBareJid()
1560 + ": preparing session accept was too slow. already terminated. nothing to do.");
1561 return;
1562 }
1563 transitionOrThrow(State.SESSION_ACCEPTED);
1564 final JinglePacket sessionAccept =
1565 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1566 send(sessionAccept);
1567 }
1568
1569 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1570 final RtpContentMap rtpContentMap) {
1571 if (this.omemoVerification.hasDeviceId()) {
1572 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1573 verifiedPayloadFuture =
1574 id.account
1575 .getAxolotlService()
1576 .encrypt(
1577 rtpContentMap,
1578 id.with,
1579 omemoVerification.getDeviceId());
1580 return Futures.transform(
1581 verifiedPayloadFuture,
1582 verifiedPayload -> {
1583 omemoVerification.setOrEnsureEqual(verifiedPayload);
1584 return verifiedPayload.getPayload();
1585 },
1586 MoreExecutors.directExecutor());
1587 } else {
1588 return Futures.immediateFuture(rtpContentMap);
1589 }
1590 }
1591
1592 synchronized void deliveryMessage(
1593 final Jid from,
1594 final Element message,
1595 final String serverMessageId,
1596 final long timestamp) {
1597 Log.d(
1598 Config.LOGTAG,
1599 id.account.getJid().asBareJid()
1600 + ": delivered message to JingleRtpConnection "
1601 + message);
1602 switch (message.getName()) {
1603 case "propose" -> receivePropose(
1604 from, Propose.upgrade(message), serverMessageId, timestamp);
1605 case "proceed" -> receiveProceed(
1606 from, Proceed.upgrade(message), serverMessageId, timestamp);
1607 case "retract" -> receiveRetract(from, serverMessageId, timestamp);
1608 case "reject" -> receiveReject(from, serverMessageId, timestamp);
1609 case "accept" -> receiveAccept(from, serverMessageId, timestamp);
1610 }
1611 }
1612
1613 void deliverFailedProceed(final String message) {
1614 Log.d(
1615 Config.LOGTAG,
1616 id.account.getJid().asBareJid()
1617 + ": receive message error for proceed message ("
1618 + Strings.nullToEmpty(message)
1619 + ")");
1620 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1621 webRTCWrapper.close();
1622 Log.d(
1623 Config.LOGTAG,
1624 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1625 this.finish();
1626 }
1627 }
1628
1629 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1630 final boolean originatedFromMyself =
1631 from.asBareJid().equals(id.account.getJid().asBareJid());
1632 if (originatedFromMyself) {
1633 if (transition(State.ACCEPTED)) {
1634 acceptedOnOtherDevice(serverMsgId, timestamp);
1635 } else {
1636 Log.d(
1637 Config.LOGTAG,
1638 id.account.getJid().asBareJid()
1639 + ": unable to transition to accept because already in state="
1640 + this.state);
1641 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1642 }
1643 } else {
1644 Log.d(
1645 Config.LOGTAG,
1646 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1647 }
1648 }
1649
1650 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1651 if (serverMsgId != null) {
1652 this.message.setServerMsgId(serverMsgId);
1653 }
1654 this.message.setTime(timestamp);
1655 this.message.setCarbon(true); // indicate that call was accepted on other device
1656 this.writeLogMessageSuccess(0);
1657 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1658 this.finish();
1659 }
1660
1661 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1662 final boolean originatedFromMyself =
1663 from.asBareJid().equals(id.account.getJid().asBareJid());
1664 // reject from another one of my clients
1665 if (originatedFromMyself) {
1666 receiveRejectFromMyself(serverMsgId, timestamp);
1667 } else if (isInitiator()) {
1668 if (from.equals(id.with)) {
1669 receiveRejectFromResponder();
1670 } else {
1671 Log.d(
1672 Config.LOGTAG,
1673 id.account.getJid()
1674 + ": ignoring reject from "
1675 + from
1676 + " for session with "
1677 + id.with);
1678 }
1679 } else {
1680 Log.d(
1681 Config.LOGTAG,
1682 id.account.getJid()
1683 + ": ignoring reject from "
1684 + from
1685 + " for session with "
1686 + id.with);
1687 }
1688 }
1689
1690 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1691 if (transition(State.REJECTED)) {
1692 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1693 this.finish();
1694 if (serverMsgId != null) {
1695 this.message.setServerMsgId(serverMsgId);
1696 }
1697 this.message.setTime(timestamp);
1698 this.message.setCarbon(true); // indicate that call was rejected on other device
1699 writeLogMessageMissed();
1700 } else {
1701 Log.d(
1702 Config.LOGTAG,
1703 "not able to transition into REJECTED because already in " + this.state);
1704 }
1705 }
1706
1707 private void receiveRejectFromResponder() {
1708 if (isInState(State.PROCEED)) {
1709 Log.d(
1710 Config.LOGTAG,
1711 id.account.getJid()
1712 + ": received reject while still in proceed. callee reconsidered");
1713 closeTransitionLogFinish(State.REJECTED_RACED);
1714 return;
1715 }
1716 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1717 Log.d(
1718 Config.LOGTAG,
1719 id.account.getJid()
1720 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1721 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1722 return;
1723 }
1724 Log.d(
1725 Config.LOGTAG,
1726 id.account.getJid()
1727 + ": ignoring reject from responder because already in state "
1728 + this.state);
1729 }
1730
1731 private void receivePropose(
1732 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1733 final boolean originatedFromMyself =
1734 from.asBareJid().equals(id.account.getJid().asBareJid());
1735 if (originatedFromMyself) {
1736 Log.d(
1737 Config.LOGTAG,
1738 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1739 } else if (transition(
1740 State.PROPOSED,
1741 () -> {
1742 final Collection<RtpDescription> descriptions =
1743 Collections2.transform(
1744 Collections2.filter(
1745 propose.getDescriptions(),
1746 d -> d instanceof RtpDescription),
1747 input -> (RtpDescription) input);
1748 final Collection<Media> media =
1749 Collections2.transform(descriptions, RtpDescription::getMedia);
1750 Preconditions.checkState(
1751 !media.contains(Media.UNKNOWN),
1752 "RTP descriptions contain unknown media");
1753 Log.d(
1754 Config.LOGTAG,
1755 id.account.getJid().asBareJid()
1756 + ": received session proposal from "
1757 + from
1758 + " for "
1759 + media);
1760 this.proposedMedia = Sets.newHashSet(media);
1761 })) {
1762 if (serverMsgId != null) {
1763 this.message.setServerMsgId(serverMsgId);
1764 }
1765 this.message.setTime(timestamp);
1766 startRinging();
1767 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1768 sendJingleMessage("ringing");
1769 }
1770 } else {
1771 Log.d(
1772 Config.LOGTAG,
1773 id.account.getJid()
1774 + ": ignoring session proposal because already in "
1775 + state);
1776 }
1777 }
1778
1779 private void startRinging() {
1780 Log.d(
1781 Config.LOGTAG,
1782 id.account.getJid().asBareJid()
1783 + ": received call from "
1784 + id.with
1785 + ". start ringing");
1786 ringingTimeoutFuture =
1787 jingleConnectionManager.schedule(
1788 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1789 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1790 }
1791
1792 private synchronized void ringingTimeout() {
1793 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1794 switch (this.state) {
1795 case PROPOSED -> {
1796 message.markUnread();
1797 rejectCallFromProposed();
1798 }
1799 case SESSION_INITIALIZED -> {
1800 message.markUnread();
1801 rejectCallFromSessionInitiate();
1802 }
1803 }
1804 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1805 }
1806
1807 private void cancelRingingTimeout() {
1808 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1809 if (future != null && !future.isCancelled()) {
1810 future.cancel(false);
1811 }
1812 }
1813
1814 private void receiveProceed(
1815 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1816 final Set<Media> media =
1817 Preconditions.checkNotNull(
1818 this.proposedMedia, "Proposed media has to be set before handling proceed");
1819 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1820 if (from.equals(id.with)) {
1821 if (isInitiator()) {
1822 if (transition(State.PROCEED)) {
1823 if (serverMsgId != null) {
1824 this.message.setServerMsgId(serverMsgId);
1825 }
1826 this.message.setTime(timestamp);
1827 final Integer remoteDeviceId = proceed.getDeviceId();
1828 if (isOmemoEnabled()) {
1829 this.omemoVerification.setDeviceId(remoteDeviceId);
1830 } else {
1831 if (remoteDeviceId != null) {
1832 Log.d(
1833 Config.LOGTAG,
1834 id.account.getJid().asBareJid()
1835 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1836 }
1837 this.omemoVerification.setDeviceId(null);
1838 }
1839 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1840 } else {
1841 Log.d(
1842 Config.LOGTAG,
1843 String.format(
1844 "%s: ignoring proceed because already in %s",
1845 id.account.getJid().asBareJid(), this.state));
1846 }
1847 } else {
1848 Log.d(
1849 Config.LOGTAG,
1850 String.format(
1851 "%s: ignoring proceed because we were not initializing",
1852 id.account.getJid().asBareJid()));
1853 }
1854 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1855 if (transition(State.ACCEPTED)) {
1856 Log.d(
1857 Config.LOGTAG,
1858 id.account.getJid().asBareJid()
1859 + ": moved session with "
1860 + id.with
1861 + " into state accepted after received carbon copied proceed");
1862 acceptedOnOtherDevice(serverMsgId, timestamp);
1863 }
1864 } else {
1865 Log.d(
1866 Config.LOGTAG,
1867 String.format(
1868 "%s: ignoring proceed from %s. was expected from %s",
1869 id.account.getJid().asBareJid(), from, id.with));
1870 }
1871 }
1872
1873 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1874 if (from.equals(id.with)) {
1875 final State target =
1876 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1877 if (transition(target)) {
1878 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1879 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1880 Log.d(
1881 Config.LOGTAG,
1882 id.account.getJid().asBareJid()
1883 + ": session with "
1884 + id.with
1885 + " has been retracted (serverMsgId="
1886 + serverMsgId
1887 + ")");
1888 if (serverMsgId != null) {
1889 this.message.setServerMsgId(serverMsgId);
1890 }
1891 this.message.setTime(timestamp);
1892 if (target == State.RETRACTED) {
1893 this.message.markUnread();
1894 }
1895 writeLogMessageMissed();
1896 finish();
1897 } else {
1898 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1899 }
1900 } else {
1901 // TODO parse retract from self
1902 Log.d(
1903 Config.LOGTAG,
1904 id.account.getJid().asBareJid()
1905 + ": received retract from "
1906 + from
1907 + ". expected retract from"
1908 + id.with
1909 + ". ignoring");
1910 }
1911 }
1912
1913 public void sendSessionInitiate() {
1914 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1915 }
1916
1917 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1918 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1919 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1920 }
1921
1922 private synchronized void sendSessionInitiate(
1923 final Set<Media> media,
1924 final State targetState,
1925 final List<PeerConnection.IceServer> iceServers) {
1926 if (isTerminated()) {
1927 Log.w(
1928 Config.LOGTAG,
1929 id.account.getJid().asBareJid()
1930 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1931 return;
1932 }
1933 final boolean includeCandidates = remoteHasSdpOfferAnswer();
1934 try {
1935 setupWebRTC(media, iceServers, !includeCandidates);
1936 } catch (final WebRTCWrapper.InitializationException e) {
1937 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1938 webRTCWrapper.close();
1939 sendRetract(Reason.ofThrowable(e));
1940 return;
1941 }
1942 try {
1943 org.webrtc.SessionDescription webRTCSessionDescription =
1944 this.webRTCWrapper.setLocalDescription(includeCandidates).get();
1945 prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
1946 } catch (final Exception e) {
1947 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1948 // exceptions
1949 failureToInitiateSession(e, targetState);
1950 }
1951 }
1952
1953 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1954 if (isTerminated()) {
1955 return;
1956 }
1957 Log.d(
1958 Config.LOGTAG,
1959 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1960 Throwables.getRootCause(throwable));
1961 webRTCWrapper.close();
1962 final Reason reason = Reason.ofThrowable(throwable);
1963 if (isInState(targetState)) {
1964 sendSessionTerminate(reason, throwable.getMessage());
1965 } else {
1966 sendRetract(reason);
1967 }
1968 }
1969
1970 private void sendRetract(final Reason reason) {
1971 // TODO embed reason into retract
1972 sendJingleMessage("retract", id.with.asBareJid());
1973 transitionOrThrow(reasonToState(reason));
1974 this.finish();
1975 }
1976
1977 private void prepareSessionInitiate(
1978 final org.webrtc.SessionDescription webRTCSessionDescription,
1979 final boolean includeCandidates,
1980 final State targetState) {
1981 final SessionDescription sessionDescription =
1982 SessionDescription.parse(webRTCSessionDescription.description);
1983 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1984 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
1985 if (includeCandidates) {
1986 candidates = parseCandidates(sessionDescription);
1987 } else {
1988 candidates = ImmutableMultimap.of();
1989 }
1990 this.initiatorRtpContentMap = rtpContentMap;
1991 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1992 encryptSessionInitiate(rtpContentMap);
1993 Futures.addCallback(
1994 outgoingContentMapFuture,
1995 new FutureCallback<>() {
1996 @Override
1997 public void onSuccess(final RtpContentMap outgoingContentMap) {
1998 if (includeCandidates) {
1999 Log.d(
2000 Config.LOGTAG,
2001 "including "
2002 + candidates.size()
2003 + " candidates in session initiate");
2004 sendSessionInitiate(
2005 outgoingContentMap.withCandidates(candidates), targetState);
2006 webRTCWrapper.resetPendingCandidates();
2007 } else {
2008 sendSessionInitiate(outgoingContentMap, targetState);
2009 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2010 }
2011 }
2012
2013 @Override
2014 public void onFailure(@NonNull final Throwable throwable) {
2015 failureToInitiateSession(throwable, targetState);
2016 }
2017 },
2018 MoreExecutors.directExecutor());
2019 }
2020
2021 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
2022 if (isTerminated()) {
2023 Log.w(
2024 Config.LOGTAG,
2025 id.account.getJid().asBareJid()
2026 + ": preparing session was too slow. already terminated. nothing to do.");
2027 return;
2028 }
2029 this.transitionOrThrow(targetState);
2030 final JinglePacket sessionInitiate =
2031 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
2032 send(sessionInitiate);
2033 }
2034
2035 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
2036 final RtpContentMap rtpContentMap) {
2037 if (this.omemoVerification.hasDeviceId()) {
2038 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
2039 verifiedPayloadFuture =
2040 id.account
2041 .getAxolotlService()
2042 .encrypt(
2043 rtpContentMap,
2044 id.with,
2045 omemoVerification.getDeviceId());
2046 final ListenableFuture<RtpContentMap> future =
2047 Futures.transform(
2048 verifiedPayloadFuture,
2049 verifiedPayload -> {
2050 omemoVerification.setSessionFingerprint(
2051 verifiedPayload.getFingerprint());
2052 return verifiedPayload.getPayload();
2053 },
2054 MoreExecutors.directExecutor());
2055 if (Config.REQUIRE_RTP_VERIFICATION) {
2056 return future;
2057 }
2058 return Futures.catching(
2059 future,
2060 CryptoFailedException.class,
2061 e -> {
2062 Log.w(
2063 Config.LOGTAG,
2064 id.account.getJid().asBareJid()
2065 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
2066 e);
2067 return rtpContentMap;
2068 },
2069 MoreExecutors.directExecutor());
2070 } else {
2071 return Futures.immediateFuture(rtpContentMap);
2072 }
2073 }
2074
2075 private void sendSessionTerminate(final Reason reason) {
2076 sendSessionTerminate(reason, null);
2077 }
2078
2079 private void sendSessionTerminate(final Reason reason, final String text) {
2080 final State previous = this.state;
2081 final State target = reasonToState(reason);
2082 transitionOrThrow(target);
2083 if (previous != State.NULL) {
2084 writeLogMessage(target);
2085 }
2086 final JinglePacket jinglePacket =
2087 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
2088 jinglePacket.setReason(reason, text);
2089 send(jinglePacket);
2090 finish();
2091 }
2092
2093 private void sendTransportInfo(
2094 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2095 final RtpContentMap transportInfo;
2096 try {
2097 final RtpContentMap rtpContentMap =
2098 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2099 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2100 } catch (final Exception e) {
2101 Log.d(
2102 Config.LOGTAG,
2103 id.account.getJid().asBareJid()
2104 + ": unable to prepare transport-info from candidate for content="
2105 + contentName);
2106 return;
2107 }
2108 final JinglePacket jinglePacket =
2109 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2110 send(jinglePacket);
2111 }
2112
2113 private void send(final JinglePacket jinglePacket) {
2114 jinglePacket.setTo(id.with);
2115 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
2116 }
2117
2118 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
2119 if (response.getType() == IqPacket.TYPE.ERROR) {
2120 handleIqErrorResponse(response);
2121 return;
2122 }
2123 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2124 handleIqTimeoutResponse(response);
2125 }
2126 }
2127
2128 private void handleIqErrorResponse(final IqPacket response) {
2129 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
2130 final String errorCondition = response.getErrorCondition();
2131 Log.d(
2132 Config.LOGTAG,
2133 id.account.getJid().asBareJid()
2134 + ": received IQ-error from "
2135 + response.getFrom()
2136 + " in RTP session. "
2137 + errorCondition);
2138 if (isTerminated()) {
2139 Log.i(
2140 Config.LOGTAG,
2141 id.account.getJid().asBareJid()
2142 + ": ignoring error because session was already terminated");
2143 return;
2144 }
2145 this.webRTCWrapper.close();
2146 final State target;
2147 if (Arrays.asList(
2148 "service-unavailable",
2149 "recipient-unavailable",
2150 "remote-server-not-found",
2151 "remote-server-timeout")
2152 .contains(errorCondition)) {
2153 target = State.TERMINATED_CONNECTIVITY_ERROR;
2154 } else {
2155 target = State.TERMINATED_APPLICATION_FAILURE;
2156 }
2157 transitionOrThrow(target);
2158 this.finish();
2159 }
2160
2161 private void handleIqTimeoutResponse(final IqPacket response) {
2162 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
2163 Log.d(
2164 Config.LOGTAG,
2165 id.account.getJid().asBareJid()
2166 + ": received IQ timeout in RTP session with "
2167 + id.with
2168 + ". terminating with connectivity error");
2169 if (isTerminated()) {
2170 Log.i(
2171 Config.LOGTAG,
2172 id.account.getJid().asBareJid()
2173 + ": ignoring error because session was already terminated");
2174 return;
2175 }
2176 this.webRTCWrapper.close();
2177 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
2178 this.finish();
2179 }
2180
2181 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
2182 Log.d(
2183 Config.LOGTAG,
2184 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
2185 this.webRTCWrapper.close();
2186 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
2187 respondWithOutOfOrder(jinglePacket);
2188 this.finish();
2189 }
2190
2191 private void respondWithTieBreak(final JinglePacket jinglePacket) {
2192 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
2193 }
2194
2195 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
2196 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
2197 }
2198
2199 private void respondWithItemNotFound(final JinglePacket jinglePacket) {
2200 respondWithJingleError(jinglePacket, null, "item-not-found", "cancel");
2201 }
2202
2203 void respondWithJingleError(
2204 final IqPacket original,
2205 String jingleCondition,
2206 String condition,
2207 String conditionType) {
2208 jingleConnectionManager.respondWithJingleError(
2209 id.account, original, jingleCondition, condition, conditionType);
2210 }
2211
2212 private void respondOk(final JinglePacket jinglePacket) {
2213 xmppConnectionService.sendIqPacket(
2214 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
2215 }
2216
2217 public RtpEndUserState getEndUserState() {
2218 switch (this.state) {
2219 case NULL, PROPOSED, SESSION_INITIALIZED -> {
2220 if (isInitiator()) {
2221 return RtpEndUserState.RINGING;
2222 } else {
2223 return RtpEndUserState.INCOMING_CALL;
2224 }
2225 }
2226 case PROCEED -> {
2227 if (isInitiator()) {
2228 return RtpEndUserState.RINGING;
2229 } else {
2230 return RtpEndUserState.ACCEPTING_CALL;
2231 }
2232 }
2233 case SESSION_INITIALIZED_PRE_APPROVED -> {
2234 if (isInitiator()) {
2235 return RtpEndUserState.RINGING;
2236 } else {
2237 return RtpEndUserState.CONNECTING;
2238 }
2239 }
2240 case SESSION_ACCEPTED -> {
2241 final ContentAddition ca = getPendingContentAddition();
2242 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2243 return RtpEndUserState.INCOMING_CONTENT_ADD;
2244 }
2245 return getPeerConnectionStateAsEndUserState();
2246 }
2247 case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> {
2248 if (isInitiator()) {
2249 return RtpEndUserState.DECLINED_OR_BUSY;
2250 } else {
2251 return RtpEndUserState.ENDED;
2252 }
2253 }
2254 case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> {
2255 return RtpEndUserState.ENDED;
2256 }
2257 case RETRACTED_RACED -> {
2258 if (isInitiator()) {
2259 return RtpEndUserState.ENDED;
2260 } else {
2261 return RtpEndUserState.RETRACTED;
2262 }
2263 }
2264 case TERMINATED_CONNECTIVITY_ERROR -> {
2265 return zeroDuration()
2266 ? RtpEndUserState.CONNECTIVITY_ERROR
2267 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2268 }
2269 case TERMINATED_APPLICATION_FAILURE -> {
2270 return RtpEndUserState.APPLICATION_ERROR;
2271 }
2272 case TERMINATED_SECURITY_ERROR -> {
2273 return RtpEndUserState.SECURITY_ERROR;
2274 }
2275 }
2276 throw new IllegalStateException(
2277 String.format("%s has no equivalent EndUserState", this.state));
2278 }
2279
2280 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2281 final PeerConnection.PeerConnectionState state;
2282 try {
2283 state = webRTCWrapper.getState();
2284 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2285 // We usually close the WebRTCWrapper *before* transitioning so we might still
2286 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2287 return RtpEndUserState.ENDING_CALL;
2288 }
2289 return switch (state) {
2290 case CONNECTED -> RtpEndUserState.CONNECTED;
2291 case NEW, CONNECTING -> RtpEndUserState.CONNECTING;
2292 case CLOSED -> RtpEndUserState.ENDING_CALL;
2293 default -> zeroDuration()
2294 ? RtpEndUserState.CONNECTIVITY_ERROR
2295 : RtpEndUserState.RECONNECTING;
2296 };
2297 }
2298
2299 public ContentAddition getPendingContentAddition() {
2300 final RtpContentMap in = this.incomingContentAdd;
2301 final RtpContentMap out = this.outgoingContentAdd;
2302 if (out != null) {
2303 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2304 } else if (in != null) {
2305 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2306 } else {
2307 return null;
2308 }
2309 }
2310
2311 public Set<Media> getMedia() {
2312 final State current = getState();
2313 if (current == State.NULL) {
2314 if (isInitiator()) {
2315 return Preconditions.checkNotNull(
2316 this.proposedMedia, "RTP connection has not been initialized properly");
2317 }
2318 throw new IllegalStateException("RTP connection has not been initialized yet");
2319 }
2320 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2321 return Preconditions.checkNotNull(
2322 this.proposedMedia, "RTP connection has not been initialized properly");
2323 }
2324 final RtpContentMap localContentMap = getLocalContentMap();
2325 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2326 if (localContentMap != null) {
2327 return localContentMap.getMedia();
2328 } else if (initiatorContentMap != null) {
2329 return initiatorContentMap.getMedia();
2330 } else if (isTerminated()) {
2331 return Collections.emptySet(); // we might fail before we ever got a chance to set media
2332 } else {
2333 return Preconditions.checkNotNull(
2334 this.proposedMedia, "RTP connection has not been initialized properly");
2335 }
2336 }
2337
2338 public boolean isVerified() {
2339 final String fingerprint = this.omemoVerification.getFingerprint();
2340 if (fingerprint == null) {
2341 return false;
2342 }
2343 final FingerprintStatus status =
2344 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2345 return status != null && status.isVerified();
2346 }
2347
2348 public boolean addMedia(final Media media) {
2349 final Set<Media> currentMedia = getMedia();
2350 if (currentMedia.contains(media)) {
2351 throw new IllegalStateException(String.format("%s has already been proposed", media));
2352 }
2353 // TODO add state protection - can only add while ACCEPTED or so
2354 Log.d(Config.LOGTAG, "adding media: " + media);
2355 return webRTCWrapper.addTrack(media);
2356 }
2357
2358 public synchronized void acceptCall() {
2359 switch (this.state) {
2360 case PROPOSED -> {
2361 cancelRingingTimeout();
2362 acceptCallFromProposed();
2363 }
2364 case SESSION_INITIALIZED -> {
2365 cancelRingingTimeout();
2366 acceptCallFromSessionInitialized();
2367 }
2368 case ACCEPTED -> Log.w(
2369 Config.LOGTAG,
2370 id.account.getJid().asBareJid()
2371 + ": the call has already been accepted with another client. UI was just lagging behind");
2372 case PROCEED, SESSION_ACCEPTED -> Log.w(
2373 Config.LOGTAG,
2374 id.account.getJid().asBareJid()
2375 + ": the call has already been accepted. user probably double tapped the UI");
2376 default -> throw new IllegalStateException("Can not accept call from " + this.state);
2377 }
2378 }
2379
2380 public void notifyPhoneCall() {
2381 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2382 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2383 rejectCall();
2384 } else {
2385 endCall();
2386 }
2387 }
2388
2389 public synchronized void rejectCall() {
2390 if (isTerminated()) {
2391 Log.w(
2392 Config.LOGTAG,
2393 id.account.getJid().asBareJid()
2394 + ": received rejectCall() when session has already been terminated. nothing to do");
2395 return;
2396 }
2397 switch (this.state) {
2398 case PROPOSED -> rejectCallFromProposed();
2399 case SESSION_INITIALIZED -> rejectCallFromSessionInitiate();
2400 default -> throw new IllegalStateException("Can not reject call from " + this.state);
2401 }
2402 }
2403
2404 public synchronized void endCall() {
2405 if (isTerminated()) {
2406 Log.w(
2407 Config.LOGTAG,
2408 id.account.getJid().asBareJid()
2409 + ": received endCall() when session has already been terminated. nothing to do");
2410 return;
2411 }
2412 if (isInState(State.PROPOSED) && !isInitiator()) {
2413 rejectCallFromProposed();
2414 return;
2415 }
2416 if (isInState(State.PROCEED)) {
2417 if (isInitiator()) {
2418 retractFromProceed();
2419 } else {
2420 rejectCallFromProceed();
2421 }
2422 return;
2423 }
2424 if (isInitiator()
2425 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2426 this.webRTCWrapper.close();
2427 sendSessionTerminate(Reason.CANCEL);
2428 return;
2429 }
2430 if (isInState(State.SESSION_INITIALIZED)) {
2431 rejectCallFromSessionInitiate();
2432 return;
2433 }
2434 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2435 this.webRTCWrapper.close();
2436 sendSessionTerminate(Reason.SUCCESS);
2437 return;
2438 }
2439 if (isInState(
2440 State.TERMINATED_APPLICATION_FAILURE,
2441 State.TERMINATED_CONNECTIVITY_ERROR,
2442 State.TERMINATED_DECLINED_OR_BUSY)) {
2443 Log.d(
2444 Config.LOGTAG,
2445 "ignoring request to end call because already in state " + this.state);
2446 return;
2447 }
2448 throw new IllegalStateException(
2449 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2450 }
2451
2452 private void retractFromProceed() {
2453 Log.d(Config.LOGTAG, "retract from proceed");
2454 this.sendJingleMessage("retract");
2455 closeTransitionLogFinish(State.RETRACTED_RACED);
2456 }
2457
2458 private void closeTransitionLogFinish(final State state) {
2459 this.webRTCWrapper.close();
2460 transitionOrThrow(state);
2461 writeLogMessage(state);
2462 finish();
2463 }
2464
2465 private void setupWebRTC(
2466 final Set<Media> media,
2467 final List<PeerConnection.IceServer> iceServers,
2468 final boolean trickle)
2469 throws WebRTCWrapper.InitializationException {
2470 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2471 this.webRTCWrapper.setup(
2472 this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2473 this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
2474 }
2475
2476 private void acceptCallFromProposed() {
2477 transitionOrThrow(State.PROCEED);
2478 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2479 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2480 this.sendJingleMessage("proceed");
2481 }
2482
2483 private void rejectCallFromProposed() {
2484 transitionOrThrow(State.REJECTED);
2485 writeLogMessageMissed();
2486 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2487 this.sendJingleMessage("reject");
2488 finish();
2489 }
2490
2491 private void rejectCallFromProceed() {
2492 this.sendJingleMessage("reject");
2493 closeTransitionLogFinish(State.REJECTED_RACED);
2494 }
2495
2496 private void rejectCallFromSessionInitiate() {
2497 webRTCWrapper.close();
2498 sendSessionTerminate(Reason.DECLINE);
2499 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2500 }
2501
2502 private void sendJingleMessage(final String action) {
2503 sendJingleMessage(action, id.with);
2504 }
2505
2506 private void sendJingleMessage(final String action, final Jid to) {
2507 final MessagePacket messagePacket = new MessagePacket();
2508 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2509 messagePacket.setTo(to);
2510 final Element intent =
2511 messagePacket
2512 .addChild(action, Namespace.JINGLE_MESSAGE)
2513 .setAttribute("id", id.sessionId);
2514 if ("proceed".equals(action)) {
2515 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2516 if (isOmemoEnabled()) {
2517 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2518 final Element device =
2519 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2520 device.setAttribute("id", deviceId);
2521 }
2522 }
2523 messagePacket.addChild("store", "urn:xmpp:hints");
2524 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2525 }
2526
2527 private boolean isOmemoEnabled() {
2528 final Conversational conversational = message.getConversation();
2529 if (conversational instanceof Conversation) {
2530 return ((Conversation) conversational).getNextEncryption()
2531 == Message.ENCRYPTION_AXOLOTL;
2532 }
2533 return false;
2534 }
2535
2536 private void acceptCallFromSessionInitialized() {
2537 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2538 sendSessionAccept();
2539 }
2540
2541 private synchronized boolean isInState(State... state) {
2542 return Arrays.asList(state).contains(this.state);
2543 }
2544
2545 private boolean transition(final State target) {
2546 return transition(target, null);
2547 }
2548
2549 private synchronized boolean transition(final State target, final Runnable runnable) {
2550 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2551 if (validTransitions != null && validTransitions.contains(target)) {
2552 this.state = target;
2553 if (runnable != null) {
2554 runnable.run();
2555 }
2556 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2557 updateEndUserState();
2558 updateOngoingCallNotification();
2559 return true;
2560 } else {
2561 return false;
2562 }
2563 }
2564
2565 void transitionOrThrow(final State target) {
2566 if (!transition(target)) {
2567 throw new IllegalStateException(
2568 String.format("Unable to transition from %s to %s", this.state, target));
2569 }
2570 }
2571
2572 @Override
2573 public void onIceCandidate(final IceCandidate iceCandidate) {
2574 final RtpContentMap rtpContentMap =
2575 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2576 final IceUdpTransportInfo.Credentials credentials;
2577 try {
2578 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2579 } catch (final IllegalArgumentException e) {
2580 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2581 return;
2582 }
2583 final String uFrag = credentials.ufrag;
2584 final IceUdpTransportInfo.Candidate candidate =
2585 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2586 if (candidate == null) {
2587 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2588 return;
2589 }
2590 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2591 sendTransportInfo(iceCandidate.sdpMid, candidate);
2592 }
2593
2594 @Override
2595 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2596 Log.d(
2597 Config.LOGTAG,
2598 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2599 this.stateHistory.add(newState);
2600 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2601 this.sessionDuration.start();
2602 updateOngoingCallNotification();
2603 } else if (this.sessionDuration.isRunning()) {
2604 this.sessionDuration.stop();
2605 updateOngoingCallNotification();
2606 }
2607
2608 final boolean neverConnected =
2609 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2610
2611 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2612 if (neverConnected) {
2613 if (isTerminated()) {
2614 Log.d(
2615 Config.LOGTAG,
2616 id.account.getJid().asBareJid()
2617 + ": not sending session-terminate after connectivity error because session is already in state "
2618 + this.state);
2619 return;
2620 }
2621 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2622 return;
2623 } else {
2624 this.restartIce();
2625 }
2626 }
2627 updateEndUserState();
2628 }
2629
2630 private void restartIce() {
2631 this.stateHistory.clear();
2632 this.webRTCWrapper.restartIceAsync();
2633 }
2634
2635 @Override
2636 public void onRenegotiationNeeded() {
2637 this.webRTCWrapper.execute(this::renegotiate);
2638 }
2639
2640 private void renegotiate() {
2641 final SessionDescription sessionDescription;
2642 try {
2643 sessionDescription = setLocalSessionDescription();
2644 } catch (final Exception e) {
2645 final Throwable cause = Throwables.getRootCause(e);
2646 webRTCWrapper.close();
2647 if (isTerminated()) {
2648 Log.d(
2649 Config.LOGTAG,
2650 "failed to renegotiate. session was already terminated",
2651 cause);
2652 return;
2653 }
2654 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2655 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2656 return;
2657 }
2658 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2659 final RtpContentMap currentContentMap = getLocalContentMap();
2660 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2661 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2662
2663 Log.d(
2664 Config.LOGTAG,
2665 id.getAccount().getJid().asBareJid()
2666 + ": renegotiate. iceRestart="
2667 + iceRestart
2668 + " content id diff="
2669 + diff);
2670
2671 if (diff.hasModifications() && iceRestart) {
2672 webRTCWrapper.close();
2673 sendSessionTerminate(
2674 Reason.FAILED_APPLICATION,
2675 "WebRTC unexpectedly tried to modify content and transport at once");
2676 return;
2677 }
2678
2679 if (iceRestart) {
2680 initiateIceRestart(rtpContentMap);
2681 return;
2682 } else if (diff.isEmpty()) {
2683 Log.d(
2684 Config.LOGTAG,
2685 "renegotiation. nothing to do. SignalingState="
2686 + this.webRTCWrapper.getSignalingState());
2687 }
2688
2689 if (diff.added.size() > 0) {
2690 modifyLocalContentMap(rtpContentMap);
2691 sendContentAdd(rtpContentMap, diff.added);
2692 }
2693 }
2694
2695 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2696 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2697 final JinglePacket jinglePacket =
2698 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2699 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2700 jinglePacket.setTo(id.with);
2701 xmppConnectionService.sendIqPacket(
2702 id.account,
2703 jinglePacket,
2704 (account, response) -> {
2705 if (response.getType() == IqPacket.TYPE.RESULT) {
2706 Log.d(Config.LOGTAG, "received success to our ice restart");
2707 setLocalContentMap(rtpContentMap);
2708 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2709 return;
2710 }
2711 if (response.getType() == IqPacket.TYPE.ERROR) {
2712 if (isTieBreak(response)) {
2713 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2714 return;
2715 }
2716 handleIqErrorResponse(response);
2717 }
2718 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2719 handleIqTimeoutResponse(response);
2720 }
2721 });
2722 }
2723
2724 private boolean isTieBreak(final IqPacket response) {
2725 final Element error = response.findChild("error");
2726 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2727 }
2728
2729 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2730 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2731 this.outgoingContentAdd = contentAdd;
2732 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2733 prepareOutgoingContentMap(contentAdd);
2734 Futures.addCallback(
2735 outgoingContentMapFuture,
2736 new FutureCallback<>() {
2737 @Override
2738 public void onSuccess(final RtpContentMap outgoingContentMap) {
2739 sendContentAdd(outgoingContentMap);
2740 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2741 }
2742
2743 @Override
2744 public void onFailure(@NonNull Throwable throwable) {
2745 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2746 }
2747 },
2748 MoreExecutors.directExecutor());
2749 }
2750
2751 private void sendContentAdd(final RtpContentMap contentAdd) {
2752
2753 final JinglePacket jinglePacket =
2754 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2755 jinglePacket.setTo(id.with);
2756 xmppConnectionService.sendIqPacket(
2757 id.account,
2758 jinglePacket,
2759 (connection, response) -> {
2760 if (response.getType() == IqPacket.TYPE.RESULT) {
2761 Log.d(
2762 Config.LOGTAG,
2763 id.getAccount().getJid().asBareJid()
2764 + ": received ACK to our content-add");
2765 return;
2766 }
2767 if (response.getType() == IqPacket.TYPE.ERROR) {
2768 if (isTieBreak(response)) {
2769 this.outgoingContentAdd = null;
2770 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2771 return;
2772 }
2773 handleIqErrorResponse(response);
2774 }
2775 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2776 handleIqTimeoutResponse(response);
2777 }
2778 });
2779 }
2780
2781 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2782 if (isInitiator()) {
2783 this.initiatorRtpContentMap = rtpContentMap;
2784 } else {
2785 this.responderRtpContentMap = rtpContentMap;
2786 }
2787 }
2788
2789 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2790 if (isInitiator()) {
2791 this.responderRtpContentMap = rtpContentMap;
2792 } else {
2793 this.initiatorRtpContentMap = rtpContentMap;
2794 }
2795 }
2796
2797 // this method is to be used for content map modifications that modify media
2798 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2799 final RtpContentMap activeContents = rtpContentMap.activeContents();
2800 setLocalContentMap(activeContents);
2801 this.webRTCWrapper.switchSpeakerPhonePreference(
2802 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2803 updateEndUserState();
2804 }
2805
2806 private SessionDescription setLocalSessionDescription()
2807 throws ExecutionException, InterruptedException {
2808 final org.webrtc.SessionDescription sessionDescription =
2809 this.webRTCWrapper.setLocalDescription(false).get();
2810 return SessionDescription.parse(sessionDescription.description);
2811 }
2812
2813 private void closeWebRTCSessionAfterFailedConnection() {
2814 this.webRTCWrapper.close();
2815 synchronized (this) {
2816 if (isTerminated()) {
2817 Log.d(
2818 Config.LOGTAG,
2819 id.account.getJid().asBareJid()
2820 + ": no need to send session-terminate after failed connection. Other party already did");
2821 return;
2822 }
2823 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2824 }
2825 }
2826
2827 public boolean zeroDuration() {
2828 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2829 }
2830
2831 public long getCallDuration() {
2832 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2833 }
2834
2835 public AppRTCAudioManager getAudioManager() {
2836 return webRTCWrapper.getAudioManager();
2837 }
2838
2839 public boolean isMicrophoneEnabled() {
2840 return webRTCWrapper.isMicrophoneEnabled();
2841 }
2842
2843 public boolean setMicrophoneEnabled(final boolean enabled) {
2844 return webRTCWrapper.setMicrophoneEnabled(enabled);
2845 }
2846
2847 public boolean isVideoEnabled() {
2848 return webRTCWrapper.isVideoEnabled();
2849 }
2850
2851 public void setVideoEnabled(final boolean enabled) {
2852 webRTCWrapper.setVideoEnabled(enabled);
2853 }
2854
2855 public boolean isCameraSwitchable() {
2856 return webRTCWrapper.isCameraSwitchable();
2857 }
2858
2859 public boolean isFrontCamera() {
2860 return webRTCWrapper.isFrontCamera();
2861 }
2862
2863 public ListenableFuture<Boolean> switchCamera() {
2864 return webRTCWrapper.switchCamera();
2865 }
2866
2867 @Override
2868 public void onAudioDeviceChanged(
2869 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2870 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2871 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2872 selectedAudioDevice, availableAudioDevices);
2873 }
2874
2875 private void updateEndUserState() {
2876 final RtpEndUserState endUserState = getEndUserState();
2877 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2878 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2879 id.account, id.with, id.sessionId, endUserState);
2880 }
2881
2882 private void updateOngoingCallNotification() {
2883 final State state = this.state;
2884 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2885 final boolean reconnecting;
2886 if (state == State.SESSION_ACCEPTED) {
2887 reconnecting =
2888 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2889 } else {
2890 reconnecting = false;
2891 }
2892 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2893 } else {
2894 xmppConnectionService.removeOngoingCall();
2895 }
2896 }
2897
2898 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2899 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2900 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2901 request.setTo(id.account.getDomain());
2902 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2903 xmppConnectionService.sendIqPacket(
2904 id.account,
2905 request,
2906 (account, response) -> {
2907 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2908 new ImmutableList.Builder<>();
2909 if (response.getType() == IqPacket.TYPE.RESULT) {
2910 final Element services =
2911 response.findChild(
2912 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2913 final List<Element> children =
2914 services == null
2915 ? Collections.emptyList()
2916 : services.getChildren();
2917 for (final Element child : children) {
2918 if ("service".equals(child.getName())) {
2919 final String type = child.getAttribute("type");
2920 final String host = child.getAttribute("host");
2921 final String sport = child.getAttribute("port");
2922 final Integer port =
2923 sport == null ? null : Ints.tryParse(sport);
2924 final String transport = child.getAttribute("transport");
2925 final String username = child.getAttribute("username");
2926 final String password = child.getAttribute("password");
2927 if (Strings.isNullOrEmpty(host) || port == null) {
2928 continue;
2929 }
2930 if (port < 0 || port > 65535) {
2931 continue;
2932 }
2933
2934 if (Arrays.asList("stun", "stuns", "turn", "turns")
2935 .contains(type)
2936 && Arrays.asList("udp", "tcp").contains(transport)) {
2937 if (Arrays.asList("stuns", "turns").contains(type)
2938 && "udp".equals(transport)) {
2939 Log.d(
2940 Config.LOGTAG,
2941 id.account.getJid().asBareJid()
2942 + ": skipping invalid combination of udp/tls in external services");
2943 continue;
2944 }
2945
2946 // STUN URLs do not support a query section since M110
2947 final String uri;
2948 if (Arrays.asList("stun", "stuns").contains(type)) {
2949 uri =
2950 String.format(
2951 "%s:%s:%s",
2952 type, IP.wrapIPv6(host), port);
2953 } else {
2954 uri =
2955 String.format(
2956 "%s:%s:%s?transport=%s",
2957 type,
2958 IP.wrapIPv6(host),
2959 port,
2960 transport);
2961 }
2962
2963 final PeerConnection.IceServer.Builder iceServerBuilder =
2964 PeerConnection.IceServer.builder(uri);
2965 iceServerBuilder.setTlsCertPolicy(
2966 PeerConnection.TlsCertPolicy
2967 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2968 if (username != null && password != null) {
2969 iceServerBuilder.setUsername(username);
2970 iceServerBuilder.setPassword(password);
2971 } else if (Arrays.asList("turn", "turns").contains(type)) {
2972 // The WebRTC spec requires throwing an
2973 // InvalidAccessError when username (from libwebrtc
2974 // source coder)
2975 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2976 Log.d(
2977 Config.LOGTAG,
2978 id.account.getJid().asBareJid()
2979 + ": skipping "
2980 + type
2981 + "/"
2982 + transport
2983 + " without username and password");
2984 continue;
2985 }
2986 final PeerConnection.IceServer iceServer =
2987 iceServerBuilder.createIceServer();
2988 Log.d(
2989 Config.LOGTAG,
2990 id.account.getJid().asBareJid()
2991 + ": discovered ICE Server: "
2992 + iceServer);
2993 listBuilder.add(iceServer);
2994 }
2995 }
2996 }
2997 }
2998 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2999 if (iceServers.size() == 0) {
3000 Log.w(
3001 Config.LOGTAG,
3002 id.account.getJid().asBareJid()
3003 + ": no ICE server found "
3004 + response);
3005 }
3006 onIceServersDiscovered.onIceServersDiscovered(iceServers);
3007 });
3008 } else {
3009 Log.w(
3010 Config.LOGTAG,
3011 id.account.getJid().asBareJid() + ": has no external service discovery");
3012 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
3013 }
3014 }
3015
3016 private void finish() {
3017 if (isTerminated()) {
3018 this.cancelRingingTimeout();
3019 this.webRTCWrapper.verifyClosed();
3020 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
3021 this.jingleConnectionManager.finishConnectionOrThrow(this);
3022 try {
3023 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
3024 log.getParentFile().mkdirs();
3025 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
3026 } catch (final IOException e) { }
3027 } else {
3028 throw new IllegalStateException(
3029 String.format("Unable to call finish from %s", this.state));
3030 }
3031 }
3032
3033 private void writeLogMessage(final State state) {
3034 final long duration = getCallDuration();
3035 if (state == State.TERMINATED_SUCCESS
3036 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
3037 writeLogMessageSuccess(duration);
3038 } else {
3039 writeLogMessageMissed();
3040 }
3041 }
3042
3043 private void writeLogMessageSuccess(final long duration) {
3044 this.message.setBody(new RtpSessionStatus(true, duration).toString());
3045 this.writeMessage();
3046 }
3047
3048 private void writeLogMessageMissed() {
3049 this.message.setBody(new RtpSessionStatus(false, 0).toString());
3050 this.writeMessage();
3051 }
3052
3053 private void writeMessage() {
3054 final Conversational conversational = message.getConversation();
3055 if (conversational instanceof Conversation) {
3056 ((Conversation) conversational).add(this.message);
3057 xmppConnectionService.createMessageAsync(message);
3058 xmppConnectionService.updateConversationUi();
3059 } else {
3060 throw new IllegalStateException("Somehow the conversation in a message was a stub");
3061 }
3062 }
3063
3064 public State getState() {
3065 return this.state;
3066 }
3067
3068 boolean isTerminated() {
3069 return TERMINATED.contains(this.state);
3070 }
3071
3072 public Optional<VideoTrack> getLocalVideoTrack() {
3073 return webRTCWrapper.getLocalVideoTrack();
3074 }
3075
3076 public Optional<VideoTrack> getRemoteVideoTrack() {
3077 return webRTCWrapper.getRemoteVideoTrack();
3078 }
3079
3080 public EglBase.Context getEglBaseContext() {
3081 return webRTCWrapper.getEglBaseContext();
3082 }
3083
3084 void setProposedMedia(final Set<Media> media) {
3085 this.proposedMedia = media;
3086 }
3087
3088 public void fireStateUpdate() {
3089 final RtpEndUserState endUserState = getEndUserState();
3090 xmppConnectionService.notifyJingleRtpConnectionUpdate(
3091 id.account, id.with, id.sessionId, endUserState);
3092 }
3093
3094 public boolean isSwitchToVideoAvailable() {
3095 final boolean prerequisite =
3096 Media.audioOnly(getMedia())
3097 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
3098 .contains(getEndUserState());
3099 return prerequisite && remoteHasVideoFeature();
3100 }
3101
3102 private boolean remoteHasVideoFeature() {
3103 return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
3104 }
3105
3106 private boolean remoteHasSdpOfferAnswer() {
3107 return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
3108 }
3109
3110 private boolean remoteHasFeature(final String feature) {
3111 final Contact contact = id.getContact();
3112 final Presence presence =
3113 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
3114 final ServiceDiscoveryResult serviceDiscoveryResult =
3115 presence == null ? null : presence.getServiceDiscoveryResult();
3116 final List<String> features =
3117 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
3118 return features != null && features.contains(feature);
3119 }
3120
3121 private interface OnIceServersDiscovered {
3122 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
3123 }
3124}