1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Joiner;
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.base.Stopwatch;
12import com.google.common.base.Strings;
13import com.google.common.base.Throwables;
14import com.google.common.collect.Collections2;
15import com.google.common.collect.ImmutableList;
16import com.google.common.collect.ImmutableMap;
17import com.google.common.collect.ImmutableMultimap;
18import com.google.common.collect.ImmutableSet;
19import com.google.common.collect.Iterables;
20import com.google.common.collect.Maps;
21import com.google.common.collect.Multimap;
22import com.google.common.collect.Sets;
23import com.google.common.primitives.Ints;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import org.webrtc.EglBase;
30import org.webrtc.IceCandidate;
31import org.webrtc.PeerConnection;
32import org.webrtc.VideoTrack;
33
34import java.util.Arrays;
35import java.util.Collection;
36import java.util.Collections;
37import java.util.LinkedList;
38import java.util.List;
39import java.util.Map;
40import java.util.Queue;
41import java.util.Set;
42import java.util.concurrent.ExecutionException;
43import java.util.concurrent.ScheduledFuture;
44import java.util.concurrent.TimeUnit;
45
46import eu.siacs.conversations.BuildConfig;
47import eu.siacs.conversations.Config;
48import eu.siacs.conversations.crypto.axolotl.AxolotlService;
49import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
50import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
51import eu.siacs.conversations.entities.Account;
52import eu.siacs.conversations.entities.Contact;
53import eu.siacs.conversations.entities.Conversation;
54import eu.siacs.conversations.entities.Conversational;
55import eu.siacs.conversations.entities.Message;
56import eu.siacs.conversations.entities.Presence;
57import eu.siacs.conversations.entities.RtpSessionStatus;
58import eu.siacs.conversations.entities.ServiceDiscoveryResult;
59import eu.siacs.conversations.services.AppRTCAudioManager;
60import eu.siacs.conversations.utils.IP;
61import eu.siacs.conversations.xml.Element;
62import eu.siacs.conversations.xml.Namespace;
63import eu.siacs.conversations.xmpp.Jid;
64import eu.siacs.conversations.xmpp.OnIqPacketReceived;
65import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
66import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
67import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
68import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
69import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
70import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
71import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
72import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
73import eu.siacs.conversations.xmpp.stanzas.IqPacket;
74import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
75
76public class JingleRtpConnection extends AbstractJingleConnection
77 implements WebRTCWrapper.EventCallback {
78
79 public static final List<State> STATES_SHOWING_ONGOING_CALL =
80 Arrays.asList(
81 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
82 private static final long BUSY_TIME_OUT = 30;
83 private static final List<State> TERMINATED =
84 Arrays.asList(
85 State.ACCEPTED,
86 State.REJECTED,
87 State.REJECTED_RACED,
88 State.RETRACTED,
89 State.RETRACTED_RACED,
90 State.TERMINATED_SUCCESS,
91 State.TERMINATED_DECLINED_OR_BUSY,
92 State.TERMINATED_CONNECTIVITY_ERROR,
93 State.TERMINATED_CANCEL_OR_TIMEOUT,
94 State.TERMINATED_APPLICATION_FAILURE,
95 State.TERMINATED_SECURITY_ERROR);
96
97 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
98
99 static {
100 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
101 new ImmutableMap.Builder<>();
102 transitionBuilder.put(
103 State.NULL,
104 ImmutableList.of(
105 State.PROPOSED,
106 State.SESSION_INITIALIZED,
107 State.TERMINATED_APPLICATION_FAILURE,
108 State.TERMINATED_SECURITY_ERROR));
109 transitionBuilder.put(
110 State.PROPOSED,
111 ImmutableList.of(
112 State.ACCEPTED,
113 State.PROCEED,
114 State.REJECTED,
115 State.RETRACTED,
116 State.TERMINATED_APPLICATION_FAILURE,
117 State.TERMINATED_SECURITY_ERROR,
118 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
119 // rebinds
120 ));
121 transitionBuilder.put(
122 State.PROCEED,
123 ImmutableList.of(
124 State.REJECTED_RACED,
125 State.RETRACTED_RACED,
126 State.SESSION_INITIALIZED_PRE_APPROVED,
127 State.TERMINATED_SUCCESS,
128 State.TERMINATED_APPLICATION_FAILURE,
129 State.TERMINATED_SECURITY_ERROR,
130 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
131 // bounces of the proceed message
132 ));
133 transitionBuilder.put(
134 State.SESSION_INITIALIZED,
135 ImmutableList.of(
136 State.SESSION_ACCEPTED,
137 State.TERMINATED_SUCCESS,
138 State.TERMINATED_DECLINED_OR_BUSY,
139 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
140 // and IQ timeouts
141 State.TERMINATED_CANCEL_OR_TIMEOUT,
142 State.TERMINATED_APPLICATION_FAILURE,
143 State.TERMINATED_SECURITY_ERROR));
144 transitionBuilder.put(
145 State.SESSION_INITIALIZED_PRE_APPROVED,
146 ImmutableList.of(
147 State.SESSION_ACCEPTED,
148 State.TERMINATED_SUCCESS,
149 State.TERMINATED_DECLINED_OR_BUSY,
150 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
151 // and IQ timeouts
152 State.TERMINATED_CANCEL_OR_TIMEOUT,
153 State.TERMINATED_APPLICATION_FAILURE,
154 State.TERMINATED_SECURITY_ERROR));
155 transitionBuilder.put(
156 State.SESSION_ACCEPTED,
157 ImmutableList.of(
158 State.TERMINATED_SUCCESS,
159 State.TERMINATED_DECLINED_OR_BUSY,
160 State.TERMINATED_CONNECTIVITY_ERROR,
161 State.TERMINATED_CANCEL_OR_TIMEOUT,
162 State.TERMINATED_APPLICATION_FAILURE,
163 State.TERMINATED_SECURITY_ERROR));
164 VALID_TRANSITIONS = transitionBuilder.build();
165 }
166
167 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
168 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
169 pendingIceCandidates = new LinkedList<>();
170 private final OmemoVerification omemoVerification = new OmemoVerification();
171 private final Message message;
172 private State state = State.NULL;
173 private Set<Media> proposedMedia;
174 private RtpContentMap initiatorRtpContentMap;
175 private RtpContentMap responderRtpContentMap;
176 private RtpContentMap incomingContentAdd;
177 private RtpContentMap outgoingContentAdd;
178 private IceUdpTransportInfo.Setup peerDtlsSetup;
179 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
180 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
181 private ScheduledFuture<?> ringingTimeoutFuture;
182
183 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
184 super(jingleConnectionManager, id, initiator);
185 final Conversation conversation =
186 jingleConnectionManager
187 .getXmppConnectionService()
188 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
189 this.message =
190 new Message(
191 conversation,
192 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
193 Message.TYPE_RTP_SESSION,
194 id.sessionId);
195 }
196
197 private static State reasonToState(Reason reason) {
198 switch (reason) {
199 case SUCCESS:
200 return State.TERMINATED_SUCCESS;
201 case DECLINE:
202 case BUSY:
203 return State.TERMINATED_DECLINED_OR_BUSY;
204 case CANCEL:
205 case TIMEOUT:
206 return State.TERMINATED_CANCEL_OR_TIMEOUT;
207 case SECURITY_ERROR:
208 return State.TERMINATED_SECURITY_ERROR;
209 case FAILED_APPLICATION:
210 case UNSUPPORTED_TRANSPORTS:
211 case UNSUPPORTED_APPLICATIONS:
212 return State.TERMINATED_APPLICATION_FAILURE;
213 default:
214 return State.TERMINATED_CONNECTIVITY_ERROR;
215 }
216 }
217
218 @Override
219 synchronized void deliverPacket(final JinglePacket jinglePacket) {
220 switch (jinglePacket.getAction()) {
221 case SESSION_INITIATE:
222 receiveSessionInitiate(jinglePacket);
223 break;
224 case TRANSPORT_INFO:
225 receiveTransportInfo(jinglePacket);
226 break;
227 case SESSION_ACCEPT:
228 receiveSessionAccept(jinglePacket);
229 break;
230 case SESSION_TERMINATE:
231 receiveSessionTerminate(jinglePacket);
232 break;
233 case CONTENT_ADD:
234 receiveContentAdd(jinglePacket);
235 break;
236 case CONTENT_ACCEPT:
237 receiveContentAccept(jinglePacket);
238 break;
239 case CONTENT_REJECT:
240 receiveContentReject(jinglePacket);
241 break;
242 case CONTENT_REMOVE:
243 receiveContentRemove(jinglePacket);
244 break;
245 case CONTENT_MODIFY:
246 receiveContentModify(jinglePacket);
247 break;
248 default:
249 respondOk(jinglePacket);
250 Log.d(
251 Config.LOGTAG,
252 String.format(
253 "%s: received unhandled jingle action %s",
254 id.account.getJid().asBareJid(), jinglePacket.getAction()));
255 break;
256 }
257 }
258
259 @Override
260 synchronized void notifyRebound() {
261 if (isTerminated()) {
262 return;
263 }
264 webRTCWrapper.close();
265 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
266 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
267 }
268 if (isInState(
269 State.SESSION_INITIALIZED,
270 State.SESSION_INITIALIZED_PRE_APPROVED,
271 State.SESSION_ACCEPTED)) {
272 // we might have already changed resources (full jid) at this point; so this might not
273 // even reach the other party
274 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
275 } else {
276 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
277 finish();
278 }
279 }
280
281 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
282 respondOk(jinglePacket);
283 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
284 final State previous = this.state;
285 Log.d(
286 Config.LOGTAG,
287 id.account.getJid().asBareJid()
288 + ": received session terminate reason="
289 + wrapper.reason
290 + "("
291 + Strings.nullToEmpty(wrapper.text)
292 + ") while in state "
293 + previous);
294 if (TERMINATED.contains(previous)) {
295 Log.d(
296 Config.LOGTAG,
297 id.account.getJid().asBareJid()
298 + ": ignoring session terminate because already in "
299 + previous);
300 return;
301 }
302 webRTCWrapper.close();
303 final State target = reasonToState(wrapper.reason);
304 transitionOrThrow(target);
305 writeLogMessage(target);
306 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
307 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
308 }
309 finish();
310 }
311
312 private void receiveTransportInfo(final JinglePacket jinglePacket) {
313 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
314 // INITIALIZED only after transport-info has been received
315 if (isInState(
316 State.NULL,
317 State.PROCEED,
318 State.SESSION_INITIALIZED,
319 State.SESSION_INITIALIZED_PRE_APPROVED,
320 State.SESSION_ACCEPTED)) {
321 final RtpContentMap contentMap;
322 try {
323 contentMap = RtpContentMap.of(jinglePacket);
324 } catch (final IllegalArgumentException | NullPointerException e) {
325 Log.d(
326 Config.LOGTAG,
327 id.account.getJid().asBareJid()
328 + ": improperly formatted contents; ignoring",
329 e);
330 respondOk(jinglePacket);
331 return;
332 }
333 receiveTransportInfo(jinglePacket, contentMap);
334 } else {
335 if (isTerminated()) {
336 respondOk(jinglePacket);
337 Log.d(
338 Config.LOGTAG,
339 id.account.getJid().asBareJid()
340 + ": ignoring out-of-order transport info; we where already terminated");
341 } else {
342 Log.d(
343 Config.LOGTAG,
344 id.account.getJid().asBareJid()
345 + ": received transport info while in state="
346 + this.state);
347 terminateWithOutOfOrder(jinglePacket);
348 }
349 }
350 }
351
352 private void receiveTransportInfo(
353 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
354 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
355 contentMap.contents.entrySet();
356 final RtpContentMap remote = getRemoteContentMap();
357 final Set<String> remoteContentIds = remote == null ? Collections.emptySet() : remote.contents.keySet();
358 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
359 Log.d(Config.LOGTAG,"received transport-info for unknown contents "+contentMap.contents.keySet()+" (known: "+remoteContentIds+")");
360 respondOk(jinglePacket);
361 pendingIceCandidates.addAll(candidates);
362 return;
363 }
364 if (this.state != State.SESSION_ACCEPTED) {
365 Log.d(Config.LOGTAG,"received transport-info prematurely. adding to backlog");
366 respondOk(jinglePacket);
367 pendingIceCandidates.addAll(candidates);
368 return;
369 }
370 // zero candidates + modified credentials are an ICE restart offer
371 if (checkForIceRestart(jinglePacket, contentMap)) {
372 return;
373 }
374 respondOk(jinglePacket);
375 try {
376 processCandidates(candidates);
377 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
378 Log.w(
379 Config.LOGTAG,
380 id.account.getJid().asBareJid()
381 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
382 }
383 }
384
385 private void receiveContentAdd(final JinglePacket jinglePacket) {
386 final RtpContentMap modification;
387 try {
388 modification = RtpContentMap.of(jinglePacket);
389 modification.requireContentDescriptions();
390 } catch (final RuntimeException e) {
391 Log.d(
392 Config.LOGTAG,
393 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
394 Throwables.getRootCause(e));
395 respondOk(jinglePacket);
396 webRTCWrapper.close();
397 sendSessionTerminate(Reason.of(e), e.getMessage());
398 return;
399 }
400 if (isInState(State.SESSION_ACCEPTED)) {
401 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
402 final ListenableFuture<RtpContentMap> future =
403 receiveRtpContentMap(
404 modification, this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
405 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
406 @Override
407 public void onSuccess(final RtpContentMap rtpContentMap) {
408 receiveContentAdd(jinglePacket, rtpContentMap);
409 }
410
411 @Override
412 public void onFailure(@NonNull Throwable throwable) {
413 respondOk(jinglePacket);
414 final Throwable rootCause = Throwables.getRootCause(throwable);
415 Log.d(
416 Config.LOGTAG,
417 id.account.getJid().asBareJid()
418 + ": improperly formatted contents in content-add",
419 throwable);
420 webRTCWrapper.close();
421 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
422 }
423 }, MoreExecutors.directExecutor());
424 } else {
425 terminateWithOutOfOrder(jinglePacket);
426 }
427 }
428
429 private void receiveContentAdd(
430 final JinglePacket jinglePacket, final RtpContentMap modification) {
431 final RtpContentMap remote = getRemoteContentMap();
432 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
433 respondOk(jinglePacket);
434 this.webRTCWrapper.close();
435 sendSessionTerminate(
436 Reason.FAILED_APPLICATION,
437 String.format(
438 "contents with names %s already exists",
439 Joiner.on(", ").join(modification.getNames())));
440 return;
441 }
442 final ContentAddition contentAddition =
443 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
444
445 final RtpContentMap outgoing = this.outgoingContentAdd;
446 final Set<ContentAddition.Summary> outgoingContentAddSummary =
447 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
448
449 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
450 if (isInitiator()) {
451 Log.d(
452 Config.LOGTAG,
453 id.getAccount().getJid().asBareJid()
454 + ": respond with tie break to matching content-add offer");
455 respondWithTieBreak(jinglePacket);
456 } else {
457 Log.d(
458 Config.LOGTAG,
459 id.getAccount().getJid().asBareJid()
460 + ": automatically accept matching content-add offer");
461 acceptContentAdd(contentAddition.summary, modification);
462 }
463 return;
464 }
465
466 // once we can display multiple video tracks we can be more loose with this condition
467 // theoretically it should also be fine to automatically accept audio only contents
468 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
469 Log.d(
470 Config.LOGTAG,
471 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
472 this.incomingContentAdd = modification;
473 respondOk(jinglePacket);
474 updateEndUserState();
475 } else {
476 respondOk(jinglePacket);
477 // TODO do we want to add a reason?
478 rejectContentAdd(modification);
479 }
480 }
481
482 private void receiveContentAccept(final JinglePacket jinglePacket) {
483 final RtpContentMap receivedContentAccept;
484 try {
485 receivedContentAccept = RtpContentMap.of(jinglePacket);
486 receivedContentAccept.requireContentDescriptions();
487 } catch (final RuntimeException e) {
488 Log.d(
489 Config.LOGTAG,
490 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
491 Throwables.getRootCause(e));
492 respondOk(jinglePacket);
493 webRTCWrapper.close();
494 sendSessionTerminate(Reason.of(e), e.getMessage());
495 return;
496 }
497
498 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
499 if (outgoingContentAdd == null) {
500 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
501 terminateWithOutOfOrder(jinglePacket);
502 return;
503 }
504 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
505 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
506 this.outgoingContentAdd = null;
507 respondOk(jinglePacket);
508 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
509 final ListenableFuture<RtpContentMap> future =
510 receiveRtpContentMap(
511 receivedContentAccept, this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
512 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
513 @Override
514 public void onSuccess(final RtpContentMap result) {
515 receiveContentAccept(result);
516 }
517
518 @Override
519 public void onFailure(@NonNull final Throwable throwable) {
520 webRTCWrapper.close();
521 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
522 }
523 }, MoreExecutors.directExecutor());
524 } else {
525 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
526 terminateWithOutOfOrder(jinglePacket);
527 }
528 }
529
530 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
531 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
532 final RtpContentMap modifiedContentMap =
533 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
534
535 setRemoteContentMap(modifiedContentMap);
536
537 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
538
539 final org.webrtc.SessionDescription sdp =
540 new org.webrtc.SessionDescription(
541 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
542
543 try {
544 this.webRTCWrapper.setRemoteDescription(sdp).get();
545 } catch (final Exception e) {
546 final Throwable cause = Throwables.getRootCause(e);
547 Log.d(
548 Config.LOGTAG,
549 id.getAccount().getJid().asBareJid()
550 + ": unable to set remote description after receiving content-accept",
551 cause);
552 webRTCWrapper.close();
553 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
554 return;
555 }
556 Log.d(
557 Config.LOGTAG,
558 id.getAccount().getJid().asBareJid()
559 + ": remote has accepted content-add "
560 + ContentAddition.summary(receivedContentAccept));
561 processCandidates(receivedContentAccept.contents.entrySet());
562 updateEndUserState();
563 }
564
565 private void receiveContentModify(final JinglePacket jinglePacket) {
566 if (this.state != State.SESSION_ACCEPTED) {
567 terminateWithOutOfOrder(jinglePacket);
568 return;
569 }
570 final Map<String, Content.Senders> modification =
571 Maps.transformEntries(
572 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
573 final boolean isInitiator = isInitiator();
574 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
575 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
576 final Set<String> currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet();
577 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
578 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
579 respondOk(jinglePacket);
580 final RtpContentMap modifiedContentMap;
581 try {
582 modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification);
583 } catch (final IllegalArgumentException e) {
584 webRTCWrapper.close();
585 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
586 return;
587 }
588 this.outgoingContentAdd = modifiedContentMap;
589 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add");
590 } else if (remoteContentMap != null && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
591 respondOk(jinglePacket);
592 final RtpContentMap modifiedRemoteContentMap;
593 try {
594 modifiedRemoteContentMap = remoteContentMap.modifiedSendersChecked(isInitiator, modification);
595 } catch (final IllegalArgumentException e) {
596 webRTCWrapper.close();
597 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
598 return;
599 }
600 final SessionDescription offer;
601 try {
602 offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator());
603 } catch (final IllegalArgumentException | NullPointerException e) {
604 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-modify to SDP", e);
605 webRTCWrapper.close();
606 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
607 return;
608 }
609 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": auto accepting content-modification");
610 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
611 } else {
612 Log.d(Config.LOGTAG,"received unsupported content modification "+modification);
613 respondWithItemNotFound(jinglePacket);
614 }
615 }
616
617 private void autoAcceptContentModify(final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
618 this.setRemoteContentMap(modifiedRemoteContentMap);
619 final org.webrtc.SessionDescription sdp =
620 new org.webrtc.SessionDescription(
621 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
622 try {
623 this.webRTCWrapper.setRemoteDescription(sdp).get();
624 // auto accept is only done when we already have tracks
625 final SessionDescription answer = setLocalSessionDescription();
626 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
627 modifyLocalContentMap(rtpContentMap);
628 // we do not need to send an answer but do we have to resend the candidates currently in SDP?
629 //resendCandidatesFromSdp(answer);
630 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
631 } catch (final Exception e) {
632 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
633 webRTCWrapper.close();
634 sendSessionTerminate(Reason.FAILED_APPLICATION);
635 }
636 }
637
638 private void resendCandidatesFromSdp(final SessionDescription answer) {
639 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
640 for(final SessionDescription.Media media : answer.media) {
641 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
642 if (Strings.isNullOrEmpty(mid)) {
643 continue;
644 }
645 for(final String sdpCandidate : media.attributes.get("candidate")) {
646 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
647 if (candidate != null) {
648 candidateBuilder.put(mid,candidate);
649 }
650 }
651 }
652 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build();
653 sendTransportInfo(candidates);
654 }
655
656 private void receiveContentReject(final JinglePacket jinglePacket) {
657 final RtpContentMap receivedContentReject;
658 try {
659 receivedContentReject = RtpContentMap.of(jinglePacket);
660 } catch (final RuntimeException e) {
661 Log.d(
662 Config.LOGTAG,
663 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
664 Throwables.getRootCause(e));
665 respondOk(jinglePacket);
666 this.webRTCWrapper.close();
667 sendSessionTerminate(Reason.of(e), e.getMessage());
668 return;
669 }
670
671 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
672 if (outgoingContentAdd == null) {
673 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
674 terminateWithOutOfOrder(jinglePacket);
675 return;
676 }
677 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
678 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
679 this.outgoingContentAdd = null;
680 respondOk(jinglePacket);
681 Log.d(Config.LOGTAG,jinglePacket.toString());
682 receiveContentReject(ourSummary);
683 } else {
684 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
685 terminateWithOutOfOrder(jinglePacket);
686 }
687 }
688
689 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
690 try {
691 this.webRTCWrapper.removeTrack(Media.VIDEO);
692 final RtpContentMap localContentMap = customRollback();
693 modifyLocalContentMap(localContentMap);
694 } catch (final Exception e) {
695 final Throwable cause = Throwables.getRootCause(e);
696 Log.d(
697 Config.LOGTAG,
698 id.getAccount().getJid().asBareJid()
699 + ": unable to rollback local description after receiving content-reject",
700 cause);
701 webRTCWrapper.close();
702 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
703 return;
704 }
705 Log.d(
706 Config.LOGTAG,
707 id.getAccount().getJid().asBareJid()
708 + ": remote has rejected our content-add "
709 + summary);
710 }
711
712 private void receiveContentRemove(final JinglePacket jinglePacket) {
713 final RtpContentMap receivedContentRemove;
714 try {
715 receivedContentRemove = RtpContentMap.of(jinglePacket);
716 receivedContentRemove.requireContentDescriptions();
717 } catch (final RuntimeException e) {
718 Log.d(
719 Config.LOGTAG,
720 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
721 Throwables.getRootCause(e));
722 respondOk(jinglePacket);
723 this.webRTCWrapper.close();
724 sendSessionTerminate(Reason.of(e), e.getMessage());
725 return;
726 }
727 respondOk(jinglePacket);
728 receiveContentRemove(receivedContentRemove);
729 }
730
731 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
732 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
733 final Set<ContentAddition.Summary> contentAddSummary =
734 incomingContentAdd == null
735 ? Collections.emptySet()
736 : ContentAddition.summary(incomingContentAdd);
737 final Set<ContentAddition.Summary> removeSummary =
738 ContentAddition.summary(receivedContentRemove);
739 if (contentAddSummary.equals(removeSummary)) {
740 this.incomingContentAdd = null;
741 updateEndUserState();
742 } else {
743 webRTCWrapper.close();
744 sendSessionTerminate(
745 Reason.FAILED_APPLICATION,
746 String.format(
747 "%s only supports %s as a means to retract a not yet accepted %s",
748 BuildConfig.APP_NAME,
749 JinglePacket.Action.CONTENT_REMOVE,
750 JinglePacket.Action.CONTENT_ADD));
751 }
752 }
753
754 public synchronized void retractContentAdd() {
755 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
756 if (outgoingContentAdd == null) {
757 throw new IllegalStateException("Not outgoing content add");
758 }
759 try {
760 webRTCWrapper.removeTrack(Media.VIDEO);
761 final RtpContentMap localContentMap = customRollback();
762 modifyLocalContentMap(localContentMap);
763 } catch (final Exception e) {
764 final Throwable cause = Throwables.getRootCause(e);
765 Log.d(
766 Config.LOGTAG,
767 id.getAccount().getJid().asBareJid()
768 + ": unable to rollback local description after trying to retract content-add",
769 cause);
770 webRTCWrapper.close();
771 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
772 return;
773 }
774 this.outgoingContentAdd = null;
775 final JinglePacket retract =
776 outgoingContentAdd
777 .toStub()
778 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
779 this.send(retract);
780 Log.d(
781 Config.LOGTAG,
782 id.getAccount().getJid()
783 + ": retract content-add "
784 + ContentAddition.summary(outgoingContentAdd));
785 }
786
787 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
788 final SessionDescription sdp = setLocalSessionDescription();
789 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
790 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
791 this.webRTCWrapper
792 .setRemoteDescription(
793 new org.webrtc.SessionDescription(
794 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
795 .get();
796 return localRtpContentMap;
797 }
798
799 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
800 final RtpContentMap currentRemote = getRemoteContentMap();
801 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
802 if (diff.isEmpty()) {
803 throw new IllegalStateException(
804 "Unexpected rollback condition. No difference between local and remote");
805 }
806 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
807 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
808 final RtpContentMap nextRemote =
809 currentRemote.addContent(
810 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
811 return SessionDescription.of(nextRemote, !isInitiator());
812 }
813 throw new IllegalStateException(
814 "Unexpected rollback condition. Senders were not uniformly none");
815 }
816
817 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
818 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
819 if (incomingContentAdd == null) {
820 throw new IllegalStateException("No incoming content add");
821 }
822
823 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
824 this.incomingContentAdd = null;
825 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
826 Log.d(Config.LOGTAG,"senders of incoming content-add: "+senders);
827 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
828 Log.d(Config.LOGTAG,"content addition is receive only. we want to upgrade to 'both'");
829 final RtpContentMap modifiedSenders = incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
830 final JinglePacket proposedContentModification = modifiedSenders.toStub().toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
831 proposedContentModification.setTo(id.with);
832 xmppConnectionService.sendIqPacket(id.account, proposedContentModification, (account, response) -> {
833 if (response.getType() == IqPacket.TYPE.RESULT) {
834 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has accepted our upgrade to senders=both");
835 acceptContentAdd(ContentAddition.summary(modifiedSenders), modifiedSenders);
836 } else {
837 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has rejected our upgrade to senders=both");
838 acceptContentAdd(contentAddition, incomingContentAdd);
839 }
840 });
841 }
842 } else {
843 throw new IllegalStateException("Accepted content add does not match pending content-add");
844 }
845 }
846
847 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
848 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
849 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
850 this.setRemoteContentMap(modifiedContentMap);
851
852 final SessionDescription offer;
853 try {
854 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
855 } catch (final IllegalArgumentException | NullPointerException e) {
856 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
857 webRTCWrapper.close();
858 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
859 return;
860 }
861 this.incomingContentAdd = null;
862 acceptContentAdd(contentAddition, offer);
863 }
864
865 private void acceptContentAdd(
866 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
867 final org.webrtc.SessionDescription sdp =
868 new org.webrtc.SessionDescription(
869 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
870 try {
871 this.webRTCWrapper.setRemoteDescription(sdp).get();
872
873 // TODO add tracks for 'media' where contentAddition.senders matches
874
875 // TODO if senders.sending(isInitiator())
876
877 this.webRTCWrapper.addTrack(Media.VIDEO);
878
879 // TODO add additional transceivers for recv only cases
880
881 final SessionDescription answer = setLocalSessionDescription();
882 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
883
884 final RtpContentMap contentAcceptMap =
885 rtpContentMap.toContentModification(
886 Collections2.transform(contentAddition, ca -> ca.name));
887
888 Log.d(
889 Config.LOGTAG,
890 id.getAccount().getJid().asBareJid()
891 + ": sending content-accept "
892 + ContentAddition.summary(contentAcceptMap));
893
894 addIceCandidatesFromBlackLog();
895
896 modifyLocalContentMap(rtpContentMap);
897 final ListenableFuture<RtpContentMap> future = prepareOutgoingContentMap(contentAcceptMap);
898 Futures.addCallback(
899 future,
900 new FutureCallback<RtpContentMap>() {
901 @Override
902 public void onSuccess(final RtpContentMap rtpContentMap) {
903 sendContentAccept(rtpContentMap);
904 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
905 }
906
907 @Override
908 public void onFailure(@NonNull final Throwable throwable) {
909 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
910 }
911 },
912 MoreExecutors.directExecutor());
913 } catch (final Exception e) {
914 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
915 webRTCWrapper.close();
916 sendSessionTerminate(Reason.FAILED_APPLICATION);
917 }
918 }
919
920 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
921 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
922 send(jinglePacket);
923 }
924
925 public synchronized void rejectContentAdd() {
926 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
927 if (incomingContentAdd == null) {
928 throw new IllegalStateException("No incoming content add");
929 }
930 this.incomingContentAdd = null;
931 updateEndUserState();
932 rejectContentAdd(incomingContentAdd);
933 }
934
935 private void rejectContentAdd(final RtpContentMap contentMap) {
936 final JinglePacket jinglePacket =
937 contentMap
938 .toStub()
939 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
940 Log.d(
941 Config.LOGTAG,
942 id.getAccount().getJid().asBareJid()
943 + ": rejecting content "
944 + ContentAddition.summary(contentMap));
945 send(jinglePacket);
946 }
947
948 private boolean checkForIceRestart(
949 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
950 final RtpContentMap existing = getRemoteContentMap();
951 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
952 final IceUdpTransportInfo.Credentials newCredentials;
953 try {
954 existingCredentials = existing.getCredentials();
955 newCredentials = rtpContentMap.getDistinctCredentials();
956 } catch (final IllegalStateException e) {
957 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
958 return false;
959 }
960 if (existingCredentials.contains(newCredentials)) {
961 return false;
962 }
963 // TODO an alternative approach is to check if we already got an iq result to our
964 // ICE-restart
965 // and if that's the case we are seeing an answer.
966 // This might be more spec compliant but also more error prone potentially
967 final boolean isSignalStateStable = this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
968 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
969 final boolean isOffer = rtpContentMap.emptyCandidates();
970 final RtpContentMap restartContentMap;
971 try {
972 if (isOffer) {
973 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
974 restartContentMap =
975 existing.modifiedCredentials(
976 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
977 } else {
978 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
979 Log.d(
980 Config.LOGTAG,
981 "received confirmation of ICE restart"
982 + newCredentials
983 + " peer_setup="
984 + setup);
985 // DTLS setup attribute needs to be rewritten to reflect current peer state
986 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
987 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
988 }
989 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
990 return isOffer;
991 } else {
992 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
993 respondWithTieBreak(jinglePacket);
994 return true;
995 }
996 } catch (final Exception exception) {
997 respondOk(jinglePacket);
998 final Throwable rootCause = Throwables.getRootCause(exception);
999 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
1000 // If this happens a termination is already in progress
1001 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
1002 return true;
1003 }
1004 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
1005 webRTCWrapper.close();
1006 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1007 return true;
1008 }
1009 }
1010
1011 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
1012 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
1013 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
1014 throw new IllegalStateException("Invalid peer setup");
1015 }
1016 return peerSetup;
1017 }
1018
1019 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
1020 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
1021 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
1022 }
1023 this.peerDtlsSetup = setup;
1024 }
1025
1026 private boolean applyIceRestart(
1027 final JinglePacket jinglePacket,
1028 final RtpContentMap restartContentMap,
1029 final boolean isOffer)
1030 throws ExecutionException, InterruptedException {
1031 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
1032 final org.webrtc.SessionDescription.Type type =
1033 isOffer
1034 ? org.webrtc.SessionDescription.Type.OFFER
1035 : org.webrtc.SessionDescription.Type.ANSWER;
1036 org.webrtc.SessionDescription sdp =
1037 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1038 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1039 if (isInitiator()) {
1040 // We ignore the offer and respond with tie-break. This will clause the responder
1041 // not to apply the content map
1042 return false;
1043 }
1044 }
1045 webRTCWrapper.setRemoteDescription(sdp).get();
1046 setRemoteContentMap(restartContentMap);
1047 if (isOffer) {
1048 final SessionDescription localSessionDescription = setLocalSessionDescription();
1049 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1050 // We need to respond OK before sending any candidates
1051 respondOk(jinglePacket);
1052 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1053 } else {
1054 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1055 }
1056 return true;
1057 }
1058
1059 private void processCandidates(
1060 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
1061 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
1062 processCandidate(content);
1063 }
1064 }
1065
1066 private void processCandidate(
1067 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
1068 final RtpContentMap rtpContentMap = getRemoteContentMap();
1069 final List<String> indices = toIdentificationTags(rtpContentMap);
1070 final String sdpMid = content.getKey(); // aka content name
1071 final IceUdpTransportInfo transport = content.getValue().transport;
1072 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1073
1074 // TODO check that credentials remained the same
1075
1076 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1077 final String sdp;
1078 try {
1079 sdp = candidate.toSdpAttribute(credentials.ufrag);
1080 } catch (final IllegalArgumentException e) {
1081 Log.d(
1082 Config.LOGTAG,
1083 id.account.getJid().asBareJid()
1084 + ": ignoring invalid ICE candidate "
1085 + e.getMessage());
1086 continue;
1087 }
1088 final int mLineIndex = indices.indexOf(sdpMid);
1089 if (mLineIndex < 0) {
1090 Log.w(
1091 Config.LOGTAG,
1092 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1093 }
1094 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1095 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1096 this.webRTCWrapper.addIceCandidate(iceCandidate);
1097 }
1098 }
1099
1100 private RtpContentMap getRemoteContentMap() {
1101 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1102 }
1103
1104 private RtpContentMap getLocalContentMap() {
1105 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1106 }
1107
1108 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1109 final Group originalGroup = rtpContentMap.group;
1110 final List<String> identificationTags =
1111 originalGroup == null
1112 ? rtpContentMap.getNames()
1113 : originalGroup.getIdentificationTags();
1114 if (identificationTags.size() == 0) {
1115 Log.w(
1116 Config.LOGTAG,
1117 id.account.getJid().asBareJid()
1118 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1119 }
1120 return identificationTags;
1121 }
1122
1123 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1124 final JinglePacket jinglePacket, final boolean expectVerification) {
1125 try {
1126 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1127 } catch (final Exception e) {
1128 return Futures.immediateFailedFuture(e);
1129 }
1130 }
1131 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final RtpContentMap receivedContentMap, final boolean expectVerification) {
1132 Log.d(
1133 Config.LOGTAG,
1134 "receiveRtpContentMap("
1135 + receivedContentMap.getClass().getSimpleName()
1136 + ",expectVerification="
1137 + expectVerification
1138 + ")");
1139 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1140 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1141 id.account
1142 .getAxolotlService()
1143 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1144 return Futures.transform(
1145 future,
1146 omemoVerifiedPayload -> {
1147 // TODO test if an exception here triggers a correct abort
1148 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1149 Log.d(
1150 Config.LOGTAG,
1151 id.account.getJid().asBareJid()
1152 + ": received verifiable DTLS fingerprint via "
1153 + omemoVerification);
1154 return omemoVerifiedPayload.getPayload();
1155 },
1156 MoreExecutors.directExecutor());
1157 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1158 return Futures.immediateFailedFuture(
1159 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1160 } else {
1161 return Futures.immediateFuture(receivedContentMap);
1162 }
1163 }
1164
1165 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1166 if (isInitiator()) {
1167 Log.d(
1168 Config.LOGTAG,
1169 String.format(
1170 "%s: received session-initiate even though we were initiating",
1171 id.account.getJid().asBareJid()));
1172 if (isTerminated()) {
1173 Log.d(
1174 Config.LOGTAG,
1175 String.format(
1176 "%s: got a reason to terminate with out-of-order. but already in state %s",
1177 id.account.getJid().asBareJid(), getState()));
1178 respondWithOutOfOrder(jinglePacket);
1179 } else {
1180 terminateWithOutOfOrder(jinglePacket);
1181 }
1182 return;
1183 }
1184 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1185 Futures.addCallback(
1186 future,
1187 new FutureCallback<RtpContentMap>() {
1188 @Override
1189 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1190 receiveSessionInitiate(jinglePacket, rtpContentMap);
1191 }
1192
1193 @Override
1194 public void onFailure(@NonNull final Throwable throwable) {
1195 respondOk(jinglePacket);
1196 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1197 }
1198 },
1199 MoreExecutors.directExecutor());
1200 }
1201
1202 private void receiveSessionInitiate(
1203 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1204 try {
1205 contentMap.requireContentDescriptions();
1206 contentMap.requireDTLSFingerprint(true);
1207 } catch (final RuntimeException e) {
1208 Log.d(
1209 Config.LOGTAG,
1210 id.account.getJid().asBareJid() + ": improperly formatted contents",
1211 Throwables.getRootCause(e));
1212 respondOk(jinglePacket);
1213 sendSessionTerminate(Reason.of(e), e.getMessage());
1214 return;
1215 }
1216 Log.d(
1217 Config.LOGTAG,
1218 "processing session-init with " + contentMap.contents.size() + " contents");
1219 final State target;
1220 if (this.state == State.PROCEED) {
1221 Preconditions.checkState(
1222 proposedMedia != null && proposedMedia.size() > 0,
1223 "proposed media must be set when processing pre-approved session-initiate");
1224 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1225 sendSessionTerminate(
1226 Reason.SECURITY_ERROR,
1227 String.format(
1228 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1229 this.proposedMedia, contentMap.getMedia()));
1230 return;
1231 }
1232 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1233 } else {
1234 target = State.SESSION_INITIALIZED;
1235 }
1236 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1237 respondOk(jinglePacket);
1238 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1239 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1240 Log.d(
1241 Config.LOGTAG,
1242 id.account.getJid().asBareJid()
1243 + ": automatically accepting session-initiate");
1244 sendSessionAccept();
1245 } else {
1246 Log.d(
1247 Config.LOGTAG,
1248 id.account.getJid().asBareJid()
1249 + ": received not pre-approved session-initiate. start ringing");
1250 startRinging();
1251 }
1252 } else {
1253 Log.d(
1254 Config.LOGTAG,
1255 String.format(
1256 "%s: received session-initiate while in state %s",
1257 id.account.getJid().asBareJid(), state));
1258 terminateWithOutOfOrder(jinglePacket);
1259 }
1260 }
1261
1262 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1263 if (!isInitiator()) {
1264 Log.d(
1265 Config.LOGTAG,
1266 String.format(
1267 "%s: received session-accept even though we were responding",
1268 id.account.getJid().asBareJid()));
1269 terminateWithOutOfOrder(jinglePacket);
1270 return;
1271 }
1272 final ListenableFuture<RtpContentMap> future =
1273 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1274 Futures.addCallback(
1275 future,
1276 new FutureCallback<RtpContentMap>() {
1277 @Override
1278 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1279 receiveSessionAccept(jinglePacket, rtpContentMap);
1280 }
1281
1282 @Override
1283 public void onFailure(@NonNull final Throwable throwable) {
1284 respondOk(jinglePacket);
1285 Log.d(
1286 Config.LOGTAG,
1287 id.account.getJid().asBareJid()
1288 + ": improperly formatted contents in session-accept",
1289 throwable);
1290 webRTCWrapper.close();
1291 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1292 }
1293 },
1294 MoreExecutors.directExecutor());
1295 }
1296
1297 private void receiveSessionAccept(
1298 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1299 try {
1300 contentMap.requireContentDescriptions();
1301 contentMap.requireDTLSFingerprint();
1302 } catch (final RuntimeException e) {
1303 respondOk(jinglePacket);
1304 Log.d(
1305 Config.LOGTAG,
1306 id.account.getJid().asBareJid()
1307 + ": improperly formatted contents in session-accept",
1308 e);
1309 webRTCWrapper.close();
1310 sendSessionTerminate(Reason.of(e), e.getMessage());
1311 return;
1312 }
1313 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1314 if (!initiatorMedia.equals(contentMap.getMedia())) {
1315 sendSessionTerminate(
1316 Reason.SECURITY_ERROR,
1317 String.format(
1318 "Your session-included included media %s but our session-initiate was %s",
1319 this.proposedMedia, contentMap.getMedia()));
1320 return;
1321 }
1322 Log.d(
1323 Config.LOGTAG,
1324 "processing session-accept with " + contentMap.contents.size() + " contents");
1325 if (transition(State.SESSION_ACCEPTED)) {
1326 respondOk(jinglePacket);
1327 receiveSessionAccept(contentMap);
1328 } else {
1329 Log.d(
1330 Config.LOGTAG,
1331 String.format(
1332 "%s: received session-accept while in state %s",
1333 id.account.getJid().asBareJid(), state));
1334 respondOk(jinglePacket);
1335 }
1336 }
1337
1338 private void receiveSessionAccept(final RtpContentMap contentMap) {
1339 this.responderRtpContentMap = contentMap;
1340 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1341 final SessionDescription sessionDescription;
1342 try {
1343 sessionDescription = SessionDescription.of(contentMap, false);
1344 } catch (final IllegalArgumentException | NullPointerException e) {
1345 Log.d(
1346 Config.LOGTAG,
1347 id.account.getJid().asBareJid()
1348 + ": unable convert offer from session-accept to SDP",
1349 e);
1350 webRTCWrapper.close();
1351 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1352 return;
1353 }
1354 final org.webrtc.SessionDescription answer =
1355 new org.webrtc.SessionDescription(
1356 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1357 try {
1358 this.webRTCWrapper.setRemoteDescription(answer).get();
1359 } catch (final Exception e) {
1360 Log.d(
1361 Config.LOGTAG,
1362 id.account.getJid().asBareJid()
1363 + ": unable to set remote description after receiving session-accept",
1364 Throwables.getRootCause(e));
1365 webRTCWrapper.close();
1366 sendSessionTerminate(
1367 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1368 return;
1369 }
1370 processCandidates(contentMap.contents.entrySet());
1371 }
1372
1373 private void sendSessionAccept() {
1374 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1375 if (rtpContentMap == null) {
1376 throw new IllegalStateException("initiator RTP Content Map has not been set");
1377 }
1378 final SessionDescription offer;
1379 try {
1380 offer = SessionDescription.of(rtpContentMap, true);
1381 } catch (final IllegalArgumentException | NullPointerException e) {
1382 Log.d(
1383 Config.LOGTAG,
1384 id.account.getJid().asBareJid()
1385 + ": unable convert offer from session-initiate to SDP",
1386 e);
1387 webRTCWrapper.close();
1388 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1389 return;
1390 }
1391 sendSessionAccept(rtpContentMap.getMedia(), offer);
1392 }
1393
1394 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1395 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1396 }
1397
1398 private synchronized void sendSessionAccept(
1399 final Set<Media> media,
1400 final SessionDescription offer,
1401 final List<PeerConnection.IceServer> iceServers) {
1402 if (isTerminated()) {
1403 Log.w(
1404 Config.LOGTAG,
1405 id.account.getJid().asBareJid()
1406 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1407 return;
1408 }
1409 try {
1410 setupWebRTC(media, iceServers);
1411 } catch (final WebRTCWrapper.InitializationException e) {
1412 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1413 webRTCWrapper.close();
1414 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1415 return;
1416 }
1417 final org.webrtc.SessionDescription sdp =
1418 new org.webrtc.SessionDescription(
1419 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1420 try {
1421 this.webRTCWrapper.setRemoteDescription(sdp).get();
1422 addIceCandidatesFromBlackLog();
1423 org.webrtc.SessionDescription webRTCSessionDescription =
1424 this.webRTCWrapper.setLocalDescription().get();
1425 prepareSessionAccept(webRTCSessionDescription);
1426 } catch (final Exception e) {
1427 failureToAcceptSession(e);
1428 }
1429 }
1430
1431 private void failureToAcceptSession(final Throwable throwable) {
1432 if (isTerminated()) {
1433 return;
1434 }
1435 final Throwable rootCause = Throwables.getRootCause(throwable);
1436 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1437 webRTCWrapper.close();
1438 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1439 }
1440
1441 private void failureToPerformAction(final JinglePacket.Action action, final Throwable throwable) {
1442 if (isTerminated()) {
1443 return;
1444 }
1445 final Throwable rootCause = Throwables.getRootCause(throwable);
1446 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1447 webRTCWrapper.close();
1448 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1449 }
1450
1451 private void addIceCandidatesFromBlackLog() {
1452 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1453 while ((foo = this.pendingIceCandidates.poll()) != null) {
1454 processCandidate(foo);
1455 Log.d(
1456 Config.LOGTAG,
1457 id.account.getJid().asBareJid() + ": added candidate from back log");
1458 }
1459 }
1460
1461 private void prepareSessionAccept(
1462 final org.webrtc.SessionDescription webRTCSessionDescription) {
1463 final SessionDescription sessionDescription =
1464 SessionDescription.parse(webRTCSessionDescription.description);
1465 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1466 this.responderRtpContentMap = respondingRtpContentMap;
1467 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1468 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1469 prepareOutgoingContentMap(respondingRtpContentMap);
1470 Futures.addCallback(
1471 outgoingContentMapFuture,
1472 new FutureCallback<RtpContentMap>() {
1473 @Override
1474 public void onSuccess(final RtpContentMap outgoingContentMap) {
1475 sendSessionAccept(outgoingContentMap);
1476 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1477 }
1478
1479 @Override
1480 public void onFailure(@NonNull Throwable throwable) {
1481 failureToAcceptSession(throwable);
1482 }
1483 },
1484 MoreExecutors.directExecutor());
1485 }
1486
1487 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1488 if (isTerminated()) {
1489 Log.w(
1490 Config.LOGTAG,
1491 id.account.getJid().asBareJid()
1492 + ": preparing session accept was too slow. already terminated. nothing to do.");
1493 return;
1494 }
1495 transitionOrThrow(State.SESSION_ACCEPTED);
1496 final JinglePacket sessionAccept =
1497 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1498 send(sessionAccept);
1499 }
1500
1501 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1502 final RtpContentMap rtpContentMap) {
1503 if (this.omemoVerification.hasDeviceId()) {
1504 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1505 verifiedPayloadFuture =
1506 id.account
1507 .getAxolotlService()
1508 .encrypt(
1509 rtpContentMap,
1510 id.with,
1511 omemoVerification.getDeviceId());
1512 return Futures.transform(
1513 verifiedPayloadFuture,
1514 verifiedPayload -> {
1515 omemoVerification.setOrEnsureEqual(verifiedPayload);
1516 return verifiedPayload.getPayload();
1517 },
1518 MoreExecutors.directExecutor());
1519 } else {
1520 return Futures.immediateFuture(rtpContentMap);
1521 }
1522 }
1523
1524 synchronized void deliveryMessage(
1525 final Jid from,
1526 final Element message,
1527 final String serverMessageId,
1528 final long timestamp) {
1529 Log.d(
1530 Config.LOGTAG,
1531 id.account.getJid().asBareJid()
1532 + ": delivered message to JingleRtpConnection "
1533 + message);
1534 switch (message.getName()) {
1535 case "propose":
1536 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1537 break;
1538 case "proceed":
1539 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1540 break;
1541 case "retract":
1542 receiveRetract(from, serverMessageId, timestamp);
1543 break;
1544 case "reject":
1545 receiveReject(from, serverMessageId, timestamp);
1546 break;
1547 case "accept":
1548 receiveAccept(from, serverMessageId, timestamp);
1549 break;
1550 default:
1551 break;
1552 }
1553 }
1554
1555 void deliverFailedProceed(final String message) {
1556 Log.d(
1557 Config.LOGTAG,
1558 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1559 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1560 webRTCWrapper.close();
1561 Log.d(
1562 Config.LOGTAG,
1563 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1564 this.finish();
1565 }
1566 }
1567
1568 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1569 final boolean originatedFromMyself =
1570 from.asBareJid().equals(id.account.getJid().asBareJid());
1571 if (originatedFromMyself) {
1572 if (transition(State.ACCEPTED)) {
1573 acceptedOnOtherDevice(serverMsgId, timestamp);
1574 } else {
1575 Log.d(
1576 Config.LOGTAG,
1577 id.account.getJid().asBareJid()
1578 + ": unable to transition to accept because already in state="
1579 + this.state);
1580 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1581 }
1582 } else {
1583 Log.d(
1584 Config.LOGTAG,
1585 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1586 }
1587 }
1588
1589 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1590 if (serverMsgId != null) {
1591 this.message.setServerMsgId(serverMsgId);
1592 }
1593 this.message.setTime(timestamp);
1594 this.message.setCarbon(true); // indicate that call was accepted on other device
1595 this.writeLogMessageSuccess(0);
1596 this.xmppConnectionService
1597 .getNotificationService()
1598 .cancelIncomingCallNotification();
1599 this.finish();
1600 }
1601
1602 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1603 final boolean originatedFromMyself =
1604 from.asBareJid().equals(id.account.getJid().asBareJid());
1605 // reject from another one of my clients
1606 if (originatedFromMyself) {
1607 receiveRejectFromMyself(serverMsgId, timestamp);
1608 } else if (isInitiator()) {
1609 if (from.equals(id.with)) {
1610 receiveRejectFromResponder();
1611 } else {
1612 Log.d(
1613 Config.LOGTAG,
1614 id.account.getJid()
1615 + ": ignoring reject from "
1616 + from
1617 + " for session with "
1618 + id.with);
1619 }
1620 } else {
1621 Log.d(
1622 Config.LOGTAG,
1623 id.account.getJid()
1624 + ": ignoring reject from "
1625 + from
1626 + " for session with "
1627 + id.with);
1628 }
1629 }
1630
1631 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1632 if (transition(State.REJECTED)) {
1633 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1634 this.finish();
1635 if (serverMsgId != null) {
1636 this.message.setServerMsgId(serverMsgId);
1637 }
1638 this.message.setTime(timestamp);
1639 this.message.setCarbon(true); // indicate that call was rejected on other device
1640 writeLogMessageMissed();
1641 } else {
1642 Log.d(
1643 Config.LOGTAG,
1644 "not able to transition into REJECTED because already in " + this.state);
1645 }
1646 }
1647
1648 private void receiveRejectFromResponder() {
1649 if (isInState(State.PROCEED)) {
1650 Log.d(
1651 Config.LOGTAG,
1652 id.account.getJid()
1653 + ": received reject while still in proceed. callee reconsidered");
1654 closeTransitionLogFinish(State.REJECTED_RACED);
1655 return;
1656 }
1657 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1658 Log.d(
1659 Config.LOGTAG,
1660 id.account.getJid()
1661 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1662 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1663 return;
1664 }
1665 Log.d(
1666 Config.LOGTAG,
1667 id.account.getJid()
1668 + ": ignoring reject from responder because already in state "
1669 + this.state);
1670 }
1671
1672 private void receivePropose(
1673 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1674 final boolean originatedFromMyself =
1675 from.asBareJid().equals(id.account.getJid().asBareJid());
1676 if (originatedFromMyself) {
1677 Log.d(
1678 Config.LOGTAG,
1679 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1680 } else if (transition(
1681 State.PROPOSED,
1682 () -> {
1683 final Collection<RtpDescription> descriptions =
1684 Collections2.transform(
1685 Collections2.filter(
1686 propose.getDescriptions(),
1687 d -> d instanceof RtpDescription),
1688 input -> (RtpDescription) input);
1689 final Collection<Media> media =
1690 Collections2.transform(descriptions, RtpDescription::getMedia);
1691 Preconditions.checkState(
1692 !media.contains(Media.UNKNOWN),
1693 "RTP descriptions contain unknown media");
1694 Log.d(
1695 Config.LOGTAG,
1696 id.account.getJid().asBareJid()
1697 + ": received session proposal from "
1698 + from
1699 + " for "
1700 + media);
1701 this.proposedMedia = Sets.newHashSet(media);
1702 })) {
1703 if (serverMsgId != null) {
1704 this.message.setServerMsgId(serverMsgId);
1705 }
1706 this.message.setTime(timestamp);
1707 startRinging();
1708 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1709 sendJingleMessage("ringing");
1710 }
1711 } else {
1712 Log.d(
1713 Config.LOGTAG,
1714 id.account.getJid()
1715 + ": ignoring session proposal because already in "
1716 + state);
1717 }
1718 }
1719
1720 private void startRinging() {
1721 Log.d(
1722 Config.LOGTAG,
1723 id.account.getJid().asBareJid()
1724 + ": received call from "
1725 + id.with
1726 + ". start ringing");
1727 ringingTimeoutFuture =
1728 jingleConnectionManager.schedule(
1729 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1730 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1731 }
1732
1733 private synchronized void ringingTimeout() {
1734 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1735 switch (this.state) {
1736 case PROPOSED:
1737 message.markUnread();
1738 rejectCallFromProposed();
1739 break;
1740 case SESSION_INITIALIZED:
1741 message.markUnread();
1742 rejectCallFromSessionInitiate();
1743 break;
1744 }
1745 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1746 }
1747
1748 private void cancelRingingTimeout() {
1749 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1750 if (future != null && !future.isCancelled()) {
1751 future.cancel(false);
1752 }
1753 }
1754
1755 private void receiveProceed(
1756 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1757 final Set<Media> media =
1758 Preconditions.checkNotNull(
1759 this.proposedMedia, "Proposed media has to be set before handling proceed");
1760 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1761 if (from.equals(id.with)) {
1762 if (isInitiator()) {
1763 if (transition(State.PROCEED)) {
1764 if (serverMsgId != null) {
1765 this.message.setServerMsgId(serverMsgId);
1766 }
1767 this.message.setTime(timestamp);
1768 final Integer remoteDeviceId = proceed.getDeviceId();
1769 if (isOmemoEnabled()) {
1770 this.omemoVerification.setDeviceId(remoteDeviceId);
1771 } else {
1772 if (remoteDeviceId != null) {
1773 Log.d(
1774 Config.LOGTAG,
1775 id.account.getJid().asBareJid()
1776 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1777 }
1778 this.omemoVerification.setDeviceId(null);
1779 }
1780 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1781 } else {
1782 Log.d(
1783 Config.LOGTAG,
1784 String.format(
1785 "%s: ignoring proceed because already in %s",
1786 id.account.getJid().asBareJid(), this.state));
1787 }
1788 } else {
1789 Log.d(
1790 Config.LOGTAG,
1791 String.format(
1792 "%s: ignoring proceed because we were not initializing",
1793 id.account.getJid().asBareJid()));
1794 }
1795 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1796 if (transition(State.ACCEPTED)) {
1797 Log.d(
1798 Config.LOGTAG,
1799 id.account.getJid().asBareJid()
1800 + ": moved session with "
1801 + id.with
1802 + " into state accepted after received carbon copied proceed");
1803 acceptedOnOtherDevice(serverMsgId, timestamp);
1804 }
1805 } else {
1806 Log.d(
1807 Config.LOGTAG,
1808 String.format(
1809 "%s: ignoring proceed from %s. was expected from %s",
1810 id.account.getJid().asBareJid(), from, id.with));
1811 }
1812 }
1813
1814 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1815 if (from.equals(id.with)) {
1816 final State target =
1817 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1818 if (transition(target)) {
1819 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1820 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1821 Log.d(
1822 Config.LOGTAG,
1823 id.account.getJid().asBareJid()
1824 + ": session with "
1825 + id.with
1826 + " has been retracted (serverMsgId="
1827 + serverMsgId
1828 + ")");
1829 if (serverMsgId != null) {
1830 this.message.setServerMsgId(serverMsgId);
1831 }
1832 this.message.setTime(timestamp);
1833 if (target == State.RETRACTED) {
1834 this.message.markUnread();
1835 }
1836 writeLogMessageMissed();
1837 finish();
1838 } else {
1839 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1840 }
1841 } else {
1842 // TODO parse retract from self
1843 Log.d(
1844 Config.LOGTAG,
1845 id.account.getJid().asBareJid()
1846 + ": received retract from "
1847 + from
1848 + ". expected retract from"
1849 + id.with
1850 + ". ignoring");
1851 }
1852 }
1853
1854 public void sendSessionInitiate() {
1855 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1856 }
1857
1858 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1859 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1860 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1861 }
1862
1863 private synchronized void sendSessionInitiate(
1864 final Set<Media> media,
1865 final State targetState,
1866 final List<PeerConnection.IceServer> iceServers) {
1867 if (isTerminated()) {
1868 Log.w(
1869 Config.LOGTAG,
1870 id.account.getJid().asBareJid()
1871 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1872 return;
1873 }
1874 try {
1875 setupWebRTC(media, iceServers);
1876 } catch (final WebRTCWrapper.InitializationException e) {
1877 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1878 webRTCWrapper.close();
1879 sendRetract(Reason.ofThrowable(e));
1880 return;
1881 }
1882 try {
1883 org.webrtc.SessionDescription webRTCSessionDescription =
1884 this.webRTCWrapper.setLocalDescription().get();
1885 prepareSessionInitiate(webRTCSessionDescription, targetState);
1886 } catch (final Exception e) {
1887 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1888 // exceptions
1889 failureToInitiateSession(e, targetState);
1890 }
1891 }
1892
1893 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1894 if (isTerminated()) {
1895 return;
1896 }
1897 Log.d(
1898 Config.LOGTAG,
1899 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1900 Throwables.getRootCause(throwable));
1901 webRTCWrapper.close();
1902 final Reason reason = Reason.ofThrowable(throwable);
1903 if (isInState(targetState)) {
1904 sendSessionTerminate(reason, throwable.getMessage());
1905 } else {
1906 sendRetract(reason);
1907 }
1908 }
1909
1910 private void sendRetract(final Reason reason) {
1911 // TODO embed reason into retract
1912 sendJingleMessage("retract", id.with.asBareJid());
1913 transitionOrThrow(reasonToState(reason));
1914 this.finish();
1915 }
1916
1917 private void prepareSessionInitiate(
1918 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1919 final SessionDescription sessionDescription =
1920 SessionDescription.parse(webRTCSessionDescription.description);
1921 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1922 this.initiatorRtpContentMap = rtpContentMap;
1923 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1924 encryptSessionInitiate(rtpContentMap);
1925 Futures.addCallback(
1926 outgoingContentMapFuture,
1927 new FutureCallback<RtpContentMap>() {
1928 @Override
1929 public void onSuccess(final RtpContentMap outgoingContentMap) {
1930 sendSessionInitiate(outgoingContentMap, targetState);
1931 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1932 }
1933
1934 @Override
1935 public void onFailure(@NonNull final Throwable throwable) {
1936 failureToInitiateSession(throwable, targetState);
1937 }
1938 },
1939 MoreExecutors.directExecutor());
1940 }
1941
1942 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1943 if (isTerminated()) {
1944 Log.w(
1945 Config.LOGTAG,
1946 id.account.getJid().asBareJid()
1947 + ": preparing session was too slow. already terminated. nothing to do.");
1948 return;
1949 }
1950 this.transitionOrThrow(targetState);
1951 final JinglePacket sessionInitiate =
1952 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1953 send(sessionInitiate);
1954 }
1955
1956 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1957 final RtpContentMap rtpContentMap) {
1958 if (this.omemoVerification.hasDeviceId()) {
1959 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1960 verifiedPayloadFuture =
1961 id.account
1962 .getAxolotlService()
1963 .encrypt(
1964 rtpContentMap,
1965 id.with,
1966 omemoVerification.getDeviceId());
1967 final ListenableFuture<RtpContentMap> future =
1968 Futures.transform(
1969 verifiedPayloadFuture,
1970 verifiedPayload -> {
1971 omemoVerification.setSessionFingerprint(
1972 verifiedPayload.getFingerprint());
1973 return verifiedPayload.getPayload();
1974 },
1975 MoreExecutors.directExecutor());
1976 if (Config.REQUIRE_RTP_VERIFICATION) {
1977 return future;
1978 }
1979 return Futures.catching(
1980 future,
1981 CryptoFailedException.class,
1982 e -> {
1983 Log.w(
1984 Config.LOGTAG,
1985 id.account.getJid().asBareJid()
1986 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1987 e);
1988 return rtpContentMap;
1989 },
1990 MoreExecutors.directExecutor());
1991 } else {
1992 return Futures.immediateFuture(rtpContentMap);
1993 }
1994 }
1995
1996 private void sendSessionTerminate(final Reason reason) {
1997 sendSessionTerminate(reason, null);
1998 }
1999
2000 private void sendSessionTerminate(final Reason reason, final String text) {
2001 final State previous = this.state;
2002 final State target = reasonToState(reason);
2003 transitionOrThrow(target);
2004 if (previous != State.NULL) {
2005 writeLogMessage(target);
2006 }
2007 final JinglePacket jinglePacket =
2008 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
2009 jinglePacket.setReason(reason, text);
2010 send(jinglePacket);
2011 finish();
2012 }
2013
2014 private void sendTransportInfo(
2015 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2016 final RtpContentMap transportInfo;
2017 try {
2018 final RtpContentMap rtpContentMap =
2019 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2020 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2021 } catch (final Exception e) {
2022 Log.d(
2023 Config.LOGTAG,
2024 id.account.getJid().asBareJid()
2025 + ": unable to prepare transport-info from candidate for content="
2026 + contentName);
2027 return;
2028 }
2029 final JinglePacket jinglePacket =
2030 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2031 send(jinglePacket);
2032 }
2033
2034 private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
2035 // TODO send all candidates in one transport-info
2036 }
2037
2038
2039 private void send(final JinglePacket jinglePacket) {
2040 jinglePacket.setTo(id.with);
2041 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
2042 }
2043
2044 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
2045 if (response.getType() == IqPacket.TYPE.ERROR) {
2046 handleIqErrorResponse(response);
2047 return;
2048 }
2049 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2050 handleIqTimeoutResponse(response);
2051 }
2052 }
2053
2054 private void handleIqErrorResponse(final IqPacket response) {
2055 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
2056 final String errorCondition = response.getErrorCondition();
2057 Log.d(
2058 Config.LOGTAG,
2059 id.account.getJid().asBareJid()
2060 + ": received IQ-error from "
2061 + response.getFrom()
2062 + " in RTP session. "
2063 + errorCondition);
2064 if (isTerminated()) {
2065 Log.i(
2066 Config.LOGTAG,
2067 id.account.getJid().asBareJid()
2068 + ": ignoring error because session was already terminated");
2069 return;
2070 }
2071 this.webRTCWrapper.close();
2072 final State target;
2073 if (Arrays.asList(
2074 "service-unavailable",
2075 "recipient-unavailable",
2076 "remote-server-not-found",
2077 "remote-server-timeout")
2078 .contains(errorCondition)) {
2079 target = State.TERMINATED_CONNECTIVITY_ERROR;
2080 } else {
2081 target = State.TERMINATED_APPLICATION_FAILURE;
2082 }
2083 transitionOrThrow(target);
2084 this.finish();
2085 }
2086
2087 private void handleIqTimeoutResponse(final IqPacket response) {
2088 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
2089 Log.d(
2090 Config.LOGTAG,
2091 id.account.getJid().asBareJid()
2092 + ": received IQ timeout in RTP session with "
2093 + id.with
2094 + ". terminating with connectivity error");
2095 if (isTerminated()) {
2096 Log.i(
2097 Config.LOGTAG,
2098 id.account.getJid().asBareJid()
2099 + ": ignoring error because session was already terminated");
2100 return;
2101 }
2102 this.webRTCWrapper.close();
2103 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
2104 this.finish();
2105 }
2106
2107 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
2108 Log.d(
2109 Config.LOGTAG,
2110 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
2111 this.webRTCWrapper.close();
2112 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
2113 respondWithOutOfOrder(jinglePacket);
2114 this.finish();
2115 }
2116
2117 private void respondWithTieBreak(final JinglePacket jinglePacket) {
2118 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
2119 }
2120
2121 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
2122 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
2123 }
2124
2125 private void respondWithItemNotFound(final JinglePacket jinglePacket) {
2126 respondWithJingleError(jinglePacket, null, "item-not-found", "cancel");
2127 }
2128
2129 void respondWithJingleError(
2130 final IqPacket original,
2131 String jingleCondition,
2132 String condition,
2133 String conditionType) {
2134 jingleConnectionManager.respondWithJingleError(
2135 id.account, original, jingleCondition, condition, conditionType);
2136 }
2137
2138 private void respondOk(final JinglePacket jinglePacket) {
2139 xmppConnectionService.sendIqPacket(
2140 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
2141 }
2142
2143 public RtpEndUserState getEndUserState() {
2144 switch (this.state) {
2145 case NULL:
2146 case PROPOSED:
2147 case SESSION_INITIALIZED:
2148 if (isInitiator()) {
2149 return RtpEndUserState.RINGING;
2150 } else {
2151 return RtpEndUserState.INCOMING_CALL;
2152 }
2153 case PROCEED:
2154 if (isInitiator()) {
2155 return RtpEndUserState.RINGING;
2156 } else {
2157 return RtpEndUserState.ACCEPTING_CALL;
2158 }
2159 case SESSION_INITIALIZED_PRE_APPROVED:
2160 if (isInitiator()) {
2161 return RtpEndUserState.RINGING;
2162 } else {
2163 return RtpEndUserState.CONNECTING;
2164 }
2165 case SESSION_ACCEPTED:
2166 final ContentAddition ca = getPendingContentAddition();
2167 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2168 return RtpEndUserState.INCOMING_CONTENT_ADD;
2169 }
2170 return getPeerConnectionStateAsEndUserState();
2171 case REJECTED:
2172 case REJECTED_RACED:
2173 case TERMINATED_DECLINED_OR_BUSY:
2174 if (isInitiator()) {
2175 return RtpEndUserState.DECLINED_OR_BUSY;
2176 } else {
2177 return RtpEndUserState.ENDED;
2178 }
2179 case TERMINATED_SUCCESS:
2180 case ACCEPTED:
2181 case RETRACTED:
2182 case TERMINATED_CANCEL_OR_TIMEOUT:
2183 return RtpEndUserState.ENDED;
2184 case RETRACTED_RACED:
2185 if (isInitiator()) {
2186 return RtpEndUserState.ENDED;
2187 } else {
2188 return RtpEndUserState.RETRACTED;
2189 }
2190 case TERMINATED_CONNECTIVITY_ERROR:
2191 return zeroDuration()
2192 ? RtpEndUserState.CONNECTIVITY_ERROR
2193 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2194 case TERMINATED_APPLICATION_FAILURE:
2195 return RtpEndUserState.APPLICATION_ERROR;
2196 case TERMINATED_SECURITY_ERROR:
2197 return RtpEndUserState.SECURITY_ERROR;
2198 }
2199 throw new IllegalStateException(
2200 String.format("%s has no equivalent EndUserState", this.state));
2201 }
2202
2203 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2204 final PeerConnection.PeerConnectionState state;
2205 try {
2206 state = webRTCWrapper.getState();
2207 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2208 // We usually close the WebRTCWrapper *before* transitioning so we might still
2209 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2210 return RtpEndUserState.ENDING_CALL;
2211 }
2212 switch (state) {
2213 case CONNECTED:
2214 return RtpEndUserState.CONNECTED;
2215 case NEW:
2216 case CONNECTING:
2217 return RtpEndUserState.CONNECTING;
2218 case CLOSED:
2219 return RtpEndUserState.ENDING_CALL;
2220 default:
2221 return zeroDuration()
2222 ? RtpEndUserState.CONNECTIVITY_ERROR
2223 : RtpEndUserState.RECONNECTING;
2224 }
2225 }
2226
2227 public ContentAddition getPendingContentAddition() {
2228 final RtpContentMap in = this.incomingContentAdd;
2229 final RtpContentMap out = this.outgoingContentAdd;
2230 if (out != null) {
2231 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2232 } else if (in != null) {
2233 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2234 } else {
2235 return null;
2236 }
2237 }
2238
2239 public Set<Media> getMedia() {
2240 final State current = getState();
2241 if (current == State.NULL) {
2242 if (isInitiator()) {
2243 return Preconditions.checkNotNull(
2244 this.proposedMedia, "RTP connection has not been initialized properly");
2245 }
2246 throw new IllegalStateException("RTP connection has not been initialized yet");
2247 }
2248 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2249 return Preconditions.checkNotNull(
2250 this.proposedMedia, "RTP connection has not been initialized properly");
2251 }
2252 final RtpContentMap localContentMap = getLocalContentMap();
2253 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2254 if (localContentMap != null) {
2255 return localContentMap.getMedia();
2256 } else if (initiatorContentMap != null) {
2257 return initiatorContentMap.getMedia();
2258 } else if (isTerminated()) {
2259 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2260 } else {
2261 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2262 }
2263 }
2264
2265 public boolean isVerified() {
2266 final String fingerprint = this.omemoVerification.getFingerprint();
2267 if (fingerprint == null) {
2268 return false;
2269 }
2270 final FingerprintStatus status =
2271 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2272 return status != null && status.isVerified();
2273 }
2274
2275 public boolean addMedia(final Media media) {
2276 final Set<Media> currentMedia = getMedia();
2277 if (currentMedia.contains(media)) {
2278 throw new IllegalStateException(String.format("%s has already been proposed", media));
2279 }
2280 // TODO add state protection - can only add while ACCEPTED or so
2281 Log.d(Config.LOGTAG,"adding media: "+media);
2282 return webRTCWrapper.addTrack(media);
2283 }
2284
2285 public synchronized void acceptCall() {
2286 switch (this.state) {
2287 case PROPOSED:
2288 cancelRingingTimeout();
2289 acceptCallFromProposed();
2290 break;
2291 case SESSION_INITIALIZED:
2292 cancelRingingTimeout();
2293 acceptCallFromSessionInitialized();
2294 break;
2295 case ACCEPTED:
2296 Log.w(
2297 Config.LOGTAG,
2298 id.account.getJid().asBareJid()
2299 + ": the call has already been accepted with another client. UI was just lagging behind");
2300 break;
2301 case PROCEED:
2302 case SESSION_ACCEPTED:
2303 Log.w(
2304 Config.LOGTAG,
2305 id.account.getJid().asBareJid()
2306 + ": the call has already been accepted. user probably double tapped the UI");
2307 break;
2308 default:
2309 throw new IllegalStateException("Can not accept call from " + this.state);
2310 }
2311 }
2312
2313 public void notifyPhoneCall() {
2314 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2315 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2316 rejectCall();
2317 } else {
2318 endCall();
2319 }
2320 }
2321
2322 public synchronized void rejectCall() {
2323 if (isTerminated()) {
2324 Log.w(
2325 Config.LOGTAG,
2326 id.account.getJid().asBareJid()
2327 + ": received rejectCall() when session has already been terminated. nothing to do");
2328 return;
2329 }
2330 switch (this.state) {
2331 case PROPOSED:
2332 rejectCallFromProposed();
2333 break;
2334 case SESSION_INITIALIZED:
2335 rejectCallFromSessionInitiate();
2336 break;
2337 default:
2338 throw new IllegalStateException("Can not reject call from " + this.state);
2339 }
2340 }
2341
2342 public synchronized void endCall() {
2343 if (isTerminated()) {
2344 Log.w(
2345 Config.LOGTAG,
2346 id.account.getJid().asBareJid()
2347 + ": received endCall() when session has already been terminated. nothing to do");
2348 return;
2349 }
2350 if (isInState(State.PROPOSED) && !isInitiator()) {
2351 rejectCallFromProposed();
2352 return;
2353 }
2354 if (isInState(State.PROCEED)) {
2355 if (isInitiator()) {
2356 retractFromProceed();
2357 } else {
2358 rejectCallFromProceed();
2359 }
2360 return;
2361 }
2362 if (isInitiator()
2363 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2364 this.webRTCWrapper.close();
2365 sendSessionTerminate(Reason.CANCEL);
2366 return;
2367 }
2368 if (isInState(State.SESSION_INITIALIZED)) {
2369 rejectCallFromSessionInitiate();
2370 return;
2371 }
2372 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2373 this.webRTCWrapper.close();
2374 sendSessionTerminate(Reason.SUCCESS);
2375 return;
2376 }
2377 if (isInState(
2378 State.TERMINATED_APPLICATION_FAILURE,
2379 State.TERMINATED_CONNECTIVITY_ERROR,
2380 State.TERMINATED_DECLINED_OR_BUSY)) {
2381 Log.d(
2382 Config.LOGTAG,
2383 "ignoring request to end call because already in state " + this.state);
2384 return;
2385 }
2386 throw new IllegalStateException(
2387 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2388 }
2389
2390 private void retractFromProceed() {
2391 Log.d(Config.LOGTAG, "retract from proceed");
2392 this.sendJingleMessage("retract");
2393 closeTransitionLogFinish(State.RETRACTED_RACED);
2394 }
2395
2396 private void closeTransitionLogFinish(final State state) {
2397 this.webRTCWrapper.close();
2398 transitionOrThrow(state);
2399 writeLogMessage(state);
2400 finish();
2401 }
2402
2403 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2404 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2405 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2406 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2407 }
2408
2409 private void acceptCallFromProposed() {
2410 transitionOrThrow(State.PROCEED);
2411 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2412 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2413 this.sendJingleMessage("proceed");
2414 }
2415
2416 private void rejectCallFromProposed() {
2417 transitionOrThrow(State.REJECTED);
2418 writeLogMessageMissed();
2419 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2420 this.sendJingleMessage("reject");
2421 finish();
2422 }
2423
2424 private void rejectCallFromProceed() {
2425 this.sendJingleMessage("reject");
2426 closeTransitionLogFinish(State.REJECTED_RACED);
2427 }
2428
2429 private void rejectCallFromSessionInitiate() {
2430 webRTCWrapper.close();
2431 sendSessionTerminate(Reason.DECLINE);
2432 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2433 }
2434
2435 private void sendJingleMessage(final String action) {
2436 sendJingleMessage(action, id.with);
2437 }
2438
2439 private void sendJingleMessage(final String action, final Jid to) {
2440 final MessagePacket messagePacket = new MessagePacket();
2441 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2442 messagePacket.setTo(to);
2443 final Element intent =
2444 messagePacket
2445 .addChild(action, Namespace.JINGLE_MESSAGE)
2446 .setAttribute("id", id.sessionId);
2447 if ("proceed".equals(action)) {
2448 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2449 if (isOmemoEnabled()) {
2450 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2451 final Element device =
2452 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2453 device.setAttribute("id", deviceId);
2454 }
2455 }
2456 messagePacket.addChild("store", "urn:xmpp:hints");
2457 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2458 }
2459
2460 private boolean isOmemoEnabled() {
2461 final Conversational conversational = message.getConversation();
2462 if (conversational instanceof Conversation) {
2463 return ((Conversation) conversational).getNextEncryption()
2464 == Message.ENCRYPTION_AXOLOTL;
2465 }
2466 return false;
2467 }
2468
2469 private void acceptCallFromSessionInitialized() {
2470 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2471 sendSessionAccept();
2472 }
2473
2474 private synchronized boolean isInState(State... state) {
2475 return Arrays.asList(state).contains(this.state);
2476 }
2477
2478 private boolean transition(final State target) {
2479 return transition(target, null);
2480 }
2481
2482 private synchronized boolean transition(final State target, final Runnable runnable) {
2483 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2484 if (validTransitions != null && validTransitions.contains(target)) {
2485 this.state = target;
2486 if (runnable != null) {
2487 runnable.run();
2488 }
2489 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2490 updateEndUserState();
2491 updateOngoingCallNotification();
2492 return true;
2493 } else {
2494 return false;
2495 }
2496 }
2497
2498 void transitionOrThrow(final State target) {
2499 if (!transition(target)) {
2500 throw new IllegalStateException(
2501 String.format("Unable to transition from %s to %s", this.state, target));
2502 }
2503 }
2504
2505 @Override
2506 public void onIceCandidate(final IceCandidate iceCandidate) {
2507 final RtpContentMap rtpContentMap =
2508 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2509 final IceUdpTransportInfo.Credentials credentials;
2510 try {
2511 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2512 } catch (final IllegalArgumentException e) {
2513 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2514 return;
2515 }
2516 final String uFrag = credentials.ufrag;
2517 final IceUdpTransportInfo.Candidate candidate =
2518 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2519 if (candidate == null) {
2520 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2521 return;
2522 }
2523 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2524 sendTransportInfo(iceCandidate.sdpMid, candidate);
2525 }
2526
2527 @Override
2528 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2529 Log.d(
2530 Config.LOGTAG,
2531 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2532 this.stateHistory.add(newState);
2533 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2534 this.sessionDuration.start();
2535 updateOngoingCallNotification();
2536 } else if (this.sessionDuration.isRunning()) {
2537 this.sessionDuration.stop();
2538 updateOngoingCallNotification();
2539 }
2540
2541 final boolean neverConnected =
2542 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2543
2544 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2545 if (neverConnected) {
2546 if (isTerminated()) {
2547 Log.d(
2548 Config.LOGTAG,
2549 id.account.getJid().asBareJid()
2550 + ": not sending session-terminate after connectivity error because session is already in state "
2551 + this.state);
2552 return;
2553 }
2554 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2555 return;
2556 } else {
2557 this.restartIce();
2558 }
2559 }
2560 updateEndUserState();
2561 }
2562
2563 private void restartIce() {
2564 this.stateHistory.clear();
2565 this.webRTCWrapper.restartIceAsync();
2566 }
2567
2568 @Override
2569 public void onRenegotiationNeeded() {
2570 this.webRTCWrapper.execute(this::renegotiate);
2571 }
2572
2573 private void renegotiate() {
2574 final SessionDescription sessionDescription;
2575 try {
2576 sessionDescription = setLocalSessionDescription();
2577 } catch (final Exception e) {
2578 final Throwable cause = Throwables.getRootCause(e);
2579 webRTCWrapper.close();
2580 if (isTerminated()) {
2581 Log.d(Config.LOGTAG, "failed to renegotiate. session was already terminated", cause);
2582 return;
2583 }
2584 Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause);
2585 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2586 return;
2587 }
2588 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2589 final RtpContentMap currentContentMap = getLocalContentMap();
2590 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2591 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2592
2593 Log.d(
2594 Config.LOGTAG,
2595 id.getAccount().getJid().asBareJid()
2596 + ": renegotiate. iceRestart="
2597 + iceRestart
2598 + " content id diff="
2599 + diff);
2600
2601 if (diff.hasModifications() && iceRestart) {
2602 webRTCWrapper.close();
2603 sendSessionTerminate(
2604 Reason.FAILED_APPLICATION,
2605 "WebRTC unexpectedly tried to modify content and transport at once");
2606 return;
2607 }
2608
2609 if (iceRestart) {
2610 initiateIceRestart(rtpContentMap);
2611 return;
2612 } else if (diff.isEmpty()) {
2613 Log.d(
2614 Config.LOGTAG,
2615 "renegotiation. nothing to do. SignalingState="
2616 + this.webRTCWrapper.getSignalingState());
2617 }
2618
2619 if (diff.added.size() > 0) {
2620 modifyLocalContentMap(rtpContentMap);
2621 sendContentAdd(rtpContentMap, diff.added);
2622 }
2623 }
2624
2625 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2626 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2627 final JinglePacket jinglePacket =
2628 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2629 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2630 jinglePacket.setTo(id.with);
2631 xmppConnectionService.sendIqPacket(
2632 id.account,
2633 jinglePacket,
2634 (account, response) -> {
2635 if (response.getType() == IqPacket.TYPE.RESULT) {
2636 Log.d(Config.LOGTAG, "received success to our ice restart");
2637 setLocalContentMap(rtpContentMap);
2638 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2639 return;
2640 }
2641 if (response.getType() == IqPacket.TYPE.ERROR) {
2642 if (isTieBreak(response)) {
2643 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2644 return;
2645 }
2646 handleIqErrorResponse(response);
2647 }
2648 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2649 handleIqTimeoutResponse(response);
2650 }
2651 });
2652 }
2653
2654 private boolean isTieBreak(final IqPacket response) {
2655 final Element error = response.findChild("error");
2656 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2657 }
2658
2659 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2660 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2661 this.outgoingContentAdd = contentAdd;
2662 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2663 prepareOutgoingContentMap(contentAdd);
2664 Futures.addCallback(
2665 outgoingContentMapFuture,
2666 new FutureCallback<RtpContentMap>() {
2667 @Override
2668 public void onSuccess(final RtpContentMap outgoingContentMap) {
2669 sendContentAdd(outgoingContentMap);
2670 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2671 }
2672
2673 @Override
2674 public void onFailure(@NonNull Throwable throwable) {
2675 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2676 }
2677 },
2678 MoreExecutors.directExecutor());
2679 }
2680
2681 private void sendContentAdd(final RtpContentMap contentAdd) {
2682
2683 final JinglePacket jinglePacket =
2684 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2685 jinglePacket.setTo(id.with);
2686 xmppConnectionService.sendIqPacket(
2687 id.account,
2688 jinglePacket,
2689 (connection, response) -> {
2690 if (response.getType() == IqPacket.TYPE.RESULT) {
2691 Log.d(
2692 Config.LOGTAG,
2693 id.getAccount().getJid().asBareJid()
2694 + ": received ACK to our content-add");
2695 return;
2696 }
2697 if (response.getType() == IqPacket.TYPE.ERROR) {
2698 if (isTieBreak(response)) {
2699 this.outgoingContentAdd = null;
2700 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2701 return;
2702 }
2703 handleIqErrorResponse(response);
2704 }
2705 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2706 handleIqTimeoutResponse(response);
2707 }
2708 });
2709 }
2710
2711 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2712 if (isInitiator()) {
2713 this.initiatorRtpContentMap = rtpContentMap;
2714 } else {
2715 this.responderRtpContentMap = rtpContentMap;
2716 }
2717 }
2718
2719 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2720 if (isInitiator()) {
2721 this.responderRtpContentMap = rtpContentMap;
2722 } else {
2723 this.initiatorRtpContentMap = rtpContentMap;
2724 }
2725 }
2726
2727 // this method is to be used for content map modifications that modify media
2728 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2729 final RtpContentMap activeContents = rtpContentMap.activeContents();
2730 setLocalContentMap(activeContents);
2731 this.webRTCWrapper.switchSpeakerPhonePreference(
2732 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2733 updateEndUserState();
2734 }
2735
2736 private SessionDescription setLocalSessionDescription()
2737 throws ExecutionException, InterruptedException {
2738 final org.webrtc.SessionDescription sessionDescription =
2739 this.webRTCWrapper.setLocalDescription().get();
2740 return SessionDescription.parse(sessionDescription.description);
2741 }
2742
2743 private void closeWebRTCSessionAfterFailedConnection() {
2744 this.webRTCWrapper.close();
2745 synchronized (this) {
2746 if (isTerminated()) {
2747 Log.d(
2748 Config.LOGTAG,
2749 id.account.getJid().asBareJid()
2750 + ": no need to send session-terminate after failed connection. Other party already did");
2751 return;
2752 }
2753 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2754 }
2755 }
2756
2757 public boolean zeroDuration() {
2758 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2759 }
2760
2761 public long getCallDuration() {
2762 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2763 }
2764
2765 public AppRTCAudioManager getAudioManager() {
2766 return webRTCWrapper.getAudioManager();
2767 }
2768
2769 public boolean isMicrophoneEnabled() {
2770 return webRTCWrapper.isMicrophoneEnabled();
2771 }
2772
2773 public boolean setMicrophoneEnabled(final boolean enabled) {
2774 return webRTCWrapper.setMicrophoneEnabled(enabled);
2775 }
2776
2777 public boolean isVideoEnabled() {
2778 return webRTCWrapper.isVideoEnabled();
2779 }
2780
2781 public void setVideoEnabled(final boolean enabled) {
2782 webRTCWrapper.setVideoEnabled(enabled);
2783 }
2784
2785 public boolean isCameraSwitchable() {
2786 return webRTCWrapper.isCameraSwitchable();
2787 }
2788
2789 public boolean isFrontCamera() {
2790 return webRTCWrapper.isFrontCamera();
2791 }
2792
2793 public ListenableFuture<Boolean> switchCamera() {
2794 return webRTCWrapper.switchCamera();
2795 }
2796
2797 @Override
2798 public void onAudioDeviceChanged(
2799 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2800 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2801 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2802 selectedAudioDevice, availableAudioDevices);
2803 }
2804
2805 private void updateEndUserState() {
2806 final RtpEndUserState endUserState = getEndUserState();
2807 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2808 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2809 id.account, id.with, id.sessionId, endUserState);
2810 }
2811
2812 private void updateOngoingCallNotification() {
2813 final State state = this.state;
2814 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2815 final boolean reconnecting;
2816 if (state == State.SESSION_ACCEPTED) {
2817 reconnecting =
2818 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2819 } else {
2820 reconnecting = false;
2821 }
2822 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2823 } else {
2824 xmppConnectionService.removeOngoingCall();
2825 }
2826 }
2827
2828 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2829 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2830 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2831 request.setTo(id.account.getDomain());
2832 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2833 xmppConnectionService.sendIqPacket(
2834 id.account,
2835 request,
2836 (account, response) -> {
2837 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2838 new ImmutableList.Builder<>();
2839 if (response.getType() == IqPacket.TYPE.RESULT) {
2840 final Element services =
2841 response.findChild(
2842 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2843 final List<Element> children =
2844 services == null
2845 ? Collections.emptyList()
2846 : services.getChildren();
2847 for (final Element child : children) {
2848 if ("service".equals(child.getName())) {
2849 final String type = child.getAttribute("type");
2850 final String host = child.getAttribute("host");
2851 final String sport = child.getAttribute("port");
2852 final Integer port =
2853 sport == null ? null : Ints.tryParse(sport);
2854 final String transport = child.getAttribute("transport");
2855 final String username = child.getAttribute("username");
2856 final String password = child.getAttribute("password");
2857 if (Strings.isNullOrEmpty(host) || port == null) {
2858 continue;
2859 }
2860 if (port < 0 || port > 65535) {
2861 continue;
2862 }
2863
2864
2865
2866
2867 if (Arrays.asList("stun", "stuns", "turn", "turns")
2868 .contains(type)
2869 && Arrays.asList("udp", "tcp").contains(transport)) {
2870 if (Arrays.asList("stuns", "turns").contains(type)
2871 && "udp".equals(transport)) {
2872 Log.d(
2873 Config.LOGTAG,
2874 id.account.getJid().asBareJid()
2875 + ": skipping invalid combination of udp/tls in external services");
2876 continue;
2877 }
2878
2879 // STUN URLs do not support a query section since M110
2880 final String uri;
2881 if (Arrays.asList("stun","stuns").contains(type)) {
2882 uri = String.format("%s:%s:%s", type, IP.wrapIPv6(host),port);
2883 } else {
2884 uri = String.format(
2885 "%s:%s:%s?transport=%s",
2886 type,
2887 IP.wrapIPv6(host),
2888 port,
2889 transport);
2890 }
2891
2892 final PeerConnection.IceServer.Builder iceServerBuilder =
2893 PeerConnection.IceServer.builder(uri);
2894 iceServerBuilder.setTlsCertPolicy(
2895 PeerConnection.TlsCertPolicy
2896 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2897 if (username != null && password != null) {
2898 iceServerBuilder.setUsername(username);
2899 iceServerBuilder.setPassword(password);
2900 } else if (Arrays.asList("turn", "turns").contains(type)) {
2901 // The WebRTC spec requires throwing an
2902 // InvalidAccessError when username (from libwebrtc
2903 // source coder)
2904 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2905 Log.d(
2906 Config.LOGTAG,
2907 id.account.getJid().asBareJid()
2908 + ": skipping "
2909 + type
2910 + "/"
2911 + transport
2912 + " without username and password");
2913 continue;
2914 }
2915 final PeerConnection.IceServer iceServer =
2916 iceServerBuilder.createIceServer();
2917 Log.d(
2918 Config.LOGTAG,
2919 id.account.getJid().asBareJid()
2920 + ": discovered ICE Server: "
2921 + iceServer);
2922 listBuilder.add(iceServer);
2923 }
2924 }
2925 }
2926 }
2927 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2928 if (iceServers.size() == 0) {
2929 Log.w(
2930 Config.LOGTAG,
2931 id.account.getJid().asBareJid()
2932 + ": no ICE server found "
2933 + response);
2934 }
2935 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2936 });
2937 } else {
2938 Log.w(
2939 Config.LOGTAG,
2940 id.account.getJid().asBareJid() + ": has no external service discovery");
2941 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2942 }
2943 }
2944
2945 private void finish() {
2946 if (isTerminated()) {
2947 this.cancelRingingTimeout();
2948 this.webRTCWrapper.verifyClosed();
2949 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2950 this.jingleConnectionManager.finishConnectionOrThrow(this);
2951 } else {
2952 throw new IllegalStateException(
2953 String.format("Unable to call finish from %s", this.state));
2954 }
2955 }
2956
2957 private void writeLogMessage(final State state) {
2958 final long duration = getCallDuration();
2959 if (state == State.TERMINATED_SUCCESS
2960 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2961 writeLogMessageSuccess(duration);
2962 } else {
2963 writeLogMessageMissed();
2964 }
2965 }
2966
2967 private void writeLogMessageSuccess(final long duration) {
2968 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2969 this.writeMessage();
2970 }
2971
2972 private void writeLogMessageMissed() {
2973 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2974 this.writeMessage();
2975 }
2976
2977 private void writeMessage() {
2978 final Conversational conversational = message.getConversation();
2979 if (conversational instanceof Conversation) {
2980 ((Conversation) conversational).add(this.message);
2981 xmppConnectionService.createMessageAsync(message);
2982 xmppConnectionService.updateConversationUi();
2983 } else {
2984 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2985 }
2986 }
2987
2988 public State getState() {
2989 return this.state;
2990 }
2991
2992 boolean isTerminated() {
2993 return TERMINATED.contains(this.state);
2994 }
2995
2996 public Optional<VideoTrack> getLocalVideoTrack() {
2997 return webRTCWrapper.getLocalVideoTrack();
2998 }
2999
3000 public Optional<VideoTrack> getRemoteVideoTrack() {
3001 return webRTCWrapper.getRemoteVideoTrack();
3002 }
3003
3004 public EglBase.Context getEglBaseContext() {
3005 return webRTCWrapper.getEglBaseContext();
3006 }
3007
3008 void setProposedMedia(final Set<Media> media) {
3009 this.proposedMedia = media;
3010 }
3011
3012 public void fireStateUpdate() {
3013 final RtpEndUserState endUserState = getEndUserState();
3014 xmppConnectionService.notifyJingleRtpConnectionUpdate(
3015 id.account, id.with, id.sessionId, endUserState);
3016 }
3017
3018 public boolean isSwitchToVideoAvailable() {
3019 final boolean prerequisite =
3020 Media.audioOnly(getMedia())
3021 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
3022 .contains(getEndUserState());
3023 return prerequisite && remoteHasVideoFeature();
3024 }
3025
3026 private boolean remoteHasVideoFeature() {
3027 final Contact contact = id.getContact();
3028 final Presence presence =
3029 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
3030 final ServiceDiscoveryResult serviceDiscoveryResult =
3031 presence == null ? null : presence.getServiceDiscoveryResult();
3032 final List<String> features =
3033 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
3034 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
3035 }
3036
3037 private interface OnIceServersDiscovered {
3038 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
3039 }
3040}