1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Joiner;
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.base.Stopwatch;
12import com.google.common.base.Strings;
13import com.google.common.base.Throwables;
14import com.google.common.collect.Collections2;
15import com.google.common.collect.ImmutableList;
16import com.google.common.collect.ImmutableMap;
17import com.google.common.collect.ImmutableMultimap;
18import com.google.common.collect.ImmutableSet;
19import com.google.common.collect.Iterables;
20import com.google.common.collect.Maps;
21import com.google.common.collect.Multimap;
22import com.google.common.collect.Sets;
23import com.google.common.primitives.Ints;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import org.webrtc.EglBase;
30import org.webrtc.IceCandidate;
31import org.webrtc.PeerConnection;
32import org.webrtc.VideoTrack;
33
34import java.util.Arrays;
35import java.util.Collection;
36import java.util.Collections;
37import java.util.LinkedList;
38import java.util.List;
39import java.util.Map;
40import java.util.Queue;
41import java.util.Set;
42import java.util.concurrent.ExecutionException;
43import java.util.concurrent.ScheduledFuture;
44import java.util.concurrent.TimeUnit;
45
46import eu.siacs.conversations.BuildConfig;
47import eu.siacs.conversations.Config;
48import eu.siacs.conversations.crypto.axolotl.AxolotlService;
49import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
50import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
51import eu.siacs.conversations.entities.Account;
52import eu.siacs.conversations.entities.Contact;
53import eu.siacs.conversations.entities.Conversation;
54import eu.siacs.conversations.entities.Conversational;
55import eu.siacs.conversations.entities.Message;
56import eu.siacs.conversations.entities.Presence;
57import eu.siacs.conversations.entities.RtpSessionStatus;
58import eu.siacs.conversations.entities.ServiceDiscoveryResult;
59import eu.siacs.conversations.services.AppRTCAudioManager;
60import eu.siacs.conversations.utils.IP;
61import eu.siacs.conversations.xml.Element;
62import eu.siacs.conversations.xml.Namespace;
63import eu.siacs.conversations.xmpp.Jid;
64import eu.siacs.conversations.xmpp.OnIqPacketReceived;
65import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
66import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
67import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
68import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
69import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
70import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
71import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
72import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
73import eu.siacs.conversations.xmpp.stanzas.IqPacket;
74import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
75
76public class JingleRtpConnection extends AbstractJingleConnection
77 implements WebRTCWrapper.EventCallback {
78
79 public static final List<State> STATES_SHOWING_ONGOING_CALL =
80 Arrays.asList(
81 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
82 private static final long BUSY_TIME_OUT = 30;
83 private static final List<State> TERMINATED =
84 Arrays.asList(
85 State.ACCEPTED,
86 State.REJECTED,
87 State.REJECTED_RACED,
88 State.RETRACTED,
89 State.RETRACTED_RACED,
90 State.TERMINATED_SUCCESS,
91 State.TERMINATED_DECLINED_OR_BUSY,
92 State.TERMINATED_CONNECTIVITY_ERROR,
93 State.TERMINATED_CANCEL_OR_TIMEOUT,
94 State.TERMINATED_APPLICATION_FAILURE,
95 State.TERMINATED_SECURITY_ERROR);
96
97 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
98
99 static {
100 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
101 new ImmutableMap.Builder<>();
102 transitionBuilder.put(
103 State.NULL,
104 ImmutableList.of(
105 State.PROPOSED,
106 State.SESSION_INITIALIZED,
107 State.TERMINATED_APPLICATION_FAILURE,
108 State.TERMINATED_SECURITY_ERROR));
109 transitionBuilder.put(
110 State.PROPOSED,
111 ImmutableList.of(
112 State.ACCEPTED,
113 State.PROCEED,
114 State.REJECTED,
115 State.RETRACTED,
116 State.TERMINATED_APPLICATION_FAILURE,
117 State.TERMINATED_SECURITY_ERROR,
118 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
119 // rebinds
120 ));
121 transitionBuilder.put(
122 State.PROCEED,
123 ImmutableList.of(
124 State.REJECTED_RACED,
125 State.RETRACTED_RACED,
126 State.SESSION_INITIALIZED_PRE_APPROVED,
127 State.TERMINATED_SUCCESS,
128 State.TERMINATED_APPLICATION_FAILURE,
129 State.TERMINATED_SECURITY_ERROR,
130 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
131 // bounces of the proceed message
132 ));
133 transitionBuilder.put(
134 State.SESSION_INITIALIZED,
135 ImmutableList.of(
136 State.SESSION_ACCEPTED,
137 State.TERMINATED_SUCCESS,
138 State.TERMINATED_DECLINED_OR_BUSY,
139 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
140 // and IQ timeouts
141 State.TERMINATED_CANCEL_OR_TIMEOUT,
142 State.TERMINATED_APPLICATION_FAILURE,
143 State.TERMINATED_SECURITY_ERROR));
144 transitionBuilder.put(
145 State.SESSION_INITIALIZED_PRE_APPROVED,
146 ImmutableList.of(
147 State.SESSION_ACCEPTED,
148 State.TERMINATED_SUCCESS,
149 State.TERMINATED_DECLINED_OR_BUSY,
150 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
151 // and IQ timeouts
152 State.TERMINATED_CANCEL_OR_TIMEOUT,
153 State.TERMINATED_APPLICATION_FAILURE,
154 State.TERMINATED_SECURITY_ERROR));
155 transitionBuilder.put(
156 State.SESSION_ACCEPTED,
157 ImmutableList.of(
158 State.TERMINATED_SUCCESS,
159 State.TERMINATED_DECLINED_OR_BUSY,
160 State.TERMINATED_CONNECTIVITY_ERROR,
161 State.TERMINATED_CANCEL_OR_TIMEOUT,
162 State.TERMINATED_APPLICATION_FAILURE,
163 State.TERMINATED_SECURITY_ERROR));
164 VALID_TRANSITIONS = transitionBuilder.build();
165 }
166
167 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
168 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
169 pendingIceCandidates = new LinkedList<>();
170 private final OmemoVerification omemoVerification = new OmemoVerification();
171 private final Message message;
172 private State state = State.NULL;
173 private Set<Media> proposedMedia;
174 private RtpContentMap initiatorRtpContentMap;
175 private RtpContentMap responderRtpContentMap;
176 private RtpContentMap incomingContentAdd;
177 private RtpContentMap outgoingContentAdd;
178 private IceUdpTransportInfo.Setup peerDtlsSetup;
179 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
180 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
181 private ScheduledFuture<?> ringingTimeoutFuture;
182
183 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
184 super(jingleConnectionManager, id, initiator);
185 final Conversation conversation =
186 jingleConnectionManager
187 .getXmppConnectionService()
188 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
189 this.message =
190 new Message(
191 conversation,
192 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
193 Message.TYPE_RTP_SESSION,
194 id.sessionId);
195 }
196
197 private static State reasonToState(Reason reason) {
198 switch (reason) {
199 case SUCCESS:
200 return State.TERMINATED_SUCCESS;
201 case DECLINE:
202 case BUSY:
203 return State.TERMINATED_DECLINED_OR_BUSY;
204 case CANCEL:
205 case TIMEOUT:
206 return State.TERMINATED_CANCEL_OR_TIMEOUT;
207 case SECURITY_ERROR:
208 return State.TERMINATED_SECURITY_ERROR;
209 case FAILED_APPLICATION:
210 case UNSUPPORTED_TRANSPORTS:
211 case UNSUPPORTED_APPLICATIONS:
212 return State.TERMINATED_APPLICATION_FAILURE;
213 default:
214 return State.TERMINATED_CONNECTIVITY_ERROR;
215 }
216 }
217
218 @Override
219 synchronized void deliverPacket(final JinglePacket jinglePacket) {
220 switch (jinglePacket.getAction()) {
221 case SESSION_INITIATE:
222 receiveSessionInitiate(jinglePacket);
223 break;
224 case TRANSPORT_INFO:
225 receiveTransportInfo(jinglePacket);
226 break;
227 case SESSION_ACCEPT:
228 receiveSessionAccept(jinglePacket);
229 break;
230 case SESSION_TERMINATE:
231 receiveSessionTerminate(jinglePacket);
232 break;
233 case CONTENT_ADD:
234 receiveContentAdd(jinglePacket);
235 break;
236 case CONTENT_ACCEPT:
237 receiveContentAccept(jinglePacket);
238 break;
239 case CONTENT_REJECT:
240 receiveContentReject(jinglePacket);
241 break;
242 case CONTENT_REMOVE:
243 receiveContentRemove(jinglePacket);
244 break;
245 case CONTENT_MODIFY:
246 receiveContentModify(jinglePacket);
247 break;
248 default:
249 respondOk(jinglePacket);
250 Log.d(
251 Config.LOGTAG,
252 String.format(
253 "%s: received unhandled jingle action %s",
254 id.account.getJid().asBareJid(), jinglePacket.getAction()));
255 break;
256 }
257 }
258
259 @Override
260 synchronized void notifyRebound() {
261 if (isTerminated()) {
262 return;
263 }
264 webRTCWrapper.close();
265 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
266 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
267 }
268 if (isInState(
269 State.SESSION_INITIALIZED,
270 State.SESSION_INITIALIZED_PRE_APPROVED,
271 State.SESSION_ACCEPTED)) {
272 // we might have already changed resources (full jid) at this point; so this might not
273 // even reach the other party
274 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
275 } else {
276 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
277 finish();
278 }
279 }
280
281 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
282 respondOk(jinglePacket);
283 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
284 final State previous = this.state;
285 Log.d(
286 Config.LOGTAG,
287 id.account.getJid().asBareJid()
288 + ": received session terminate reason="
289 + wrapper.reason
290 + "("
291 + Strings.nullToEmpty(wrapper.text)
292 + ") while in state "
293 + previous);
294 if (TERMINATED.contains(previous)) {
295 Log.d(
296 Config.LOGTAG,
297 id.account.getJid().asBareJid()
298 + ": ignoring session terminate because already in "
299 + previous);
300 return;
301 }
302 webRTCWrapper.close();
303 final State target = reasonToState(wrapper.reason);
304 transitionOrThrow(target);
305 writeLogMessage(target);
306 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
307 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
308 }
309 finish();
310 }
311
312 private void receiveTransportInfo(final JinglePacket jinglePacket) {
313 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
314 // INITIALIZED only after transport-info has been received
315 if (isInState(
316 State.NULL,
317 State.PROCEED,
318 State.SESSION_INITIALIZED,
319 State.SESSION_INITIALIZED_PRE_APPROVED,
320 State.SESSION_ACCEPTED)) {
321 final RtpContentMap contentMap;
322 try {
323 contentMap = RtpContentMap.of(jinglePacket);
324 } catch (final IllegalArgumentException | NullPointerException e) {
325 Log.d(
326 Config.LOGTAG,
327 id.account.getJid().asBareJid()
328 + ": improperly formatted contents; ignoring",
329 e);
330 respondOk(jinglePacket);
331 return;
332 }
333 receiveTransportInfo(jinglePacket, contentMap);
334 } else {
335 if (isTerminated()) {
336 respondOk(jinglePacket);
337 Log.d(
338 Config.LOGTAG,
339 id.account.getJid().asBareJid()
340 + ": ignoring out-of-order transport info; we where already terminated");
341 } else {
342 Log.d(
343 Config.LOGTAG,
344 id.account.getJid().asBareJid()
345 + ": received transport info while in state="
346 + this.state);
347 terminateWithOutOfOrder(jinglePacket);
348 }
349 }
350 }
351
352 private void receiveTransportInfo(
353 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
354 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
355 contentMap.contents.entrySet();
356 final RtpContentMap remote = getRemoteContentMap();
357 final Set<String> remoteContentIds = remote == null ? Collections.emptySet() : remote.contents.keySet();
358 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
359 Log.d(Config.LOGTAG,"received transport-info for unknown contents "+contentMap.contents.keySet()+" (known: "+remoteContentIds+")");
360 respondOk(jinglePacket);
361 pendingIceCandidates.addAll(candidates);
362 return;
363 }
364 if (this.state != State.SESSION_ACCEPTED) {
365 Log.d(Config.LOGTAG,"received transport-info prematurely. adding to backlog");
366 respondOk(jinglePacket);
367 pendingIceCandidates.addAll(candidates);
368 return;
369 }
370 // zero candidates + modified credentials are an ICE restart offer
371 if (checkForIceRestart(jinglePacket, contentMap)) {
372 return;
373 }
374 respondOk(jinglePacket);
375 try {
376 processCandidates(candidates);
377 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
378 Log.w(
379 Config.LOGTAG,
380 id.account.getJid().asBareJid()
381 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
382 }
383 }
384
385 private void receiveContentAdd(final JinglePacket jinglePacket) {
386 final RtpContentMap modification;
387 try {
388 modification = RtpContentMap.of(jinglePacket);
389 modification.requireContentDescriptions();
390 } catch (final RuntimeException e) {
391 Log.d(
392 Config.LOGTAG,
393 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
394 Throwables.getRootCause(e));
395 respondOk(jinglePacket);
396 webRTCWrapper.close();
397 sendSessionTerminate(Reason.of(e), e.getMessage());
398 return;
399 }
400 if (isInState(State.SESSION_ACCEPTED)) {
401 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
402 final ListenableFuture<RtpContentMap> future =
403 receiveRtpContentMap(
404 modification, this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
405 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
406 @Override
407 public void onSuccess(final RtpContentMap rtpContentMap) {
408 receiveContentAdd(jinglePacket, rtpContentMap);
409 }
410
411 @Override
412 public void onFailure(@NonNull Throwable throwable) {
413 respondOk(jinglePacket);
414 final Throwable rootCause = Throwables.getRootCause(throwable);
415 Log.d(
416 Config.LOGTAG,
417 id.account.getJid().asBareJid()
418 + ": improperly formatted contents in content-add",
419 throwable);
420 webRTCWrapper.close();
421 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
422 }
423 }, MoreExecutors.directExecutor());
424 } else {
425 terminateWithOutOfOrder(jinglePacket);
426 }
427 }
428
429 private void receiveContentAdd(
430 final JinglePacket jinglePacket, final RtpContentMap modification) {
431 final RtpContentMap remote = getRemoteContentMap();
432 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
433 respondOk(jinglePacket);
434 this.webRTCWrapper.close();
435 sendSessionTerminate(
436 Reason.FAILED_APPLICATION,
437 String.format(
438 "contents with names %s already exists",
439 Joiner.on(", ").join(modification.getNames())));
440 return;
441 }
442 final ContentAddition contentAddition =
443 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
444
445 final RtpContentMap outgoing = this.outgoingContentAdd;
446 final Set<ContentAddition.Summary> outgoingContentAddSummary =
447 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
448
449 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
450 if (isInitiator()) {
451 Log.d(
452 Config.LOGTAG,
453 id.getAccount().getJid().asBareJid()
454 + ": respond with tie break to matching content-add offer");
455 respondWithTieBreak(jinglePacket);
456 } else {
457 Log.d(
458 Config.LOGTAG,
459 id.getAccount().getJid().asBareJid()
460 + ": automatically accept matching content-add offer");
461 acceptContentAdd(contentAddition.summary, modification);
462 }
463 return;
464 }
465
466 // once we can display multiple video tracks we can be more loose with this condition
467 // theoretically it should also be fine to automatically accept audio only contents
468 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
469 Log.d(
470 Config.LOGTAG,
471 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
472 this.incomingContentAdd = modification;
473 respondOk(jinglePacket);
474 updateEndUserState();
475 } else {
476 respondOk(jinglePacket);
477 // TODO do we want to add a reason?
478 rejectContentAdd(modification);
479 }
480 }
481
482 private void receiveContentAccept(final JinglePacket jinglePacket) {
483 final RtpContentMap receivedContentAccept;
484 try {
485 receivedContentAccept = RtpContentMap.of(jinglePacket);
486 receivedContentAccept.requireContentDescriptions();
487 } catch (final RuntimeException e) {
488 Log.d(
489 Config.LOGTAG,
490 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
491 Throwables.getRootCause(e));
492 respondOk(jinglePacket);
493 webRTCWrapper.close();
494 sendSessionTerminate(Reason.of(e), e.getMessage());
495 return;
496 }
497
498 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
499 if (outgoingContentAdd == null) {
500 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
501 terminateWithOutOfOrder(jinglePacket);
502 return;
503 }
504 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
505 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
506 this.outgoingContentAdd = null;
507 respondOk(jinglePacket);
508 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
509 final ListenableFuture<RtpContentMap> future =
510 receiveRtpContentMap(
511 receivedContentAccept, this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
512 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
513 @Override
514 public void onSuccess(final RtpContentMap result) {
515 receiveContentAccept(result);
516 }
517
518 @Override
519 public void onFailure(@NonNull final Throwable throwable) {
520 webRTCWrapper.close();
521 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
522 }
523 }, MoreExecutors.directExecutor());
524 } else {
525 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
526 terminateWithOutOfOrder(jinglePacket);
527 }
528 }
529
530 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
531 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
532 final RtpContentMap modifiedContentMap =
533 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
534
535 setRemoteContentMap(modifiedContentMap);
536
537 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
538
539 final org.webrtc.SessionDescription sdp =
540 new org.webrtc.SessionDescription(
541 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
542
543 try {
544 this.webRTCWrapper.setRemoteDescription(sdp).get();
545 } catch (final Exception e) {
546 final Throwable cause = Throwables.getRootCause(e);
547 Log.d(
548 Config.LOGTAG,
549 id.getAccount().getJid().asBareJid()
550 + ": unable to set remote description after receiving content-accept",
551 cause);
552 webRTCWrapper.close();
553 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
554 return;
555 }
556 Log.d(
557 Config.LOGTAG,
558 id.getAccount().getJid().asBareJid()
559 + ": remote has accepted content-add "
560 + ContentAddition.summary(receivedContentAccept));
561 processCandidates(receivedContentAccept.contents.entrySet());
562 updateEndUserState();
563 }
564
565 private void receiveContentModify(final JinglePacket jinglePacket) {
566 if (this.state != State.SESSION_ACCEPTED) {
567 terminateWithOutOfOrder(jinglePacket);
568 return;
569 }
570 final Map<String, Content.Senders> modification =
571 Maps.transformEntries(
572 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
573 final boolean isInitiator = isInitiator();
574 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
575 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
576 final Set<String> currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet();
577 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
578 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
579 respondOk(jinglePacket);
580 final RtpContentMap modifiedContentMap;
581 try {
582 modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification);
583 } catch (final IllegalArgumentException e) {
584 webRTCWrapper.close();
585 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
586 return;
587 }
588 this.outgoingContentAdd = modifiedContentMap;
589 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add");
590 } else if (remoteContentMap != null && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
591 respondOk(jinglePacket);
592 final RtpContentMap modifiedRemoteContentMap;
593 try {
594 modifiedRemoteContentMap = remoteContentMap.modifiedSendersChecked(isInitiator, modification);
595 } catch (final IllegalArgumentException e) {
596 webRTCWrapper.close();
597 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
598 return;
599 }
600 final SessionDescription offer;
601 try {
602 offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator());
603 } catch (final IllegalArgumentException | NullPointerException e) {
604 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-modify to SDP", e);
605 webRTCWrapper.close();
606 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
607 return;
608 }
609 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": auto accepting content-modification");
610 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
611 } else {
612 Log.d(Config.LOGTAG,"received unsupported content modification "+modification);
613 respondWithItemNotFound(jinglePacket);
614 }
615 }
616
617 private void autoAcceptContentModify(final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
618 this.setRemoteContentMap(modifiedRemoteContentMap);
619 final org.webrtc.SessionDescription sdp =
620 new org.webrtc.SessionDescription(
621 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
622 try {
623 this.webRTCWrapper.setRemoteDescription(sdp).get();
624 // auto accept is only done when we already have tracks
625 final SessionDescription answer = setLocalSessionDescription();
626 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
627 modifyLocalContentMap(rtpContentMap);
628 // we do not need to send an answer but do we have to resend the candidates currently in SDP?
629 //resendCandidatesFromSdp(answer);
630 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
631 } catch (final Exception e) {
632 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
633 webRTCWrapper.close();
634 sendSessionTerminate(Reason.FAILED_APPLICATION);
635 }
636 }
637
638 private void resendCandidatesFromSdp(final SessionDescription answer) {
639 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
640 for(final SessionDescription.Media media : answer.media) {
641 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
642 if (Strings.isNullOrEmpty(mid)) {
643 continue;
644 }
645 for(final String sdpCandidate : media.attributes.get("candidate")) {
646 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
647 if (candidate != null) {
648 candidateBuilder.put(mid,candidate);
649 }
650 }
651 }
652 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build();
653 sendTransportInfo(candidates);
654 }
655
656 private void receiveContentReject(final JinglePacket jinglePacket) {
657 final RtpContentMap receivedContentReject;
658 try {
659 receivedContentReject = RtpContentMap.of(jinglePacket);
660 } catch (final RuntimeException e) {
661 Log.d(
662 Config.LOGTAG,
663 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
664 Throwables.getRootCause(e));
665 respondOk(jinglePacket);
666 this.webRTCWrapper.close();
667 sendSessionTerminate(Reason.of(e), e.getMessage());
668 return;
669 }
670
671 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
672 if (outgoingContentAdd == null) {
673 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
674 terminateWithOutOfOrder(jinglePacket);
675 return;
676 }
677 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
678 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
679 this.outgoingContentAdd = null;
680 respondOk(jinglePacket);
681 Log.d(Config.LOGTAG,jinglePacket.toString());
682 receiveContentReject(ourSummary);
683 } else {
684 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
685 terminateWithOutOfOrder(jinglePacket);
686 }
687 }
688
689 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
690 try {
691 this.webRTCWrapper.removeTrack(Media.VIDEO);
692 final RtpContentMap localContentMap = customRollback();
693 modifyLocalContentMap(localContentMap);
694 } catch (final Exception e) {
695 final Throwable cause = Throwables.getRootCause(e);
696 Log.d(
697 Config.LOGTAG,
698 id.getAccount().getJid().asBareJid()
699 + ": unable to rollback local description after receiving content-reject",
700 cause);
701 webRTCWrapper.close();
702 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
703 return;
704 }
705 Log.d(
706 Config.LOGTAG,
707 id.getAccount().getJid().asBareJid()
708 + ": remote has rejected our content-add "
709 + summary);
710 }
711
712 private void receiveContentRemove(final JinglePacket jinglePacket) {
713 final RtpContentMap receivedContentRemove;
714 try {
715 receivedContentRemove = RtpContentMap.of(jinglePacket);
716 receivedContentRemove.requireContentDescriptions();
717 } catch (final RuntimeException e) {
718 Log.d(
719 Config.LOGTAG,
720 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
721 Throwables.getRootCause(e));
722 respondOk(jinglePacket);
723 this.webRTCWrapper.close();
724 sendSessionTerminate(Reason.of(e), e.getMessage());
725 return;
726 }
727 respondOk(jinglePacket);
728 receiveContentRemove(receivedContentRemove);
729 }
730
731 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
732 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
733 final Set<ContentAddition.Summary> contentAddSummary =
734 incomingContentAdd == null
735 ? Collections.emptySet()
736 : ContentAddition.summary(incomingContentAdd);
737 final Set<ContentAddition.Summary> removeSummary =
738 ContentAddition.summary(receivedContentRemove);
739 if (contentAddSummary.equals(removeSummary)) {
740 this.incomingContentAdd = null;
741 updateEndUserState();
742 } else {
743 webRTCWrapper.close();
744 sendSessionTerminate(
745 Reason.FAILED_APPLICATION,
746 String.format(
747 "%s only supports %s as a means to retract a not yet accepted %s",
748 BuildConfig.APP_NAME,
749 JinglePacket.Action.CONTENT_REMOVE,
750 JinglePacket.Action.CONTENT_ADD));
751 }
752 }
753
754 public synchronized void retractContentAdd() {
755 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
756 if (outgoingContentAdd == null) {
757 throw new IllegalStateException("Not outgoing content add");
758 }
759 try {
760 webRTCWrapper.removeTrack(Media.VIDEO);
761 final RtpContentMap localContentMap = customRollback();
762 modifyLocalContentMap(localContentMap);
763 } catch (final Exception e) {
764 final Throwable cause = Throwables.getRootCause(e);
765 Log.d(
766 Config.LOGTAG,
767 id.getAccount().getJid().asBareJid()
768 + ": unable to rollback local description after trying to retract content-add",
769 cause);
770 webRTCWrapper.close();
771 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
772 return;
773 }
774 this.outgoingContentAdd = null;
775 final JinglePacket retract =
776 outgoingContentAdd
777 .toStub()
778 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
779 this.send(retract);
780 Log.d(
781 Config.LOGTAG,
782 id.getAccount().getJid()
783 + ": retract content-add "
784 + ContentAddition.summary(outgoingContentAdd));
785 }
786
787 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
788 final SessionDescription sdp = setLocalSessionDescription();
789 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
790 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
791 this.webRTCWrapper
792 .setRemoteDescription(
793 new org.webrtc.SessionDescription(
794 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
795 .get();
796 return localRtpContentMap;
797 }
798
799 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
800 final RtpContentMap currentRemote = getRemoteContentMap();
801 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
802 if (diff.isEmpty()) {
803 throw new IllegalStateException(
804 "Unexpected rollback condition. No difference between local and remote");
805 }
806 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
807 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
808 final RtpContentMap nextRemote =
809 currentRemote.addContent(
810 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
811 return SessionDescription.of(nextRemote, !isInitiator());
812 }
813 throw new IllegalStateException(
814 "Unexpected rollback condition. Senders were not uniformly none");
815 }
816
817 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
818 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
819 if (incomingContentAdd == null) {
820 throw new IllegalStateException("No incoming content add");
821 }
822
823 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
824 this.incomingContentAdd = null;
825 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
826 Log.d(Config.LOGTAG,"senders of incoming content-add: "+senders);
827 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
828 Log.d(Config.LOGTAG,"content addition is receive only. we want to upgrade to 'both'");
829 final RtpContentMap modifiedSenders = incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
830 final JinglePacket proposedContentModification = modifiedSenders.toStub().toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
831 proposedContentModification.setTo(id.with);
832 xmppConnectionService.sendIqPacket(id.account, proposedContentModification, (account, response) -> {
833 if (response.getType() == IqPacket.TYPE.RESULT) {
834 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has accepted our upgrade to senders=both");
835 acceptContentAdd(ContentAddition.summary(modifiedSenders), modifiedSenders);
836 } else {
837 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has rejected our upgrade to senders=both");
838 acceptContentAdd(contentAddition, incomingContentAdd);
839 }
840 });
841 }
842 } else {
843 throw new IllegalStateException("Accepted content add does not match pending content-add");
844 }
845 }
846
847 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
848 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
849 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
850 this.setRemoteContentMap(modifiedContentMap);
851
852 final SessionDescription offer;
853 try {
854 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
855 } catch (final IllegalArgumentException | NullPointerException e) {
856 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
857 webRTCWrapper.close();
858 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
859 return;
860 }
861 this.incomingContentAdd = null;
862 acceptContentAdd(contentAddition, offer);
863 }
864
865 private void acceptContentAdd(
866 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
867 final org.webrtc.SessionDescription sdp =
868 new org.webrtc.SessionDescription(
869 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
870 try {
871 this.webRTCWrapper.setRemoteDescription(sdp).get();
872
873 // TODO add tracks for 'media' where contentAddition.senders matches
874
875 // TODO if senders.sending(isInitiator())
876
877 this.webRTCWrapper.addTrack(Media.VIDEO);
878
879 // TODO add additional transceivers for recv only cases
880
881 final SessionDescription answer = setLocalSessionDescription();
882 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
883
884 final RtpContentMap contentAcceptMap =
885 rtpContentMap.toContentModification(
886 Collections2.transform(contentAddition, ca -> ca.name));
887
888 Log.d(
889 Config.LOGTAG,
890 id.getAccount().getJid().asBareJid()
891 + ": sending content-accept "
892 + ContentAddition.summary(contentAcceptMap));
893
894 addIceCandidatesFromBlackLog();
895
896 modifyLocalContentMap(rtpContentMap);
897 final ListenableFuture<RtpContentMap> future = prepareOutgoingContentMap(contentAcceptMap);
898 Futures.addCallback(
899 future,
900 new FutureCallback<RtpContentMap>() {
901 @Override
902 public void onSuccess(final RtpContentMap rtpContentMap) {
903 sendContentAccept(rtpContentMap);
904 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
905 }
906
907 @Override
908 public void onFailure(@NonNull final Throwable throwable) {
909 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
910 }
911 },
912 MoreExecutors.directExecutor());
913 } catch (final Exception e) {
914 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
915 webRTCWrapper.close();
916 sendSessionTerminate(Reason.FAILED_APPLICATION);
917 }
918 }
919
920 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
921 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
922 send(jinglePacket);
923 }
924
925 public synchronized void rejectContentAdd() {
926 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
927 if (incomingContentAdd == null) {
928 throw new IllegalStateException("No incoming content add");
929 }
930 this.incomingContentAdd = null;
931 updateEndUserState();
932 rejectContentAdd(incomingContentAdd);
933 }
934
935 private void rejectContentAdd(final RtpContentMap contentMap) {
936 final JinglePacket jinglePacket =
937 contentMap
938 .toStub()
939 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
940 Log.d(
941 Config.LOGTAG,
942 id.getAccount().getJid().asBareJid()
943 + ": rejecting content "
944 + ContentAddition.summary(contentMap));
945 send(jinglePacket);
946 }
947
948 private boolean checkForIceRestart(
949 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
950 final RtpContentMap existing = getRemoteContentMap();
951 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
952 final IceUdpTransportInfo.Credentials newCredentials;
953 try {
954 existingCredentials = existing.getCredentials();
955 newCredentials = rtpContentMap.getDistinctCredentials();
956 } catch (final IllegalStateException e) {
957 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
958 return false;
959 }
960 if (existingCredentials.contains(newCredentials)) {
961 return false;
962 }
963 // TODO an alternative approach is to check if we already got an iq result to our
964 // ICE-restart
965 // and if that's the case we are seeing an answer.
966 // This might be more spec compliant but also more error prone potentially
967 final boolean isSignalStateStable = this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
968 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
969 final boolean isOffer = rtpContentMap.emptyCandidates();
970 final RtpContentMap restartContentMap;
971 try {
972 if (isOffer) {
973 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
974 restartContentMap =
975 existing.modifiedCredentials(
976 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
977 } else {
978 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
979 Log.d(
980 Config.LOGTAG,
981 "received confirmation of ICE restart"
982 + newCredentials
983 + " peer_setup="
984 + setup);
985 // DTLS setup attribute needs to be rewritten to reflect current peer state
986 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
987 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
988 }
989 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
990 return isOffer;
991 } else {
992 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
993 respondWithTieBreak(jinglePacket);
994 return true;
995 }
996 } catch (final Exception exception) {
997 respondOk(jinglePacket);
998 final Throwable rootCause = Throwables.getRootCause(exception);
999 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
1000 // If this happens a termination is already in progress
1001 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
1002 return true;
1003 }
1004 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
1005 webRTCWrapper.close();
1006 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1007 return true;
1008 }
1009 }
1010
1011 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
1012 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
1013 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
1014 throw new IllegalStateException("Invalid peer setup");
1015 }
1016 return peerSetup;
1017 }
1018
1019 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
1020 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
1021 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
1022 }
1023 this.peerDtlsSetup = setup;
1024 }
1025
1026 private boolean applyIceRestart(
1027 final JinglePacket jinglePacket,
1028 final RtpContentMap restartContentMap,
1029 final boolean isOffer)
1030 throws ExecutionException, InterruptedException {
1031 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
1032 final org.webrtc.SessionDescription.Type type =
1033 isOffer
1034 ? org.webrtc.SessionDescription.Type.OFFER
1035 : org.webrtc.SessionDescription.Type.ANSWER;
1036 org.webrtc.SessionDescription sdp =
1037 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1038 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1039 if (isInitiator()) {
1040 // We ignore the offer and respond with tie-break. This will clause the responder
1041 // not to apply the content map
1042 return false;
1043 }
1044 }
1045 webRTCWrapper.setRemoteDescription(sdp).get();
1046 setRemoteContentMap(restartContentMap);
1047 if (isOffer) {
1048 final SessionDescription localSessionDescription = setLocalSessionDescription();
1049 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1050 // We need to respond OK before sending any candidates
1051 respondOk(jinglePacket);
1052 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1053 } else {
1054 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1055 }
1056 return true;
1057 }
1058
1059 private void processCandidates(
1060 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
1061 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
1062 processCandidate(content);
1063 }
1064 }
1065
1066 private void processCandidate(
1067 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
1068 final RtpContentMap rtpContentMap = getRemoteContentMap();
1069 final List<String> indices = toIdentificationTags(rtpContentMap);
1070 final String sdpMid = content.getKey(); // aka content name
1071 final IceUdpTransportInfo transport = content.getValue().transport;
1072 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1073
1074 // TODO check that credentials remained the same
1075
1076 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1077 final String sdp;
1078 try {
1079 sdp = candidate.toSdpAttribute(credentials.ufrag);
1080 } catch (final IllegalArgumentException e) {
1081 Log.d(
1082 Config.LOGTAG,
1083 id.account.getJid().asBareJid()
1084 + ": ignoring invalid ICE candidate "
1085 + e.getMessage());
1086 continue;
1087 }
1088 final int mLineIndex = indices.indexOf(sdpMid);
1089 if (mLineIndex < 0) {
1090 Log.w(
1091 Config.LOGTAG,
1092 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1093 }
1094 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1095 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1096 this.webRTCWrapper.addIceCandidate(iceCandidate);
1097 }
1098 }
1099
1100 private RtpContentMap getRemoteContentMap() {
1101 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1102 }
1103
1104 private RtpContentMap getLocalContentMap() {
1105 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1106 }
1107
1108 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1109 final Group originalGroup = rtpContentMap.group;
1110 final List<String> identificationTags =
1111 originalGroup == null
1112 ? rtpContentMap.getNames()
1113 : originalGroup.getIdentificationTags();
1114 if (identificationTags.size() == 0) {
1115 Log.w(
1116 Config.LOGTAG,
1117 id.account.getJid().asBareJid()
1118 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1119 }
1120 return identificationTags;
1121 }
1122
1123 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1124 final JinglePacket jinglePacket, final boolean expectVerification) {
1125 try {
1126 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1127 } catch (final Exception e) {
1128 return Futures.immediateFailedFuture(e);
1129 }
1130 }
1131 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final RtpContentMap receivedContentMap, final boolean expectVerification) {
1132 Log.d(
1133 Config.LOGTAG,
1134 "receiveRtpContentMap("
1135 + receivedContentMap.getClass().getSimpleName()
1136 + ",expectVerification="
1137 + expectVerification
1138 + ")");
1139 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1140 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1141 id.account
1142 .getAxolotlService()
1143 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1144 return Futures.transform(
1145 future,
1146 omemoVerifiedPayload -> {
1147 // TODO test if an exception here triggers a correct abort
1148 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1149 Log.d(
1150 Config.LOGTAG,
1151 id.account.getJid().asBareJid()
1152 + ": received verifiable DTLS fingerprint via "
1153 + omemoVerification);
1154 return omemoVerifiedPayload.getPayload();
1155 },
1156 MoreExecutors.directExecutor());
1157 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1158 return Futures.immediateFailedFuture(
1159 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1160 } else {
1161 return Futures.immediateFuture(receivedContentMap);
1162 }
1163 }
1164
1165 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1166 if (isInitiator()) {
1167 Log.d(
1168 Config.LOGTAG,
1169 String.format(
1170 "%s: received session-initiate even though we were initiating",
1171 id.account.getJid().asBareJid()));
1172 if (isTerminated()) {
1173 Log.d(
1174 Config.LOGTAG,
1175 String.format(
1176 "%s: got a reason to terminate with out-of-order. but already in state %s",
1177 id.account.getJid().asBareJid(), getState()));
1178 respondWithOutOfOrder(jinglePacket);
1179 } else {
1180 terminateWithOutOfOrder(jinglePacket);
1181 }
1182 return;
1183 }
1184 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1185 Futures.addCallback(
1186 future,
1187 new FutureCallback<RtpContentMap>() {
1188 @Override
1189 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1190 receiveSessionInitiate(jinglePacket, rtpContentMap);
1191 }
1192
1193 @Override
1194 public void onFailure(@NonNull final Throwable throwable) {
1195 respondOk(jinglePacket);
1196 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1197 }
1198 },
1199 MoreExecutors.directExecutor());
1200 }
1201
1202 private void receiveSessionInitiate(
1203 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1204 try {
1205 contentMap.requireContentDescriptions();
1206 contentMap.requireDTLSFingerprint(true);
1207 } catch (final RuntimeException e) {
1208 Log.d(
1209 Config.LOGTAG,
1210 id.account.getJid().asBareJid() + ": improperly formatted contents",
1211 Throwables.getRootCause(e));
1212 respondOk(jinglePacket);
1213 sendSessionTerminate(Reason.of(e), e.getMessage());
1214 return;
1215 }
1216 Log.d(
1217 Config.LOGTAG,
1218 "processing session-init with " + contentMap.contents.size() + " contents");
1219 final State target;
1220 if (this.state == State.PROCEED) {
1221 Preconditions.checkState(
1222 proposedMedia != null && proposedMedia.size() > 0,
1223 "proposed media must be set when processing pre-approved session-initiate");
1224 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1225 sendSessionTerminate(
1226 Reason.SECURITY_ERROR,
1227 String.format(
1228 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1229 this.proposedMedia, contentMap.getMedia()));
1230 return;
1231 }
1232 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1233 } else {
1234 target = State.SESSION_INITIALIZED;
1235 }
1236 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1237 respondOk(jinglePacket);
1238 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1239 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1240 Log.d(
1241 Config.LOGTAG,
1242 id.account.getJid().asBareJid()
1243 + ": automatically accepting session-initiate");
1244 sendSessionAccept();
1245 } else {
1246 Log.d(
1247 Config.LOGTAG,
1248 id.account.getJid().asBareJid()
1249 + ": received not pre-approved session-initiate. start ringing");
1250 startRinging();
1251 }
1252 } else {
1253 Log.d(
1254 Config.LOGTAG,
1255 String.format(
1256 "%s: received session-initiate while in state %s",
1257 id.account.getJid().asBareJid(), state));
1258 terminateWithOutOfOrder(jinglePacket);
1259 }
1260 }
1261
1262 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1263 if (!isInitiator()) {
1264 Log.d(
1265 Config.LOGTAG,
1266 String.format(
1267 "%s: received session-accept even though we were responding",
1268 id.account.getJid().asBareJid()));
1269 terminateWithOutOfOrder(jinglePacket);
1270 return;
1271 }
1272 final ListenableFuture<RtpContentMap> future =
1273 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1274 Futures.addCallback(
1275 future,
1276 new FutureCallback<RtpContentMap>() {
1277 @Override
1278 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1279 receiveSessionAccept(jinglePacket, rtpContentMap);
1280 }
1281
1282 @Override
1283 public void onFailure(@NonNull final Throwable throwable) {
1284 respondOk(jinglePacket);
1285 Log.d(
1286 Config.LOGTAG,
1287 id.account.getJid().asBareJid()
1288 + ": improperly formatted contents in session-accept",
1289 throwable);
1290 webRTCWrapper.close();
1291 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1292 }
1293 },
1294 MoreExecutors.directExecutor());
1295 }
1296
1297 private void receiveSessionAccept(
1298 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1299 try {
1300 contentMap.requireContentDescriptions();
1301 contentMap.requireDTLSFingerprint();
1302 } catch (final RuntimeException e) {
1303 respondOk(jinglePacket);
1304 Log.d(
1305 Config.LOGTAG,
1306 id.account.getJid().asBareJid()
1307 + ": improperly formatted contents in session-accept",
1308 e);
1309 webRTCWrapper.close();
1310 sendSessionTerminate(Reason.of(e), e.getMessage());
1311 return;
1312 }
1313 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1314 if (!initiatorMedia.equals(contentMap.getMedia())) {
1315 sendSessionTerminate(
1316 Reason.SECURITY_ERROR,
1317 String.format(
1318 "Your session-included included media %s but our session-initiate was %s",
1319 this.proposedMedia, contentMap.getMedia()));
1320 return;
1321 }
1322 Log.d(
1323 Config.LOGTAG,
1324 "processing session-accept with " + contentMap.contents.size() + " contents");
1325 if (transition(State.SESSION_ACCEPTED)) {
1326 respondOk(jinglePacket);
1327 receiveSessionAccept(contentMap);
1328 } else {
1329 Log.d(
1330 Config.LOGTAG,
1331 String.format(
1332 "%s: received session-accept while in state %s",
1333 id.account.getJid().asBareJid(), state));
1334 respondOk(jinglePacket);
1335 }
1336 }
1337
1338 private void receiveSessionAccept(final RtpContentMap contentMap) {
1339 this.responderRtpContentMap = contentMap;
1340 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1341 final SessionDescription sessionDescription;
1342 try {
1343 sessionDescription = SessionDescription.of(contentMap, false);
1344 } catch (final IllegalArgumentException | NullPointerException e) {
1345 Log.d(
1346 Config.LOGTAG,
1347 id.account.getJid().asBareJid()
1348 + ": unable convert offer from session-accept to SDP",
1349 e);
1350 webRTCWrapper.close();
1351 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1352 return;
1353 }
1354 final org.webrtc.SessionDescription answer =
1355 new org.webrtc.SessionDescription(
1356 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1357 try {
1358 this.webRTCWrapper.setRemoteDescription(answer).get();
1359 } catch (final Exception e) {
1360 Log.d(
1361 Config.LOGTAG,
1362 id.account.getJid().asBareJid()
1363 + ": unable to set remote description after receiving session-accept",
1364 Throwables.getRootCause(e));
1365 webRTCWrapper.close();
1366 sendSessionTerminate(
1367 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1368 return;
1369 }
1370 processCandidates(contentMap.contents.entrySet());
1371 }
1372
1373 private void sendSessionAccept() {
1374 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1375 if (rtpContentMap == null) {
1376 throw new IllegalStateException("initiator RTP Content Map has not been set");
1377 }
1378 final SessionDescription offer;
1379 try {
1380 offer = SessionDescription.of(rtpContentMap, true);
1381 } catch (final IllegalArgumentException | NullPointerException e) {
1382 Log.d(
1383 Config.LOGTAG,
1384 id.account.getJid().asBareJid()
1385 + ": unable convert offer from session-initiate to SDP",
1386 e);
1387 webRTCWrapper.close();
1388 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1389 return;
1390 }
1391 sendSessionAccept(rtpContentMap.getMedia(), offer);
1392 }
1393
1394 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1395 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1396 }
1397
1398 private synchronized void sendSessionAccept(
1399 final Set<Media> media,
1400 final SessionDescription offer,
1401 final List<PeerConnection.IceServer> iceServers) {
1402 if (isTerminated()) {
1403 Log.w(
1404 Config.LOGTAG,
1405 id.account.getJid().asBareJid()
1406 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1407 return;
1408 }
1409 try {
1410 setupWebRTC(media, iceServers);
1411 } catch (final WebRTCWrapper.InitializationException e) {
1412 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1413 webRTCWrapper.close();
1414 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1415 return;
1416 }
1417 final org.webrtc.SessionDescription sdp =
1418 new org.webrtc.SessionDescription(
1419 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1420 try {
1421 this.webRTCWrapper.setRemoteDescription(sdp).get();
1422 addIceCandidatesFromBlackLog();
1423 org.webrtc.SessionDescription webRTCSessionDescription =
1424 this.webRTCWrapper.setLocalDescription().get();
1425 prepareSessionAccept(webRTCSessionDescription);
1426 } catch (final Exception e) {
1427 failureToAcceptSession(e);
1428 }
1429 }
1430
1431 private void failureToAcceptSession(final Throwable throwable) {
1432 if (isTerminated()) {
1433 return;
1434 }
1435 final Throwable rootCause = Throwables.getRootCause(throwable);
1436 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1437 webRTCWrapper.close();
1438 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1439 }
1440
1441 private void failureToPerformAction(final JinglePacket.Action action, final Throwable throwable) {
1442 if (isTerminated()) {
1443 return;
1444 }
1445 final Throwable rootCause = Throwables.getRootCause(throwable);
1446 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1447 webRTCWrapper.close();
1448 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1449 }
1450
1451 private void addIceCandidatesFromBlackLog() {
1452 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1453 while ((foo = this.pendingIceCandidates.poll()) != null) {
1454 processCandidate(foo);
1455 Log.d(
1456 Config.LOGTAG,
1457 id.account.getJid().asBareJid() + ": added candidate from back log");
1458 }
1459 }
1460
1461 private void prepareSessionAccept(
1462 final org.webrtc.SessionDescription webRTCSessionDescription) {
1463 final SessionDescription sessionDescription =
1464 SessionDescription.parse(webRTCSessionDescription.description);
1465 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1466 this.responderRtpContentMap = respondingRtpContentMap;
1467 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1468 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1469 prepareOutgoingContentMap(respondingRtpContentMap);
1470 Futures.addCallback(
1471 outgoingContentMapFuture,
1472 new FutureCallback<RtpContentMap>() {
1473 @Override
1474 public void onSuccess(final RtpContentMap outgoingContentMap) {
1475 sendSessionAccept(outgoingContentMap);
1476 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1477 }
1478
1479 @Override
1480 public void onFailure(@NonNull Throwable throwable) {
1481 failureToAcceptSession(throwable);
1482 }
1483 },
1484 MoreExecutors.directExecutor());
1485 }
1486
1487 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1488 if (isTerminated()) {
1489 Log.w(
1490 Config.LOGTAG,
1491 id.account.getJid().asBareJid()
1492 + ": preparing session accept was too slow. already terminated. nothing to do.");
1493 return;
1494 }
1495 transitionOrThrow(State.SESSION_ACCEPTED);
1496 final JinglePacket sessionAccept =
1497 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1498 send(sessionAccept);
1499 }
1500
1501 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1502 final RtpContentMap rtpContentMap) {
1503 if (this.omemoVerification.hasDeviceId()) {
1504 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1505 verifiedPayloadFuture =
1506 id.account
1507 .getAxolotlService()
1508 .encrypt(
1509 rtpContentMap,
1510 id.with,
1511 omemoVerification.getDeviceId());
1512 return Futures.transform(
1513 verifiedPayloadFuture,
1514 verifiedPayload -> {
1515 omemoVerification.setOrEnsureEqual(verifiedPayload);
1516 return verifiedPayload.getPayload();
1517 },
1518 MoreExecutors.directExecutor());
1519 } else {
1520 return Futures.immediateFuture(rtpContentMap);
1521 }
1522 }
1523
1524 synchronized void deliveryMessage(
1525 final Jid from,
1526 final Element message,
1527 final String serverMessageId,
1528 final long timestamp) {
1529 Log.d(
1530 Config.LOGTAG,
1531 id.account.getJid().asBareJid()
1532 + ": delivered message to JingleRtpConnection "
1533 + message);
1534 switch (message.getName()) {
1535 case "propose":
1536 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1537 break;
1538 case "proceed":
1539 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1540 break;
1541 case "retract":
1542 receiveRetract(from, serverMessageId, timestamp);
1543 break;
1544 case "reject":
1545 receiveReject(from, serverMessageId, timestamp);
1546 break;
1547 case "accept":
1548 receiveAccept(from, serverMessageId, timestamp);
1549 break;
1550 default:
1551 break;
1552 }
1553 }
1554
1555 void deliverFailedProceed(final String message) {
1556 Log.d(
1557 Config.LOGTAG,
1558 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1559 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1560 webRTCWrapper.close();
1561 Log.d(
1562 Config.LOGTAG,
1563 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1564 this.finish();
1565 }
1566 }
1567
1568 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1569 final boolean originatedFromMyself =
1570 from.asBareJid().equals(id.account.getJid().asBareJid());
1571 if (originatedFromMyself) {
1572 if (transition(State.ACCEPTED)) {
1573 acceptedOnOtherDevice(serverMsgId, timestamp);
1574 } else {
1575 Log.d(
1576 Config.LOGTAG,
1577 id.account.getJid().asBareJid()
1578 + ": unable to transition to accept because already in state="
1579 + this.state);
1580 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1581 }
1582 } else {
1583 Log.d(
1584 Config.LOGTAG,
1585 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1586 }
1587 }
1588
1589 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1590 if (serverMsgId != null) {
1591 this.message.setServerMsgId(serverMsgId);
1592 }
1593 this.message.setTime(timestamp);
1594 this.message.setCarbon(true); // indicate that call was accepted on other device
1595 this.writeLogMessageSuccess(0);
1596 this.xmppConnectionService
1597 .getNotificationService()
1598 .cancelIncomingCallNotification();
1599 this.finish();
1600 }
1601
1602 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1603 final boolean originatedFromMyself =
1604 from.asBareJid().equals(id.account.getJid().asBareJid());
1605 // reject from another one of my clients
1606 if (originatedFromMyself) {
1607 receiveRejectFromMyself(serverMsgId, timestamp);
1608 } else if (isInitiator()) {
1609 if (from.equals(id.with)) {
1610 receiveRejectFromResponder();
1611 } else {
1612 Log.d(
1613 Config.LOGTAG,
1614 id.account.getJid()
1615 + ": ignoring reject from "
1616 + from
1617 + " for session with "
1618 + id.with);
1619 }
1620 } else {
1621 Log.d(
1622 Config.LOGTAG,
1623 id.account.getJid()
1624 + ": ignoring reject from "
1625 + from
1626 + " for session with "
1627 + id.with);
1628 }
1629 }
1630
1631 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1632 if (transition(State.REJECTED)) {
1633 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1634 this.finish();
1635 if (serverMsgId != null) {
1636 this.message.setServerMsgId(serverMsgId);
1637 }
1638 this.message.setTime(timestamp);
1639 this.message.setCarbon(true); // indicate that call was rejected on other device
1640 writeLogMessageMissed();
1641 } else {
1642 Log.d(
1643 Config.LOGTAG,
1644 "not able to transition into REJECTED because already in " + this.state);
1645 }
1646 }
1647
1648 private void receiveRejectFromResponder() {
1649 if (isInState(State.PROCEED)) {
1650 Log.d(
1651 Config.LOGTAG,
1652 id.account.getJid()
1653 + ": received reject while still in proceed. callee reconsidered");
1654 closeTransitionLogFinish(State.REJECTED_RACED);
1655 return;
1656 }
1657 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1658 Log.d(
1659 Config.LOGTAG,
1660 id.account.getJid()
1661 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1662 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1663 return;
1664 }
1665 Log.d(
1666 Config.LOGTAG,
1667 id.account.getJid()
1668 + ": ignoring reject from responder because already in state "
1669 + this.state);
1670 }
1671
1672 private void receivePropose(
1673 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1674 final boolean originatedFromMyself =
1675 from.asBareJid().equals(id.account.getJid().asBareJid());
1676 if (originatedFromMyself) {
1677 Log.d(
1678 Config.LOGTAG,
1679 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1680 } else if (transition(
1681 State.PROPOSED,
1682 () -> {
1683 final Collection<RtpDescription> descriptions =
1684 Collections2.transform(
1685 Collections2.filter(
1686 propose.getDescriptions(),
1687 d -> d instanceof RtpDescription),
1688 input -> (RtpDescription) input);
1689 final Collection<Media> media =
1690 Collections2.transform(descriptions, RtpDescription::getMedia);
1691 Preconditions.checkState(
1692 !media.contains(Media.UNKNOWN),
1693 "RTP descriptions contain unknown media");
1694 Log.d(
1695 Config.LOGTAG,
1696 id.account.getJid().asBareJid()
1697 + ": received session proposal from "
1698 + from
1699 + " for "
1700 + media);
1701 this.proposedMedia = Sets.newHashSet(media);
1702 })) {
1703 if (serverMsgId != null) {
1704 this.message.setServerMsgId(serverMsgId);
1705 }
1706 this.message.setTime(timestamp);
1707 startRinging();
1708 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1709 sendJingleMessage("ringing");
1710 }
1711 } else {
1712 Log.d(
1713 Config.LOGTAG,
1714 id.account.getJid()
1715 + ": ignoring session proposal because already in "
1716 + state);
1717 }
1718 }
1719
1720 private void startRinging() {
1721 Log.d(
1722 Config.LOGTAG,
1723 id.account.getJid().asBareJid()
1724 + ": received call from "
1725 + id.with
1726 + ". start ringing");
1727 ringingTimeoutFuture =
1728 jingleConnectionManager.schedule(
1729 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1730 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1731 }
1732
1733 private synchronized void ringingTimeout() {
1734 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1735 switch (this.state) {
1736 case PROPOSED:
1737 message.markUnread();
1738 rejectCallFromProposed();
1739 break;
1740 case SESSION_INITIALIZED:
1741 message.markUnread();
1742 rejectCallFromSessionInitiate();
1743 break;
1744 }
1745 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1746 }
1747
1748 private void cancelRingingTimeout() {
1749 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1750 if (future != null && !future.isCancelled()) {
1751 future.cancel(false);
1752 }
1753 }
1754
1755 private void receiveProceed(
1756 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1757 final Set<Media> media =
1758 Preconditions.checkNotNull(
1759 this.proposedMedia, "Proposed media has to be set before handling proceed");
1760 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1761 if (from.equals(id.with)) {
1762 if (isInitiator()) {
1763 if (transition(State.PROCEED)) {
1764 if (serverMsgId != null) {
1765 this.message.setServerMsgId(serverMsgId);
1766 }
1767 this.message.setTime(timestamp);
1768 final Integer remoteDeviceId = proceed.getDeviceId();
1769 if (isOmemoEnabled()) {
1770 this.omemoVerification.setDeviceId(remoteDeviceId);
1771 } else {
1772 if (remoteDeviceId != null) {
1773 Log.d(
1774 Config.LOGTAG,
1775 id.account.getJid().asBareJid()
1776 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1777 }
1778 this.omemoVerification.setDeviceId(null);
1779 }
1780 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1781 } else {
1782 Log.d(
1783 Config.LOGTAG,
1784 String.format(
1785 "%s: ignoring proceed because already in %s",
1786 id.account.getJid().asBareJid(), this.state));
1787 }
1788 } else {
1789 Log.d(
1790 Config.LOGTAG,
1791 String.format(
1792 "%s: ignoring proceed because we were not initializing",
1793 id.account.getJid().asBareJid()));
1794 }
1795 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1796 if (transition(State.ACCEPTED)) {
1797 Log.d(
1798 Config.LOGTAG,
1799 id.account.getJid().asBareJid()
1800 + ": moved session with "
1801 + id.with
1802 + " into state accepted after received carbon copied proceed");
1803 acceptedOnOtherDevice(serverMsgId, timestamp);
1804 }
1805 } else {
1806 Log.d(
1807 Config.LOGTAG,
1808 String.format(
1809 "%s: ignoring proceed from %s. was expected from %s",
1810 id.account.getJid().asBareJid(), from, id.with));
1811 }
1812 }
1813
1814 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1815 if (from.equals(id.with)) {
1816 final State target =
1817 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1818 if (transition(target)) {
1819 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1820 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1821 Log.d(
1822 Config.LOGTAG,
1823 id.account.getJid().asBareJid()
1824 + ": session with "
1825 + id.with
1826 + " has been retracted (serverMsgId="
1827 + serverMsgId
1828 + ")");
1829 if (serverMsgId != null) {
1830 this.message.setServerMsgId(serverMsgId);
1831 }
1832 this.message.setTime(timestamp);
1833 if (target == State.RETRACTED) {
1834 this.message.markUnread();
1835 }
1836 writeLogMessageMissed();
1837 finish();
1838 } else {
1839 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1840 }
1841 } else {
1842 // TODO parse retract from self
1843 Log.d(
1844 Config.LOGTAG,
1845 id.account.getJid().asBareJid()
1846 + ": received retract from "
1847 + from
1848 + ". expected retract from"
1849 + id.with
1850 + ". ignoring");
1851 }
1852 }
1853
1854 public void sendSessionInitiate() {
1855 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1856 }
1857
1858 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1859 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1860 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1861 }
1862
1863 private synchronized void sendSessionInitiate(
1864 final Set<Media> media,
1865 final State targetState,
1866 final List<PeerConnection.IceServer> iceServers) {
1867 if (isTerminated()) {
1868 Log.w(
1869 Config.LOGTAG,
1870 id.account.getJid().asBareJid()
1871 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1872 return;
1873 }
1874 try {
1875 setupWebRTC(media, iceServers);
1876 } catch (final WebRTCWrapper.InitializationException e) {
1877 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1878 webRTCWrapper.close();
1879 sendRetract(Reason.ofThrowable(e));
1880 return;
1881 }
1882 try {
1883 org.webrtc.SessionDescription webRTCSessionDescription =
1884 this.webRTCWrapper.setLocalDescription().get();
1885 prepareSessionInitiate(webRTCSessionDescription, targetState);
1886 } catch (final Exception e) {
1887 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1888 // exceptions
1889 failureToInitiateSession(e, targetState);
1890 }
1891 }
1892
1893 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1894 if (isTerminated()) {
1895 return;
1896 }
1897 Log.d(
1898 Config.LOGTAG,
1899 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1900 Throwables.getRootCause(throwable));
1901 webRTCWrapper.close();
1902 final Reason reason = Reason.ofThrowable(throwable);
1903 if (isInState(targetState)) {
1904 sendSessionTerminate(reason, throwable.getMessage());
1905 } else {
1906 sendRetract(reason);
1907 }
1908 }
1909
1910 private void sendRetract(final Reason reason) {
1911 // TODO embed reason into retract
1912 sendJingleMessage("retract", id.with.asBareJid());
1913 transitionOrThrow(reasonToState(reason));
1914 this.finish();
1915 }
1916
1917 private void prepareSessionInitiate(
1918 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1919 final SessionDescription sessionDescription =
1920 SessionDescription.parse(webRTCSessionDescription.description);
1921 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1922 this.initiatorRtpContentMap = rtpContentMap;
1923 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1924 encryptSessionInitiate(rtpContentMap);
1925 Futures.addCallback(
1926 outgoingContentMapFuture,
1927 new FutureCallback<RtpContentMap>() {
1928 @Override
1929 public void onSuccess(final RtpContentMap outgoingContentMap) {
1930 sendSessionInitiate(outgoingContentMap, targetState);
1931 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1932 }
1933
1934 @Override
1935 public void onFailure(@NonNull final Throwable throwable) {
1936 failureToInitiateSession(throwable, targetState);
1937 }
1938 },
1939 MoreExecutors.directExecutor());
1940 }
1941
1942 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1943 if (isTerminated()) {
1944 Log.w(
1945 Config.LOGTAG,
1946 id.account.getJid().asBareJid()
1947 + ": preparing session was too slow. already terminated. nothing to do.");
1948 return;
1949 }
1950 this.transitionOrThrow(targetState);
1951 final JinglePacket sessionInitiate =
1952 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1953 send(sessionInitiate);
1954 }
1955
1956 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1957 final RtpContentMap rtpContentMap) {
1958 if (this.omemoVerification.hasDeviceId()) {
1959 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1960 verifiedPayloadFuture =
1961 id.account
1962 .getAxolotlService()
1963 .encrypt(
1964 rtpContentMap,
1965 id.with,
1966 omemoVerification.getDeviceId());
1967 final ListenableFuture<RtpContentMap> future =
1968 Futures.transform(
1969 verifiedPayloadFuture,
1970 verifiedPayload -> {
1971 omemoVerification.setSessionFingerprint(
1972 verifiedPayload.getFingerprint());
1973 return verifiedPayload.getPayload();
1974 },
1975 MoreExecutors.directExecutor());
1976 if (Config.REQUIRE_RTP_VERIFICATION) {
1977 return future;
1978 }
1979 return Futures.catching(
1980 future,
1981 CryptoFailedException.class,
1982 e -> {
1983 Log.w(
1984 Config.LOGTAG,
1985 id.account.getJid().asBareJid()
1986 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1987 e);
1988 return rtpContentMap;
1989 },
1990 MoreExecutors.directExecutor());
1991 } else {
1992 return Futures.immediateFuture(rtpContentMap);
1993 }
1994 }
1995
1996 private void sendSessionTerminate(final Reason reason) {
1997 sendSessionTerminate(reason, null);
1998 }
1999
2000 private void sendSessionTerminate(final Reason reason, final String text) {
2001 final State previous = this.state;
2002 final State target = reasonToState(reason);
2003 transitionOrThrow(target);
2004 if (previous != State.NULL) {
2005 writeLogMessage(target);
2006 }
2007 final JinglePacket jinglePacket =
2008 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
2009 jinglePacket.setReason(reason, text);
2010 Log.d(Config.LOGTAG, jinglePacket.toString());
2011 send(jinglePacket);
2012 finish();
2013 }
2014
2015 private void sendTransportInfo(
2016 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2017 final RtpContentMap transportInfo;
2018 try {
2019 final RtpContentMap rtpContentMap =
2020 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2021 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2022 } catch (final Exception e) {
2023 Log.d(
2024 Config.LOGTAG,
2025 id.account.getJid().asBareJid()
2026 + ": unable to prepare transport-info from candidate for content="
2027 + contentName);
2028 return;
2029 }
2030 final JinglePacket jinglePacket =
2031 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2032 send(jinglePacket);
2033 }
2034
2035 private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
2036 // TODO send all candidates in one transport-info
2037 }
2038
2039
2040 private void send(final JinglePacket jinglePacket) {
2041 jinglePacket.setTo(id.with);
2042 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
2043 }
2044
2045 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
2046 if (response.getType() == IqPacket.TYPE.ERROR) {
2047 handleIqErrorResponse(response);
2048 return;
2049 }
2050 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2051 handleIqTimeoutResponse(response);
2052 }
2053 }
2054
2055 private void handleIqErrorResponse(final IqPacket response) {
2056 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
2057 final String errorCondition = response.getErrorCondition();
2058 Log.d(
2059 Config.LOGTAG,
2060 id.account.getJid().asBareJid()
2061 + ": received IQ-error from "
2062 + response.getFrom()
2063 + " in RTP session. "
2064 + errorCondition);
2065 if (isTerminated()) {
2066 Log.i(
2067 Config.LOGTAG,
2068 id.account.getJid().asBareJid()
2069 + ": ignoring error because session was already terminated");
2070 return;
2071 }
2072 this.webRTCWrapper.close();
2073 final State target;
2074 if (Arrays.asList(
2075 "service-unavailable",
2076 "recipient-unavailable",
2077 "remote-server-not-found",
2078 "remote-server-timeout")
2079 .contains(errorCondition)) {
2080 target = State.TERMINATED_CONNECTIVITY_ERROR;
2081 } else {
2082 target = State.TERMINATED_APPLICATION_FAILURE;
2083 }
2084 transitionOrThrow(target);
2085 this.finish();
2086 }
2087
2088 private void handleIqTimeoutResponse(final IqPacket response) {
2089 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
2090 Log.d(
2091 Config.LOGTAG,
2092 id.account.getJid().asBareJid()
2093 + ": received IQ timeout in RTP session with "
2094 + id.with
2095 + ". terminating with connectivity error");
2096 if (isTerminated()) {
2097 Log.i(
2098 Config.LOGTAG,
2099 id.account.getJid().asBareJid()
2100 + ": ignoring error because session was already terminated");
2101 return;
2102 }
2103 this.webRTCWrapper.close();
2104 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
2105 this.finish();
2106 }
2107
2108 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
2109 Log.d(
2110 Config.LOGTAG,
2111 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
2112 this.webRTCWrapper.close();
2113 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
2114 respondWithOutOfOrder(jinglePacket);
2115 this.finish();
2116 }
2117
2118 private void respondWithTieBreak(final JinglePacket jinglePacket) {
2119 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
2120 }
2121
2122 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
2123 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
2124 }
2125
2126 private void respondWithItemNotFound(final JinglePacket jinglePacket) {
2127 respondWithJingleError(jinglePacket, null, "item-not-found", "cancel");
2128 }
2129
2130 void respondWithJingleError(
2131 final IqPacket original,
2132 String jingleCondition,
2133 String condition,
2134 String conditionType) {
2135 jingleConnectionManager.respondWithJingleError(
2136 id.account, original, jingleCondition, condition, conditionType);
2137 }
2138
2139 private void respondOk(final JinglePacket jinglePacket) {
2140 xmppConnectionService.sendIqPacket(
2141 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
2142 }
2143
2144 public RtpEndUserState getEndUserState() {
2145 switch (this.state) {
2146 case NULL:
2147 case PROPOSED:
2148 case SESSION_INITIALIZED:
2149 if (isInitiator()) {
2150 return RtpEndUserState.RINGING;
2151 } else {
2152 return RtpEndUserState.INCOMING_CALL;
2153 }
2154 case PROCEED:
2155 if (isInitiator()) {
2156 return RtpEndUserState.RINGING;
2157 } else {
2158 return RtpEndUserState.ACCEPTING_CALL;
2159 }
2160 case SESSION_INITIALIZED_PRE_APPROVED:
2161 if (isInitiator()) {
2162 return RtpEndUserState.RINGING;
2163 } else {
2164 return RtpEndUserState.CONNECTING;
2165 }
2166 case SESSION_ACCEPTED:
2167 final ContentAddition ca = getPendingContentAddition();
2168 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2169 return RtpEndUserState.INCOMING_CONTENT_ADD;
2170 }
2171 return getPeerConnectionStateAsEndUserState();
2172 case REJECTED:
2173 case REJECTED_RACED:
2174 case TERMINATED_DECLINED_OR_BUSY:
2175 if (isInitiator()) {
2176 return RtpEndUserState.DECLINED_OR_BUSY;
2177 } else {
2178 return RtpEndUserState.ENDED;
2179 }
2180 case TERMINATED_SUCCESS:
2181 case ACCEPTED:
2182 case RETRACTED:
2183 case TERMINATED_CANCEL_OR_TIMEOUT:
2184 return RtpEndUserState.ENDED;
2185 case RETRACTED_RACED:
2186 if (isInitiator()) {
2187 return RtpEndUserState.ENDED;
2188 } else {
2189 return RtpEndUserState.RETRACTED;
2190 }
2191 case TERMINATED_CONNECTIVITY_ERROR:
2192 return zeroDuration()
2193 ? RtpEndUserState.CONNECTIVITY_ERROR
2194 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2195 case TERMINATED_APPLICATION_FAILURE:
2196 return RtpEndUserState.APPLICATION_ERROR;
2197 case TERMINATED_SECURITY_ERROR:
2198 return RtpEndUserState.SECURITY_ERROR;
2199 }
2200 throw new IllegalStateException(
2201 String.format("%s has no equivalent EndUserState", this.state));
2202 }
2203
2204 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2205 final PeerConnection.PeerConnectionState state;
2206 try {
2207 state = webRTCWrapper.getState();
2208 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2209 // We usually close the WebRTCWrapper *before* transitioning so we might still
2210 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2211 return RtpEndUserState.ENDING_CALL;
2212 }
2213 switch (state) {
2214 case CONNECTED:
2215 return RtpEndUserState.CONNECTED;
2216 case NEW:
2217 case CONNECTING:
2218 return RtpEndUserState.CONNECTING;
2219 case CLOSED:
2220 return RtpEndUserState.ENDING_CALL;
2221 default:
2222 return zeroDuration()
2223 ? RtpEndUserState.CONNECTIVITY_ERROR
2224 : RtpEndUserState.RECONNECTING;
2225 }
2226 }
2227
2228 public ContentAddition getPendingContentAddition() {
2229 final RtpContentMap in = this.incomingContentAdd;
2230 final RtpContentMap out = this.outgoingContentAdd;
2231 if (out != null) {
2232 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2233 } else if (in != null) {
2234 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2235 } else {
2236 return null;
2237 }
2238 }
2239
2240 public Set<Media> getMedia() {
2241 final State current = getState();
2242 if (current == State.NULL) {
2243 if (isInitiator()) {
2244 return Preconditions.checkNotNull(
2245 this.proposedMedia, "RTP connection has not been initialized properly");
2246 }
2247 throw new IllegalStateException("RTP connection has not been initialized yet");
2248 }
2249 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2250 return Preconditions.checkNotNull(
2251 this.proposedMedia, "RTP connection has not been initialized properly");
2252 }
2253 final RtpContentMap localContentMap = getLocalContentMap();
2254 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2255 if (localContentMap != null) {
2256 return localContentMap.getMedia();
2257 } else if (initiatorContentMap != null) {
2258 return initiatorContentMap.getMedia();
2259 } else if (isTerminated()) {
2260 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2261 } else {
2262 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2263 }
2264 }
2265
2266 public boolean isVerified() {
2267 final String fingerprint = this.omemoVerification.getFingerprint();
2268 if (fingerprint == null) {
2269 return false;
2270 }
2271 final FingerprintStatus status =
2272 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2273 return status != null && status.isVerified();
2274 }
2275
2276 public boolean addMedia(final Media media) {
2277 final Set<Media> currentMedia = getMedia();
2278 if (currentMedia.contains(media)) {
2279 throw new IllegalStateException(String.format("%s has already been proposed", media));
2280 }
2281 // TODO add state protection - can only add while ACCEPTED or so
2282 Log.d(Config.LOGTAG,"adding media: "+media);
2283 return webRTCWrapper.addTrack(media);
2284 }
2285
2286 public synchronized void acceptCall() {
2287 switch (this.state) {
2288 case PROPOSED:
2289 cancelRingingTimeout();
2290 acceptCallFromProposed();
2291 break;
2292 case SESSION_INITIALIZED:
2293 cancelRingingTimeout();
2294 acceptCallFromSessionInitialized();
2295 break;
2296 case ACCEPTED:
2297 Log.w(
2298 Config.LOGTAG,
2299 id.account.getJid().asBareJid()
2300 + ": the call has already been accepted with another client. UI was just lagging behind");
2301 break;
2302 case PROCEED:
2303 case SESSION_ACCEPTED:
2304 Log.w(
2305 Config.LOGTAG,
2306 id.account.getJid().asBareJid()
2307 + ": the call has already been accepted. user probably double tapped the UI");
2308 break;
2309 default:
2310 throw new IllegalStateException("Can not accept call from " + this.state);
2311 }
2312 }
2313
2314 public void notifyPhoneCall() {
2315 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2316 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2317 rejectCall();
2318 } else {
2319 endCall();
2320 }
2321 }
2322
2323 public synchronized void rejectCall() {
2324 if (isTerminated()) {
2325 Log.w(
2326 Config.LOGTAG,
2327 id.account.getJid().asBareJid()
2328 + ": received rejectCall() when session has already been terminated. nothing to do");
2329 return;
2330 }
2331 switch (this.state) {
2332 case PROPOSED:
2333 rejectCallFromProposed();
2334 break;
2335 case SESSION_INITIALIZED:
2336 rejectCallFromSessionInitiate();
2337 break;
2338 default:
2339 throw new IllegalStateException("Can not reject call from " + this.state);
2340 }
2341 }
2342
2343 public synchronized void endCall() {
2344 if (isTerminated()) {
2345 Log.w(
2346 Config.LOGTAG,
2347 id.account.getJid().asBareJid()
2348 + ": received endCall() when session has already been terminated. nothing to do");
2349 return;
2350 }
2351 if (isInState(State.PROPOSED) && !isInitiator()) {
2352 rejectCallFromProposed();
2353 return;
2354 }
2355 if (isInState(State.PROCEED)) {
2356 if (isInitiator()) {
2357 retractFromProceed();
2358 } else {
2359 rejectCallFromProceed();
2360 }
2361 return;
2362 }
2363 if (isInitiator()
2364 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2365 this.webRTCWrapper.close();
2366 sendSessionTerminate(Reason.CANCEL);
2367 return;
2368 }
2369 if (isInState(State.SESSION_INITIALIZED)) {
2370 rejectCallFromSessionInitiate();
2371 return;
2372 }
2373 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2374 this.webRTCWrapper.close();
2375 sendSessionTerminate(Reason.SUCCESS);
2376 return;
2377 }
2378 if (isInState(
2379 State.TERMINATED_APPLICATION_FAILURE,
2380 State.TERMINATED_CONNECTIVITY_ERROR,
2381 State.TERMINATED_DECLINED_OR_BUSY)) {
2382 Log.d(
2383 Config.LOGTAG,
2384 "ignoring request to end call because already in state " + this.state);
2385 return;
2386 }
2387 throw new IllegalStateException(
2388 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2389 }
2390
2391 private void retractFromProceed() {
2392 Log.d(Config.LOGTAG, "retract from proceed");
2393 this.sendJingleMessage("retract");
2394 closeTransitionLogFinish(State.RETRACTED_RACED);
2395 }
2396
2397 private void closeTransitionLogFinish(final State state) {
2398 this.webRTCWrapper.close();
2399 transitionOrThrow(state);
2400 writeLogMessage(state);
2401 finish();
2402 }
2403
2404 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2405 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2406 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2407 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2408 }
2409
2410 private void acceptCallFromProposed() {
2411 transitionOrThrow(State.PROCEED);
2412 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2413 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2414 this.sendJingleMessage("proceed");
2415 }
2416
2417 private void rejectCallFromProposed() {
2418 transitionOrThrow(State.REJECTED);
2419 writeLogMessageMissed();
2420 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2421 this.sendJingleMessage("reject");
2422 finish();
2423 }
2424
2425 private void rejectCallFromProceed() {
2426 this.sendJingleMessage("reject");
2427 closeTransitionLogFinish(State.REJECTED_RACED);
2428 }
2429
2430 private void rejectCallFromSessionInitiate() {
2431 webRTCWrapper.close();
2432 sendSessionTerminate(Reason.DECLINE);
2433 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2434 }
2435
2436 private void sendJingleMessage(final String action) {
2437 sendJingleMessage(action, id.with);
2438 }
2439
2440 private void sendJingleMessage(final String action, final Jid to) {
2441 final MessagePacket messagePacket = new MessagePacket();
2442 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2443 messagePacket.setTo(to);
2444 final Element intent =
2445 messagePacket
2446 .addChild(action, Namespace.JINGLE_MESSAGE)
2447 .setAttribute("id", id.sessionId);
2448 if ("proceed".equals(action)) {
2449 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2450 if (isOmemoEnabled()) {
2451 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2452 final Element device =
2453 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2454 device.setAttribute("id", deviceId);
2455 }
2456 }
2457 messagePacket.addChild("store", "urn:xmpp:hints");
2458 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2459 }
2460
2461 private boolean isOmemoEnabled() {
2462 final Conversational conversational = message.getConversation();
2463 if (conversational instanceof Conversation) {
2464 return ((Conversation) conversational).getNextEncryption()
2465 == Message.ENCRYPTION_AXOLOTL;
2466 }
2467 return false;
2468 }
2469
2470 private void acceptCallFromSessionInitialized() {
2471 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2472 sendSessionAccept();
2473 }
2474
2475 private synchronized boolean isInState(State... state) {
2476 return Arrays.asList(state).contains(this.state);
2477 }
2478
2479 private boolean transition(final State target) {
2480 return transition(target, null);
2481 }
2482
2483 private synchronized boolean transition(final State target, final Runnable runnable) {
2484 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2485 if (validTransitions != null && validTransitions.contains(target)) {
2486 this.state = target;
2487 if (runnable != null) {
2488 runnable.run();
2489 }
2490 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2491 updateEndUserState();
2492 updateOngoingCallNotification();
2493 return true;
2494 } else {
2495 return false;
2496 }
2497 }
2498
2499 void transitionOrThrow(final State target) {
2500 if (!transition(target)) {
2501 throw new IllegalStateException(
2502 String.format("Unable to transition from %s to %s", this.state, target));
2503 }
2504 }
2505
2506 @Override
2507 public void onIceCandidate(final IceCandidate iceCandidate) {
2508 final RtpContentMap rtpContentMap =
2509 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2510 final IceUdpTransportInfo.Credentials credentials;
2511 try {
2512 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2513 } catch (final IllegalArgumentException e) {
2514 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2515 return;
2516 }
2517 final String uFrag = credentials.ufrag;
2518 final IceUdpTransportInfo.Candidate candidate =
2519 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2520 if (candidate == null) {
2521 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2522 return;
2523 }
2524 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2525 sendTransportInfo(iceCandidate.sdpMid, candidate);
2526 }
2527
2528 @Override
2529 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2530 Log.d(
2531 Config.LOGTAG,
2532 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2533 this.stateHistory.add(newState);
2534 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2535 this.sessionDuration.start();
2536 updateOngoingCallNotification();
2537 } else if (this.sessionDuration.isRunning()) {
2538 this.sessionDuration.stop();
2539 updateOngoingCallNotification();
2540 }
2541
2542 final boolean neverConnected =
2543 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2544
2545 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2546 if (neverConnected) {
2547 if (isTerminated()) {
2548 Log.d(
2549 Config.LOGTAG,
2550 id.account.getJid().asBareJid()
2551 + ": not sending session-terminate after connectivity error because session is already in state "
2552 + this.state);
2553 return;
2554 }
2555 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2556 return;
2557 } else {
2558 this.restartIce();
2559 }
2560 }
2561 updateEndUserState();
2562 }
2563
2564 private void restartIce() {
2565 this.stateHistory.clear();
2566 this.webRTCWrapper.restartIceAsync();
2567 }
2568
2569 @Override
2570 public void onRenegotiationNeeded() {
2571 this.webRTCWrapper.execute(this::renegotiate);
2572 }
2573
2574 private void renegotiate() {
2575 final SessionDescription sessionDescription;
2576 try {
2577 sessionDescription = setLocalSessionDescription();
2578 } catch (final Exception e) {
2579 final Throwable cause = Throwables.getRootCause(e);
2580 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2581 webRTCWrapper.close();
2582 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2583 return;
2584 }
2585 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2586 final RtpContentMap currentContentMap = getLocalContentMap();
2587 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2588 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2589
2590 Log.d(
2591 Config.LOGTAG,
2592 id.getAccount().getJid().asBareJid()
2593 + ": renegotiate. iceRestart="
2594 + iceRestart
2595 + " content id diff="
2596 + diff);
2597
2598 if (diff.hasModifications() && iceRestart) {
2599 webRTCWrapper.close();
2600 sendSessionTerminate(
2601 Reason.FAILED_APPLICATION,
2602 "WebRTC unexpectedly tried to modify content and transport at once");
2603 return;
2604 }
2605
2606 if (iceRestart) {
2607 initiateIceRestart(rtpContentMap);
2608 return;
2609 } else if (diff.isEmpty()) {
2610 Log.d(
2611 Config.LOGTAG,
2612 "renegotiation. nothing to do. SignalingState="
2613 + this.webRTCWrapper.getSignalingState());
2614 }
2615
2616 if (diff.added.size() > 0) {
2617 modifyLocalContentMap(rtpContentMap);
2618 sendContentAdd(rtpContentMap, diff.added);
2619 }
2620 }
2621
2622 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2623 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2624 final JinglePacket jinglePacket =
2625 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2626 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2627 jinglePacket.setTo(id.with);
2628 xmppConnectionService.sendIqPacket(
2629 id.account,
2630 jinglePacket,
2631 (account, response) -> {
2632 if (response.getType() == IqPacket.TYPE.RESULT) {
2633 Log.d(Config.LOGTAG, "received success to our ice restart");
2634 setLocalContentMap(rtpContentMap);
2635 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2636 return;
2637 }
2638 if (response.getType() == IqPacket.TYPE.ERROR) {
2639 if (isTieBreak(response)) {
2640 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2641 return;
2642 }
2643 handleIqErrorResponse(response);
2644 }
2645 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2646 handleIqTimeoutResponse(response);
2647 }
2648 });
2649 }
2650
2651 private boolean isTieBreak(final IqPacket response) {
2652 final Element error = response.findChild("error");
2653 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2654 }
2655
2656 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2657 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2658 this.outgoingContentAdd = contentAdd;
2659 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2660 prepareOutgoingContentMap(contentAdd);
2661 Futures.addCallback(
2662 outgoingContentMapFuture,
2663 new FutureCallback<RtpContentMap>() {
2664 @Override
2665 public void onSuccess(final RtpContentMap outgoingContentMap) {
2666 sendContentAdd(outgoingContentMap);
2667 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2668 }
2669
2670 @Override
2671 public void onFailure(@NonNull Throwable throwable) {
2672 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2673 }
2674 },
2675 MoreExecutors.directExecutor());
2676 }
2677
2678 private void sendContentAdd(final RtpContentMap contentAdd) {
2679
2680 final JinglePacket jinglePacket =
2681 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2682 jinglePacket.setTo(id.with);
2683 xmppConnectionService.sendIqPacket(
2684 id.account,
2685 jinglePacket,
2686 (connection, response) -> {
2687 if (response.getType() == IqPacket.TYPE.RESULT) {
2688 Log.d(
2689 Config.LOGTAG,
2690 id.getAccount().getJid().asBareJid()
2691 + ": received ACK to our content-add");
2692 return;
2693 }
2694 if (response.getType() == IqPacket.TYPE.ERROR) {
2695 if (isTieBreak(response)) {
2696 this.outgoingContentAdd = null;
2697 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2698 return;
2699 }
2700 handleIqErrorResponse(response);
2701 }
2702 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2703 handleIqTimeoutResponse(response);
2704 }
2705 });
2706 }
2707
2708 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2709 if (isInitiator()) {
2710 this.initiatorRtpContentMap = rtpContentMap;
2711 } else {
2712 this.responderRtpContentMap = rtpContentMap;
2713 }
2714 }
2715
2716 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2717 if (isInitiator()) {
2718 this.responderRtpContentMap = rtpContentMap;
2719 } else {
2720 this.initiatorRtpContentMap = rtpContentMap;
2721 }
2722 }
2723
2724 // this method is to be used for content map modifications that modify media
2725 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2726 final RtpContentMap activeContents = rtpContentMap.activeContents();
2727 setLocalContentMap(activeContents);
2728 this.webRTCWrapper.switchSpeakerPhonePreference(
2729 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2730 updateEndUserState();
2731 }
2732
2733 private SessionDescription setLocalSessionDescription()
2734 throws ExecutionException, InterruptedException {
2735 final org.webrtc.SessionDescription sessionDescription =
2736 this.webRTCWrapper.setLocalDescription().get();
2737 return SessionDescription.parse(sessionDescription.description);
2738 }
2739
2740 private void closeWebRTCSessionAfterFailedConnection() {
2741 this.webRTCWrapper.close();
2742 synchronized (this) {
2743 if (isTerminated()) {
2744 Log.d(
2745 Config.LOGTAG,
2746 id.account.getJid().asBareJid()
2747 + ": no need to send session-terminate after failed connection. Other party already did");
2748 return;
2749 }
2750 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2751 }
2752 }
2753
2754 public boolean zeroDuration() {
2755 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2756 }
2757
2758 public long getCallDuration() {
2759 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2760 }
2761
2762 public AppRTCAudioManager getAudioManager() {
2763 return webRTCWrapper.getAudioManager();
2764 }
2765
2766 public boolean isMicrophoneEnabled() {
2767 return webRTCWrapper.isMicrophoneEnabled();
2768 }
2769
2770 public boolean setMicrophoneEnabled(final boolean enabled) {
2771 return webRTCWrapper.setMicrophoneEnabled(enabled);
2772 }
2773
2774 public boolean isVideoEnabled() {
2775 return webRTCWrapper.isVideoEnabled();
2776 }
2777
2778 public void setVideoEnabled(final boolean enabled) {
2779 webRTCWrapper.setVideoEnabled(enabled);
2780 }
2781
2782 public boolean isCameraSwitchable() {
2783 return webRTCWrapper.isCameraSwitchable();
2784 }
2785
2786 public boolean isFrontCamera() {
2787 return webRTCWrapper.isFrontCamera();
2788 }
2789
2790 public ListenableFuture<Boolean> switchCamera() {
2791 return webRTCWrapper.switchCamera();
2792 }
2793
2794 @Override
2795 public void onAudioDeviceChanged(
2796 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2797 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2798 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2799 selectedAudioDevice, availableAudioDevices);
2800 }
2801
2802 private void updateEndUserState() {
2803 final RtpEndUserState endUserState = getEndUserState();
2804 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2805 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2806 id.account, id.with, id.sessionId, endUserState);
2807 }
2808
2809 private void updateOngoingCallNotification() {
2810 final State state = this.state;
2811 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2812 final boolean reconnecting;
2813 if (state == State.SESSION_ACCEPTED) {
2814 reconnecting =
2815 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2816 } else {
2817 reconnecting = false;
2818 }
2819 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2820 } else {
2821 xmppConnectionService.removeOngoingCall();
2822 }
2823 }
2824
2825 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2826 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2827 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2828 request.setTo(id.account.getDomain());
2829 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2830 xmppConnectionService.sendIqPacket(
2831 id.account,
2832 request,
2833 (account, response) -> {
2834 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2835 new ImmutableList.Builder<>();
2836 if (response.getType() == IqPacket.TYPE.RESULT) {
2837 final Element services =
2838 response.findChild(
2839 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2840 final List<Element> children =
2841 services == null
2842 ? Collections.emptyList()
2843 : services.getChildren();
2844 for (final Element child : children) {
2845 if ("service".equals(child.getName())) {
2846 final String type = child.getAttribute("type");
2847 final String host = child.getAttribute("host");
2848 final String sport = child.getAttribute("port");
2849 final Integer port =
2850 sport == null ? null : Ints.tryParse(sport);
2851 final String transport = child.getAttribute("transport");
2852 final String username = child.getAttribute("username");
2853 final String password = child.getAttribute("password");
2854 if (Strings.isNullOrEmpty(host) || port == null) {
2855 continue;
2856 }
2857 if (port < 0 || port > 65535) {
2858 continue;
2859 }
2860
2861
2862
2863
2864 if (Arrays.asList("stun", "stuns", "turn", "turns")
2865 .contains(type)
2866 && Arrays.asList("udp", "tcp").contains(transport)) {
2867 if (Arrays.asList("stuns", "turns").contains(type)
2868 && "udp".equals(transport)) {
2869 Log.d(
2870 Config.LOGTAG,
2871 id.account.getJid().asBareJid()
2872 + ": skipping invalid combination of udp/tls in external services");
2873 continue;
2874 }
2875
2876 // STUN URLs do not support a query section since M110
2877 final String uri;
2878 if (Arrays.asList("stun","stuns").contains(type)) {
2879 uri = String.format("%s:%s:%s", type, IP.wrapIPv6(host),port);
2880 } else {
2881 uri = String.format(
2882 "%s:%s:%s?transport=%s",
2883 type,
2884 IP.wrapIPv6(host),
2885 port,
2886 transport);
2887 }
2888
2889 final PeerConnection.IceServer.Builder iceServerBuilder =
2890 PeerConnection.IceServer.builder(uri);
2891 iceServerBuilder.setTlsCertPolicy(
2892 PeerConnection.TlsCertPolicy
2893 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2894 if (username != null && password != null) {
2895 iceServerBuilder.setUsername(username);
2896 iceServerBuilder.setPassword(password);
2897 } else if (Arrays.asList("turn", "turns").contains(type)) {
2898 // The WebRTC spec requires throwing an
2899 // InvalidAccessError when username (from libwebrtc
2900 // source coder)
2901 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2902 Log.d(
2903 Config.LOGTAG,
2904 id.account.getJid().asBareJid()
2905 + ": skipping "
2906 + type
2907 + "/"
2908 + transport
2909 + " without username and password");
2910 continue;
2911 }
2912 final PeerConnection.IceServer iceServer =
2913 iceServerBuilder.createIceServer();
2914 Log.d(
2915 Config.LOGTAG,
2916 id.account.getJid().asBareJid()
2917 + ": discovered ICE Server: "
2918 + iceServer);
2919 listBuilder.add(iceServer);
2920 }
2921 }
2922 }
2923 }
2924 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2925 if (iceServers.size() == 0) {
2926 Log.w(
2927 Config.LOGTAG,
2928 id.account.getJid().asBareJid()
2929 + ": no ICE server found "
2930 + response);
2931 }
2932 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2933 });
2934 } else {
2935 Log.w(
2936 Config.LOGTAG,
2937 id.account.getJid().asBareJid() + ": has no external service discovery");
2938 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2939 }
2940 }
2941
2942 private void finish() {
2943 if (isTerminated()) {
2944 this.cancelRingingTimeout();
2945 this.webRTCWrapper.verifyClosed();
2946 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2947 this.jingleConnectionManager.finishConnectionOrThrow(this);
2948 } else {
2949 throw new IllegalStateException(
2950 String.format("Unable to call finish from %s", this.state));
2951 }
2952 }
2953
2954 private void writeLogMessage(final State state) {
2955 final long duration = getCallDuration();
2956 if (state == State.TERMINATED_SUCCESS
2957 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2958 writeLogMessageSuccess(duration);
2959 } else {
2960 writeLogMessageMissed();
2961 }
2962 }
2963
2964 private void writeLogMessageSuccess(final long duration) {
2965 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2966 this.writeMessage();
2967 }
2968
2969 private void writeLogMessageMissed() {
2970 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2971 this.writeMessage();
2972 }
2973
2974 private void writeMessage() {
2975 final Conversational conversational = message.getConversation();
2976 if (conversational instanceof Conversation) {
2977 ((Conversation) conversational).add(this.message);
2978 xmppConnectionService.createMessageAsync(message);
2979 xmppConnectionService.updateConversationUi();
2980 } else {
2981 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2982 }
2983 }
2984
2985 public State getState() {
2986 return this.state;
2987 }
2988
2989 boolean isTerminated() {
2990 return TERMINATED.contains(this.state);
2991 }
2992
2993 public Optional<VideoTrack> getLocalVideoTrack() {
2994 return webRTCWrapper.getLocalVideoTrack();
2995 }
2996
2997 public Optional<VideoTrack> getRemoteVideoTrack() {
2998 return webRTCWrapper.getRemoteVideoTrack();
2999 }
3000
3001 public EglBase.Context getEglBaseContext() {
3002 return webRTCWrapper.getEglBaseContext();
3003 }
3004
3005 void setProposedMedia(final Set<Media> media) {
3006 this.proposedMedia = media;
3007 }
3008
3009 public void fireStateUpdate() {
3010 final RtpEndUserState endUserState = getEndUserState();
3011 xmppConnectionService.notifyJingleRtpConnectionUpdate(
3012 id.account, id.with, id.sessionId, endUserState);
3013 }
3014
3015 public boolean isSwitchToVideoAvailable() {
3016 final boolean prerequisite =
3017 Media.audioOnly(getMedia())
3018 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
3019 .contains(getEndUserState());
3020 return prerequisite && remoteHasVideoFeature();
3021 }
3022
3023 private boolean remoteHasVideoFeature() {
3024 final Contact contact = id.getContact();
3025 final Presence presence =
3026 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
3027 final ServiceDiscoveryResult serviceDiscoveryResult =
3028 presence == null ? null : presence.getServiceDiscoveryResult();
3029 final List<String> features =
3030 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
3031 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
3032 }
3033
3034 private interface OnIceServersDiscovered {
3035 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
3036 }
3037}