1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Joiner;
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.base.Stopwatch;
12import com.google.common.base.Strings;
13import com.google.common.base.Throwables;
14import com.google.common.collect.Collections2;
15import com.google.common.collect.ImmutableList;
16import com.google.common.collect.ImmutableMap;
17import com.google.common.collect.ImmutableMultimap;
18import com.google.common.collect.ImmutableSet;
19import com.google.common.collect.Iterables;
20import com.google.common.collect.Maps;
21import com.google.common.collect.Multimap;
22import com.google.common.collect.Sets;
23import com.google.common.primitives.Ints;
24import com.google.common.util.concurrent.FutureCallback;
25import com.google.common.util.concurrent.Futures;
26import com.google.common.util.concurrent.ListenableFuture;
27import com.google.common.util.concurrent.MoreExecutors;
28
29import org.webrtc.EglBase;
30import org.webrtc.IceCandidate;
31import org.webrtc.PeerConnection;
32import org.webrtc.VideoTrack;
33
34import java.util.Arrays;
35import java.util.Collection;
36import java.util.Collections;
37import java.util.LinkedList;
38import java.util.List;
39import java.util.Map;
40import java.util.Queue;
41import java.util.Set;
42import java.util.concurrent.ExecutionException;
43import java.util.concurrent.ScheduledFuture;
44import java.util.concurrent.TimeUnit;
45
46import eu.siacs.conversations.BuildConfig;
47import eu.siacs.conversations.Config;
48import eu.siacs.conversations.crypto.axolotl.AxolotlService;
49import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
50import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
51import eu.siacs.conversations.entities.Account;
52import eu.siacs.conversations.entities.Contact;
53import eu.siacs.conversations.entities.Conversation;
54import eu.siacs.conversations.entities.Conversational;
55import eu.siacs.conversations.entities.Message;
56import eu.siacs.conversations.entities.Presence;
57import eu.siacs.conversations.entities.RtpSessionStatus;
58import eu.siacs.conversations.entities.ServiceDiscoveryResult;
59import eu.siacs.conversations.services.AppRTCAudioManager;
60import eu.siacs.conversations.utils.IP;
61import eu.siacs.conversations.xml.Element;
62import eu.siacs.conversations.xml.Namespace;
63import eu.siacs.conversations.xmpp.Jid;
64import eu.siacs.conversations.xmpp.OnIqPacketReceived;
65import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
66import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
67import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
68import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
69import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
70import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
71import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
72import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
73import eu.siacs.conversations.xmpp.stanzas.IqPacket;
74import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
75
76public class JingleRtpConnection extends AbstractJingleConnection
77 implements WebRTCWrapper.EventCallback {
78
79 public static final List<State> STATES_SHOWING_ONGOING_CALL =
80 Arrays.asList(
81 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
82 private static final long BUSY_TIME_OUT = 30;
83 private static final List<State> TERMINATED =
84 Arrays.asList(
85 State.ACCEPTED,
86 State.REJECTED,
87 State.REJECTED_RACED,
88 State.RETRACTED,
89 State.RETRACTED_RACED,
90 State.TERMINATED_SUCCESS,
91 State.TERMINATED_DECLINED_OR_BUSY,
92 State.TERMINATED_CONNECTIVITY_ERROR,
93 State.TERMINATED_CANCEL_OR_TIMEOUT,
94 State.TERMINATED_APPLICATION_FAILURE,
95 State.TERMINATED_SECURITY_ERROR);
96
97 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
98
99 static {
100 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
101 new ImmutableMap.Builder<>();
102 transitionBuilder.put(
103 State.NULL,
104 ImmutableList.of(
105 State.PROPOSED,
106 State.SESSION_INITIALIZED,
107 State.TERMINATED_APPLICATION_FAILURE,
108 State.TERMINATED_SECURITY_ERROR));
109 transitionBuilder.put(
110 State.PROPOSED,
111 ImmutableList.of(
112 State.ACCEPTED,
113 State.PROCEED,
114 State.REJECTED,
115 State.RETRACTED,
116 State.TERMINATED_APPLICATION_FAILURE,
117 State.TERMINATED_SECURITY_ERROR,
118 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
119 // rebinds
120 ));
121 transitionBuilder.put(
122 State.PROCEED,
123 ImmutableList.of(
124 State.REJECTED_RACED,
125 State.RETRACTED_RACED,
126 State.SESSION_INITIALIZED_PRE_APPROVED,
127 State.TERMINATED_SUCCESS,
128 State.TERMINATED_APPLICATION_FAILURE,
129 State.TERMINATED_SECURITY_ERROR,
130 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
131 // bounces of the proceed message
132 ));
133 transitionBuilder.put(
134 State.SESSION_INITIALIZED,
135 ImmutableList.of(
136 State.SESSION_ACCEPTED,
137 State.TERMINATED_SUCCESS,
138 State.TERMINATED_DECLINED_OR_BUSY,
139 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
140 // and IQ timeouts
141 State.TERMINATED_CANCEL_OR_TIMEOUT,
142 State.TERMINATED_APPLICATION_FAILURE,
143 State.TERMINATED_SECURITY_ERROR));
144 transitionBuilder.put(
145 State.SESSION_INITIALIZED_PRE_APPROVED,
146 ImmutableList.of(
147 State.SESSION_ACCEPTED,
148 State.TERMINATED_SUCCESS,
149 State.TERMINATED_DECLINED_OR_BUSY,
150 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
151 // and IQ timeouts
152 State.TERMINATED_CANCEL_OR_TIMEOUT,
153 State.TERMINATED_APPLICATION_FAILURE,
154 State.TERMINATED_SECURITY_ERROR));
155 transitionBuilder.put(
156 State.SESSION_ACCEPTED,
157 ImmutableList.of(
158 State.TERMINATED_SUCCESS,
159 State.TERMINATED_DECLINED_OR_BUSY,
160 State.TERMINATED_CONNECTIVITY_ERROR,
161 State.TERMINATED_CANCEL_OR_TIMEOUT,
162 State.TERMINATED_APPLICATION_FAILURE,
163 State.TERMINATED_SECURITY_ERROR));
164 VALID_TRANSITIONS = transitionBuilder.build();
165 }
166
167 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
168 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
169 pendingIceCandidates = new LinkedList<>();
170 private final OmemoVerification omemoVerification = new OmemoVerification();
171 private final Message message;
172 private State state = State.NULL;
173 private Set<Media> proposedMedia;
174 private RtpContentMap initiatorRtpContentMap;
175 private RtpContentMap responderRtpContentMap;
176 private RtpContentMap incomingContentAdd;
177 private RtpContentMap outgoingContentAdd;
178 private IceUdpTransportInfo.Setup peerDtlsSetup;
179 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
180 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
181 private ScheduledFuture<?> ringingTimeoutFuture;
182
183 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
184 super(jingleConnectionManager, id, initiator);
185 final Conversation conversation =
186 jingleConnectionManager
187 .getXmppConnectionService()
188 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
189 this.message =
190 new Message(
191 conversation,
192 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
193 Message.TYPE_RTP_SESSION,
194 id.sessionId);
195 }
196
197 private static State reasonToState(Reason reason) {
198 switch (reason) {
199 case SUCCESS:
200 return State.TERMINATED_SUCCESS;
201 case DECLINE:
202 case BUSY:
203 return State.TERMINATED_DECLINED_OR_BUSY;
204 case CANCEL:
205 case TIMEOUT:
206 return State.TERMINATED_CANCEL_OR_TIMEOUT;
207 case SECURITY_ERROR:
208 return State.TERMINATED_SECURITY_ERROR;
209 case FAILED_APPLICATION:
210 case UNSUPPORTED_TRANSPORTS:
211 case UNSUPPORTED_APPLICATIONS:
212 return State.TERMINATED_APPLICATION_FAILURE;
213 default:
214 return State.TERMINATED_CONNECTIVITY_ERROR;
215 }
216 }
217
218 @Override
219 synchronized void deliverPacket(final JinglePacket jinglePacket) {
220 switch (jinglePacket.getAction()) {
221 case SESSION_INITIATE:
222 receiveSessionInitiate(jinglePacket);
223 break;
224 case TRANSPORT_INFO:
225 receiveTransportInfo(jinglePacket);
226 break;
227 case SESSION_ACCEPT:
228 receiveSessionAccept(jinglePacket);
229 break;
230 case SESSION_TERMINATE:
231 receiveSessionTerminate(jinglePacket);
232 break;
233 case CONTENT_ADD:
234 receiveContentAdd(jinglePacket);
235 break;
236 case CONTENT_ACCEPT:
237 receiveContentAccept(jinglePacket);
238 break;
239 case CONTENT_REJECT:
240 receiveContentReject(jinglePacket);
241 break;
242 case CONTENT_REMOVE:
243 receiveContentRemove(jinglePacket);
244 break;
245 case CONTENT_MODIFY:
246 receiveContentModify(jinglePacket);
247 break;
248 default:
249 respondOk(jinglePacket);
250 Log.d(
251 Config.LOGTAG,
252 String.format(
253 "%s: received unhandled jingle action %s",
254 id.account.getJid().asBareJid(), jinglePacket.getAction()));
255 break;
256 }
257 }
258
259 @Override
260 synchronized void notifyRebound() {
261 if (isTerminated()) {
262 return;
263 }
264 webRTCWrapper.close();
265 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
266 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
267 }
268 if (isInState(
269 State.SESSION_INITIALIZED,
270 State.SESSION_INITIALIZED_PRE_APPROVED,
271 State.SESSION_ACCEPTED)) {
272 // we might have already changed resources (full jid) at this point; so this might not
273 // even reach the other party
274 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
275 } else {
276 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
277 finish();
278 }
279 }
280
281 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
282 respondOk(jinglePacket);
283 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
284 final State previous = this.state;
285 Log.d(
286 Config.LOGTAG,
287 id.account.getJid().asBareJid()
288 + ": received session terminate reason="
289 + wrapper.reason
290 + "("
291 + Strings.nullToEmpty(wrapper.text)
292 + ") while in state "
293 + previous);
294 if (TERMINATED.contains(previous)) {
295 Log.d(
296 Config.LOGTAG,
297 id.account.getJid().asBareJid()
298 + ": ignoring session terminate because already in "
299 + previous);
300 return;
301 }
302 webRTCWrapper.close();
303 final State target = reasonToState(wrapper.reason);
304 transitionOrThrow(target);
305 writeLogMessage(target);
306 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
307 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
308 }
309 finish();
310 }
311
312 private void receiveTransportInfo(final JinglePacket jinglePacket) {
313 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
314 // INITIALIZED only after transport-info has been received
315 if (isInState(
316 State.NULL,
317 State.PROCEED,
318 State.SESSION_INITIALIZED,
319 State.SESSION_INITIALIZED_PRE_APPROVED,
320 State.SESSION_ACCEPTED)) {
321 final RtpContentMap contentMap;
322 try {
323 contentMap = RtpContentMap.of(jinglePacket);
324 } catch (final IllegalArgumentException | NullPointerException e) {
325 Log.d(
326 Config.LOGTAG,
327 id.account.getJid().asBareJid()
328 + ": improperly formatted contents; ignoring",
329 e);
330 respondOk(jinglePacket);
331 return;
332 }
333 receiveTransportInfo(jinglePacket, contentMap);
334 } else {
335 if (isTerminated()) {
336 respondOk(jinglePacket);
337 Log.d(
338 Config.LOGTAG,
339 id.account.getJid().asBareJid()
340 + ": ignoring out-of-order transport info; we where already terminated");
341 } else {
342 Log.d(
343 Config.LOGTAG,
344 id.account.getJid().asBareJid()
345 + ": received transport info while in state="
346 + this.state);
347 terminateWithOutOfOrder(jinglePacket);
348 }
349 }
350 }
351
352 private void receiveTransportInfo(
353 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
354 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
355 contentMap.contents.entrySet();
356 final RtpContentMap remote = getRemoteContentMap();
357 final Set<String> remoteContentIds = remote == null ? Collections.emptySet() : remote.contents.keySet();
358 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
359 Log.d(Config.LOGTAG,"received transport-info for unknown contents "+contentMap.contents.keySet()+" (known: "+remoteContentIds+")");
360 respondOk(jinglePacket);
361 pendingIceCandidates.addAll(candidates);
362 return;
363 }
364 if (this.state != State.SESSION_ACCEPTED) {
365 Log.d(Config.LOGTAG,"received transport-info prematurely. adding to backlog");
366 respondOk(jinglePacket);
367 pendingIceCandidates.addAll(candidates);
368 return;
369 }
370 // zero candidates + modified credentials are an ICE restart offer
371 if (checkForIceRestart(jinglePacket, contentMap)) {
372 return;
373 }
374 respondOk(jinglePacket);
375 try {
376 processCandidates(candidates);
377 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
378 Log.w(
379 Config.LOGTAG,
380 id.account.getJid().asBareJid()
381 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
382 }
383 }
384
385 private void receiveContentAdd(final JinglePacket jinglePacket) {
386 final RtpContentMap modification;
387 try {
388 modification = RtpContentMap.of(jinglePacket);
389 modification.requireContentDescriptions();
390 } catch (final RuntimeException e) {
391 Log.d(
392 Config.LOGTAG,
393 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
394 Throwables.getRootCause(e));
395 respondOk(jinglePacket);
396 webRTCWrapper.close();
397 sendSessionTerminate(Reason.of(e), e.getMessage());
398 return;
399 }
400 if (isInState(State.SESSION_ACCEPTED)) {
401 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
402 final ListenableFuture<RtpContentMap> future =
403 receiveRtpContentMap(
404 modification, this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
405 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
406 @Override
407 public void onSuccess(final RtpContentMap rtpContentMap) {
408 receiveContentAdd(jinglePacket, rtpContentMap);
409 }
410
411 @Override
412 public void onFailure(@NonNull Throwable throwable) {
413 respondOk(jinglePacket);
414 final Throwable rootCause = Throwables.getRootCause(throwable);
415 Log.d(
416 Config.LOGTAG,
417 id.account.getJid().asBareJid()
418 + ": improperly formatted contents in content-add",
419 throwable);
420 webRTCWrapper.close();
421 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
422 }
423 }, MoreExecutors.directExecutor());
424 } else {
425 terminateWithOutOfOrder(jinglePacket);
426 }
427 }
428
429 private void receiveContentAdd(
430 final JinglePacket jinglePacket, final RtpContentMap modification) {
431 final RtpContentMap remote = getRemoteContentMap();
432 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
433 respondOk(jinglePacket);
434 this.webRTCWrapper.close();
435 sendSessionTerminate(
436 Reason.FAILED_APPLICATION,
437 String.format(
438 "contents with names %s already exists",
439 Joiner.on(", ").join(modification.getNames())));
440 return;
441 }
442 final ContentAddition contentAddition =
443 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
444
445 final RtpContentMap outgoing = this.outgoingContentAdd;
446 final Set<ContentAddition.Summary> outgoingContentAddSummary =
447 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
448
449 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
450 if (isInitiator()) {
451 Log.d(
452 Config.LOGTAG,
453 id.getAccount().getJid().asBareJid()
454 + ": respond with tie break to matching content-add offer");
455 respondWithTieBreak(jinglePacket);
456 } else {
457 Log.d(
458 Config.LOGTAG,
459 id.getAccount().getJid().asBareJid()
460 + ": automatically accept matching content-add offer");
461 acceptContentAdd(contentAddition.summary, modification);
462 }
463 return;
464 }
465
466 // once we can display multiple video tracks we can be more loose with this condition
467 // theoretically it should also be fine to automatically accept audio only contents
468 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
469 Log.d(
470 Config.LOGTAG,
471 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
472 this.incomingContentAdd = modification;
473 respondOk(jinglePacket);
474 updateEndUserState();
475 } else {
476 respondOk(jinglePacket);
477 // TODO do we want to add a reason?
478 rejectContentAdd(modification);
479 }
480 }
481
482 private void receiveContentAccept(final JinglePacket jinglePacket) {
483 final RtpContentMap receivedContentAccept;
484 try {
485 receivedContentAccept = RtpContentMap.of(jinglePacket);
486 receivedContentAccept.requireContentDescriptions();
487 } catch (final RuntimeException e) {
488 Log.d(
489 Config.LOGTAG,
490 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
491 Throwables.getRootCause(e));
492 respondOk(jinglePacket);
493 webRTCWrapper.close();
494 sendSessionTerminate(Reason.of(e), e.getMessage());
495 return;
496 }
497
498 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
499 if (outgoingContentAdd == null) {
500 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
501 terminateWithOutOfOrder(jinglePacket);
502 return;
503 }
504 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
505 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
506 this.outgoingContentAdd = null;
507 respondOk(jinglePacket);
508 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
509 final ListenableFuture<RtpContentMap> future =
510 receiveRtpContentMap(
511 receivedContentAccept, this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
512 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
513 @Override
514 public void onSuccess(final RtpContentMap result) {
515 receiveContentAccept(result);
516 }
517
518 @Override
519 public void onFailure(@NonNull final Throwable throwable) {
520 webRTCWrapper.close();
521 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
522 }
523 }, MoreExecutors.directExecutor());
524 } else {
525 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
526 terminateWithOutOfOrder(jinglePacket);
527 }
528 }
529
530 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
531 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
532 final RtpContentMap modifiedContentMap =
533 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
534
535 setRemoteContentMap(modifiedContentMap);
536
537 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
538
539 final org.webrtc.SessionDescription sdp =
540 new org.webrtc.SessionDescription(
541 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
542
543 try {
544 this.webRTCWrapper.setRemoteDescription(sdp).get();
545 } catch (final Exception e) {
546 final Throwable cause = Throwables.getRootCause(e);
547 Log.d(
548 Config.LOGTAG,
549 id.getAccount().getJid().asBareJid()
550 + ": unable to set remote description after receiving content-accept",
551 cause);
552 webRTCWrapper.close();
553 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
554 return;
555 }
556 processCandidates(receivedContentAccept.contents.entrySet());
557 updateEndUserState();
558 Log.d(
559 Config.LOGTAG,
560 id.getAccount().getJid().asBareJid()
561 + ": remote has accepted content-add "
562 + ContentAddition.summary(receivedContentAccept));
563 }
564
565 private void receiveContentModify(final JinglePacket jinglePacket) {
566 // TODO check session accepted
567 final Map<String, Content.Senders> modification =
568 Maps.transformEntries(
569 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
570 final boolean isInitiator = isInitiator();
571 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
572 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
573 final Set<String> currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet();
574 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
575 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
576 respondOk(jinglePacket);
577 final RtpContentMap modifiedContentMap;
578 try {
579 modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification);
580 } catch (final IllegalArgumentException e) {
581 webRTCWrapper.close();
582 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
583 return;
584 }
585 this.outgoingContentAdd = modifiedContentMap;
586 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add");
587 } else if (remoteContentMap != null && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
588 respondOk(jinglePacket);
589 final RtpContentMap modifiedRemoteContentMap;
590 try {
591 modifiedRemoteContentMap = remoteContentMap.modifiedSendersChecked(isInitiator, modification);
592 } catch (final IllegalArgumentException e) {
593 webRTCWrapper.close();
594 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
595 return;
596 }
597 final SessionDescription offer;
598 try {
599 offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator());
600 } catch (final IllegalArgumentException | NullPointerException e) {
601 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-modify to SDP", e);
602 webRTCWrapper.close();
603 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
604 return;
605 }
606 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": auto accepting content-modification");
607 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
608 } else {
609 Log.d(Config.LOGTAG,"received unsupported content modification "+modification);
610 respondWithItemNotFound(jinglePacket);
611 }
612 }
613
614 private void autoAcceptContentModify(final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
615 this.setRemoteContentMap(modifiedRemoteContentMap);
616 final org.webrtc.SessionDescription sdp =
617 new org.webrtc.SessionDescription(
618 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
619 try {
620 this.webRTCWrapper.setRemoteDescription(sdp).get();
621 // auto accept is only done when we already have tracks
622 final SessionDescription answer = setLocalSessionDescription();
623 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
624 modifyLocalContentMap(rtpContentMap);
625 // we do not need to send an answer but do we have to resend the candidates currently in SDP?
626 //resendCandidatesFromSdp(answer);
627 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
628 } catch (final Exception e) {
629 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
630 webRTCWrapper.close();
631 sendSessionTerminate(Reason.FAILED_APPLICATION);
632 }
633 }
634
635 private void resendCandidatesFromSdp(final SessionDescription answer) {
636 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
637 for(final SessionDescription.Media media : answer.media) {
638 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
639 if (Strings.isNullOrEmpty(mid)) {
640 continue;
641 }
642 for(final String sdpCandidate : media.attributes.get("candidate")) {
643 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
644 if (candidate != null) {
645 candidateBuilder.put(mid,candidate);
646 }
647 }
648 }
649 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build();
650 sendTransportInfo(candidates);
651 }
652
653 private void receiveContentReject(final JinglePacket jinglePacket) {
654 final RtpContentMap receivedContentReject;
655 try {
656 receivedContentReject = RtpContentMap.of(jinglePacket);
657 } catch (final RuntimeException e) {
658 Log.d(
659 Config.LOGTAG,
660 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
661 Throwables.getRootCause(e));
662 respondOk(jinglePacket);
663 this.webRTCWrapper.close();
664 sendSessionTerminate(Reason.of(e), e.getMessage());
665 return;
666 }
667
668 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
669 if (outgoingContentAdd == null) {
670 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
671 terminateWithOutOfOrder(jinglePacket);
672 return;
673 }
674 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
675 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
676 this.outgoingContentAdd = null;
677 respondOk(jinglePacket);
678 Log.d(Config.LOGTAG,jinglePacket.toString());
679 receiveContentReject(ourSummary);
680 } else {
681 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
682 terminateWithOutOfOrder(jinglePacket);
683 }
684 }
685
686 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
687 try {
688 this.webRTCWrapper.removeTrack(Media.VIDEO);
689 final RtpContentMap localContentMap = customRollback();
690 modifyLocalContentMap(localContentMap);
691 } catch (final Exception e) {
692 final Throwable cause = Throwables.getRootCause(e);
693 Log.d(
694 Config.LOGTAG,
695 id.getAccount().getJid().asBareJid()
696 + ": unable to rollback local description after receiving content-reject",
697 cause);
698 webRTCWrapper.close();
699 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
700 return;
701 }
702 Log.d(
703 Config.LOGTAG,
704 id.getAccount().getJid().asBareJid()
705 + ": remote has rejected our content-add "
706 + summary);
707 }
708
709 private void receiveContentRemove(final JinglePacket jinglePacket) {
710 final RtpContentMap receivedContentRemove;
711 try {
712 receivedContentRemove = RtpContentMap.of(jinglePacket);
713 receivedContentRemove.requireContentDescriptions();
714 } catch (final RuntimeException e) {
715 Log.d(
716 Config.LOGTAG,
717 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
718 Throwables.getRootCause(e));
719 respondOk(jinglePacket);
720 this.webRTCWrapper.close();
721 sendSessionTerminate(Reason.of(e), e.getMessage());
722 return;
723 }
724 respondOk(jinglePacket);
725 receiveContentRemove(receivedContentRemove);
726 }
727
728 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
729 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
730 final Set<ContentAddition.Summary> contentAddSummary =
731 incomingContentAdd == null
732 ? Collections.emptySet()
733 : ContentAddition.summary(incomingContentAdd);
734 final Set<ContentAddition.Summary> removeSummary =
735 ContentAddition.summary(receivedContentRemove);
736 if (contentAddSummary.equals(removeSummary)) {
737 this.incomingContentAdd = null;
738 updateEndUserState();
739 } else {
740 webRTCWrapper.close();
741 sendSessionTerminate(
742 Reason.FAILED_APPLICATION,
743 String.format(
744 "%s only supports %s as a means to retract a not yet accepted %s",
745 BuildConfig.APP_NAME,
746 JinglePacket.Action.CONTENT_REMOVE,
747 JinglePacket.Action.CONTENT_ADD));
748 }
749 }
750
751 public synchronized void retractContentAdd() {
752 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
753 if (outgoingContentAdd == null) {
754 throw new IllegalStateException("Not outgoing content add");
755 }
756 try {
757 webRTCWrapper.removeTrack(Media.VIDEO);
758 final RtpContentMap localContentMap = customRollback();
759 modifyLocalContentMap(localContentMap);
760 } catch (final Exception e) {
761 final Throwable cause = Throwables.getRootCause(e);
762 Log.d(
763 Config.LOGTAG,
764 id.getAccount().getJid().asBareJid()
765 + ": unable to rollback local description after trying to retract content-add",
766 cause);
767 webRTCWrapper.close();
768 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
769 return;
770 }
771 this.outgoingContentAdd = null;
772 final JinglePacket retract =
773 outgoingContentAdd
774 .toStub()
775 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
776 this.send(retract);
777 Log.d(
778 Config.LOGTAG,
779 id.getAccount().getJid()
780 + ": retract content-add "
781 + ContentAddition.summary(outgoingContentAdd));
782 }
783
784 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
785 final SessionDescription sdp = setLocalSessionDescription();
786 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
787 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
788 this.webRTCWrapper
789 .setRemoteDescription(
790 new org.webrtc.SessionDescription(
791 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
792 .get();
793 return localRtpContentMap;
794 }
795
796 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
797 final RtpContentMap currentRemote = getRemoteContentMap();
798 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
799 if (diff.isEmpty()) {
800 throw new IllegalStateException(
801 "Unexpected rollback condition. No difference between local and remote");
802 }
803 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
804 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
805 final RtpContentMap nextRemote =
806 currentRemote.addContent(
807 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
808 return SessionDescription.of(nextRemote, !isInitiator());
809 }
810 throw new IllegalStateException(
811 "Unexpected rollback condition. Senders were not uniformly none");
812 }
813
814 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
815 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
816 if (incomingContentAdd == null) {
817 throw new IllegalStateException("No incoming content add");
818 }
819
820 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
821 this.incomingContentAdd = null;
822 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
823 Log.d(Config.LOGTAG,"senders of incoming content-add: "+senders);
824 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
825 Log.d(Config.LOGTAG,"content addition is receive only. we want to upgrade to 'both'");
826 final RtpContentMap modifiedSenders = incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
827 final JinglePacket proposedContentModification = modifiedSenders.toStub().toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
828 proposedContentModification.setTo(id.with);
829 xmppConnectionService.sendIqPacket(id.account, proposedContentModification, (account, response) -> {
830 if (response.getType() == IqPacket.TYPE.RESULT) {
831 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has accepted our upgrade to senders=both");
832 acceptContentAdd(ContentAddition.summary(modifiedSenders), modifiedSenders);
833 } else {
834 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has rejected our upgrade to senders=both");
835 acceptContentAdd(contentAddition, incomingContentAdd);
836 }
837 });
838 }
839 } else {
840 throw new IllegalStateException("Accepted content add does not match pending content-add");
841 }
842 }
843
844 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
845 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
846 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
847 this.setRemoteContentMap(modifiedContentMap);
848
849 final SessionDescription offer;
850 try {
851 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
852 } catch (final IllegalArgumentException | NullPointerException e) {
853 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
854 webRTCWrapper.close();
855 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
856 return;
857 }
858 this.incomingContentAdd = null;
859 acceptContentAdd(contentAddition, offer);
860 }
861
862 private void acceptContentAdd(
863 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
864 final org.webrtc.SessionDescription sdp =
865 new org.webrtc.SessionDescription(
866 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
867 try {
868 this.webRTCWrapper.setRemoteDescription(sdp).get();
869
870 // TODO add tracks for 'media' where contentAddition.senders matches
871
872 // TODO if senders.sending(isInitiator())
873
874 this.webRTCWrapper.addTrack(Media.VIDEO);
875
876 // TODO add additional transceivers for recv only cases
877
878 final SessionDescription answer = setLocalSessionDescription();
879 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
880
881 final RtpContentMap contentAcceptMap =
882 rtpContentMap.toContentModification(
883 Collections2.transform(contentAddition, ca -> ca.name));
884
885 Log.d(
886 Config.LOGTAG,
887 id.getAccount().getJid().asBareJid()
888 + ": sending content-accept "
889 + ContentAddition.summary(contentAcceptMap));
890
891 addIceCandidatesFromBlackLog();
892
893 modifyLocalContentMap(rtpContentMap);
894 final ListenableFuture<RtpContentMap> future = prepareOutgoingContentMap(contentAcceptMap);
895 Futures.addCallback(
896 future,
897 new FutureCallback<RtpContentMap>() {
898 @Override
899 public void onSuccess(final RtpContentMap rtpContentMap) {
900 sendContentAccept(rtpContentMap);
901 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
902 }
903
904 @Override
905 public void onFailure(@NonNull final Throwable throwable) {
906 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
907 }
908 },
909 MoreExecutors.directExecutor());
910 } catch (final Exception e) {
911 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
912 webRTCWrapper.close();
913 sendSessionTerminate(Reason.FAILED_APPLICATION);
914 }
915 }
916
917 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
918 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
919 send(jinglePacket);
920 }
921
922 public synchronized void rejectContentAdd() {
923 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
924 if (incomingContentAdd == null) {
925 throw new IllegalStateException("No incoming content add");
926 }
927 this.incomingContentAdd = null;
928 updateEndUserState();
929 rejectContentAdd(incomingContentAdd);
930 }
931
932 private void rejectContentAdd(final RtpContentMap contentMap) {
933 final JinglePacket jinglePacket =
934 contentMap
935 .toStub()
936 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
937 Log.d(
938 Config.LOGTAG,
939 id.getAccount().getJid().asBareJid()
940 + ": rejecting content "
941 + ContentAddition.summary(contentMap));
942 send(jinglePacket);
943 }
944
945 private boolean checkForIceRestart(
946 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
947 final RtpContentMap existing = getRemoteContentMap();
948 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
949 final IceUdpTransportInfo.Credentials newCredentials;
950 try {
951 existingCredentials = existing.getCredentials();
952 newCredentials = rtpContentMap.getDistinctCredentials();
953 } catch (final IllegalStateException e) {
954 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
955 return false;
956 }
957 if (existingCredentials.contains(newCredentials)) {
958 return false;
959 }
960 // TODO an alternative approach is to check if we already got an iq result to our
961 // ICE-restart
962 // and if that's the case we are seeing an answer.
963 // This might be more spec compliant but also more error prone potentially
964 final boolean isSignalStateStable = this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
965 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
966 final boolean isOffer = rtpContentMap.emptyCandidates();
967 final RtpContentMap restartContentMap;
968 try {
969 if (isOffer) {
970 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
971 restartContentMap =
972 existing.modifiedCredentials(
973 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
974 } else {
975 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
976 Log.d(
977 Config.LOGTAG,
978 "received confirmation of ICE restart"
979 + newCredentials
980 + " peer_setup="
981 + setup);
982 // DTLS setup attribute needs to be rewritten to reflect current peer state
983 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
984 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
985 }
986 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
987 return isOffer;
988 } else {
989 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
990 respondWithTieBreak(jinglePacket);
991 return true;
992 }
993 } catch (final Exception exception) {
994 respondOk(jinglePacket);
995 final Throwable rootCause = Throwables.getRootCause(exception);
996 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
997 // If this happens a termination is already in progress
998 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
999 return true;
1000 }
1001 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
1002 webRTCWrapper.close();
1003 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1004 return true;
1005 }
1006 }
1007
1008 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
1009 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
1010 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
1011 throw new IllegalStateException("Invalid peer setup");
1012 }
1013 return peerSetup;
1014 }
1015
1016 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
1017 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
1018 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
1019 }
1020 this.peerDtlsSetup = setup;
1021 }
1022
1023 private boolean applyIceRestart(
1024 final JinglePacket jinglePacket,
1025 final RtpContentMap restartContentMap,
1026 final boolean isOffer)
1027 throws ExecutionException, InterruptedException {
1028 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
1029 final org.webrtc.SessionDescription.Type type =
1030 isOffer
1031 ? org.webrtc.SessionDescription.Type.OFFER
1032 : org.webrtc.SessionDescription.Type.ANSWER;
1033 org.webrtc.SessionDescription sdp =
1034 new org.webrtc.SessionDescription(type, sessionDescription.toString());
1035 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
1036 if (isInitiator()) {
1037 // We ignore the offer and respond with tie-break. This will clause the responder
1038 // not to apply the content map
1039 return false;
1040 }
1041 }
1042 webRTCWrapper.setRemoteDescription(sdp).get();
1043 setRemoteContentMap(restartContentMap);
1044 if (isOffer) {
1045 final SessionDescription localSessionDescription = setLocalSessionDescription();
1046 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1047 // We need to respond OK before sending any candidates
1048 respondOk(jinglePacket);
1049 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1050 } else {
1051 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1052 }
1053 return true;
1054 }
1055
1056 private void processCandidates(
1057 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
1058 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
1059 processCandidate(content);
1060 }
1061 }
1062
1063 private void processCandidate(
1064 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
1065 final RtpContentMap rtpContentMap = getRemoteContentMap();
1066 final List<String> indices = toIdentificationTags(rtpContentMap);
1067 final String sdpMid = content.getKey(); // aka content name
1068 final IceUdpTransportInfo transport = content.getValue().transport;
1069 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1070
1071 // TODO check that credentials remained the same
1072
1073 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1074 final String sdp;
1075 try {
1076 sdp = candidate.toSdpAttribute(credentials.ufrag);
1077 } catch (final IllegalArgumentException e) {
1078 Log.d(
1079 Config.LOGTAG,
1080 id.account.getJid().asBareJid()
1081 + ": ignoring invalid ICE candidate "
1082 + e.getMessage());
1083 continue;
1084 }
1085 final int mLineIndex = indices.indexOf(sdpMid);
1086 if (mLineIndex < 0) {
1087 Log.w(
1088 Config.LOGTAG,
1089 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1090 }
1091 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1092 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1093 this.webRTCWrapper.addIceCandidate(iceCandidate);
1094 }
1095 }
1096
1097 private RtpContentMap getRemoteContentMap() {
1098 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1099 }
1100
1101 private RtpContentMap getLocalContentMap() {
1102 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1103 }
1104
1105 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1106 final Group originalGroup = rtpContentMap.group;
1107 final List<String> identificationTags =
1108 originalGroup == null
1109 ? rtpContentMap.getNames()
1110 : originalGroup.getIdentificationTags();
1111 if (identificationTags.size() == 0) {
1112 Log.w(
1113 Config.LOGTAG,
1114 id.account.getJid().asBareJid()
1115 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1116 }
1117 return identificationTags;
1118 }
1119
1120 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1121 final JinglePacket jinglePacket, final boolean expectVerification) {
1122 try {
1123 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1124 } catch (final Exception e) {
1125 return Futures.immediateFailedFuture(e);
1126 }
1127 }
1128 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final RtpContentMap receivedContentMap, final boolean expectVerification) {
1129 Log.d(
1130 Config.LOGTAG,
1131 "receiveRtpContentMap("
1132 + receivedContentMap.getClass().getSimpleName()
1133 + ",expectVerification="
1134 + expectVerification
1135 + ")");
1136 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1137 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1138 id.account
1139 .getAxolotlService()
1140 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1141 return Futures.transform(
1142 future,
1143 omemoVerifiedPayload -> {
1144 // TODO test if an exception here triggers a correct abort
1145 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1146 Log.d(
1147 Config.LOGTAG,
1148 id.account.getJid().asBareJid()
1149 + ": received verifiable DTLS fingerprint via "
1150 + omemoVerification);
1151 return omemoVerifiedPayload.getPayload();
1152 },
1153 MoreExecutors.directExecutor());
1154 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1155 return Futures.immediateFailedFuture(
1156 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1157 } else {
1158 return Futures.immediateFuture(receivedContentMap);
1159 }
1160 }
1161
1162 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1163 if (isInitiator()) {
1164 Log.d(
1165 Config.LOGTAG,
1166 String.format(
1167 "%s: received session-initiate even though we were initiating",
1168 id.account.getJid().asBareJid()));
1169 if (isTerminated()) {
1170 Log.d(
1171 Config.LOGTAG,
1172 String.format(
1173 "%s: got a reason to terminate with out-of-order. but already in state %s",
1174 id.account.getJid().asBareJid(), getState()));
1175 respondWithOutOfOrder(jinglePacket);
1176 } else {
1177 terminateWithOutOfOrder(jinglePacket);
1178 }
1179 return;
1180 }
1181 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1182 Futures.addCallback(
1183 future,
1184 new FutureCallback<RtpContentMap>() {
1185 @Override
1186 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1187 receiveSessionInitiate(jinglePacket, rtpContentMap);
1188 }
1189
1190 @Override
1191 public void onFailure(@NonNull final Throwable throwable) {
1192 respondOk(jinglePacket);
1193 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1194 }
1195 },
1196 MoreExecutors.directExecutor());
1197 }
1198
1199 private void receiveSessionInitiate(
1200 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1201 try {
1202 contentMap.requireContentDescriptions();
1203 contentMap.requireDTLSFingerprint(true);
1204 } catch (final RuntimeException e) {
1205 Log.d(
1206 Config.LOGTAG,
1207 id.account.getJid().asBareJid() + ": improperly formatted contents",
1208 Throwables.getRootCause(e));
1209 respondOk(jinglePacket);
1210 sendSessionTerminate(Reason.of(e), e.getMessage());
1211 return;
1212 }
1213 Log.d(
1214 Config.LOGTAG,
1215 "processing session-init with " + contentMap.contents.size() + " contents");
1216 final State target;
1217 if (this.state == State.PROCEED) {
1218 Preconditions.checkState(
1219 proposedMedia != null && proposedMedia.size() > 0,
1220 "proposed media must be set when processing pre-approved session-initiate");
1221 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1222 sendSessionTerminate(
1223 Reason.SECURITY_ERROR,
1224 String.format(
1225 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1226 this.proposedMedia, contentMap.getMedia()));
1227 return;
1228 }
1229 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1230 } else {
1231 target = State.SESSION_INITIALIZED;
1232 }
1233 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1234 respondOk(jinglePacket);
1235 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1236 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1237 Log.d(
1238 Config.LOGTAG,
1239 id.account.getJid().asBareJid()
1240 + ": automatically accepting session-initiate");
1241 sendSessionAccept();
1242 } else {
1243 Log.d(
1244 Config.LOGTAG,
1245 id.account.getJid().asBareJid()
1246 + ": received not pre-approved session-initiate. start ringing");
1247 startRinging();
1248 }
1249 } else {
1250 Log.d(
1251 Config.LOGTAG,
1252 String.format(
1253 "%s: received session-initiate while in state %s",
1254 id.account.getJid().asBareJid(), state));
1255 terminateWithOutOfOrder(jinglePacket);
1256 }
1257 }
1258
1259 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1260 if (!isInitiator()) {
1261 Log.d(
1262 Config.LOGTAG,
1263 String.format(
1264 "%s: received session-accept even though we were responding",
1265 id.account.getJid().asBareJid()));
1266 terminateWithOutOfOrder(jinglePacket);
1267 return;
1268 }
1269 final ListenableFuture<RtpContentMap> future =
1270 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1271 Futures.addCallback(
1272 future,
1273 new FutureCallback<RtpContentMap>() {
1274 @Override
1275 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1276 receiveSessionAccept(jinglePacket, rtpContentMap);
1277 }
1278
1279 @Override
1280 public void onFailure(@NonNull final Throwable throwable) {
1281 respondOk(jinglePacket);
1282 Log.d(
1283 Config.LOGTAG,
1284 id.account.getJid().asBareJid()
1285 + ": improperly formatted contents in session-accept",
1286 throwable);
1287 webRTCWrapper.close();
1288 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1289 }
1290 },
1291 MoreExecutors.directExecutor());
1292 }
1293
1294 private void receiveSessionAccept(
1295 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1296 try {
1297 contentMap.requireContentDescriptions();
1298 contentMap.requireDTLSFingerprint();
1299 } catch (final RuntimeException e) {
1300 respondOk(jinglePacket);
1301 Log.d(
1302 Config.LOGTAG,
1303 id.account.getJid().asBareJid()
1304 + ": improperly formatted contents in session-accept",
1305 e);
1306 webRTCWrapper.close();
1307 sendSessionTerminate(Reason.of(e), e.getMessage());
1308 return;
1309 }
1310 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1311 if (!initiatorMedia.equals(contentMap.getMedia())) {
1312 sendSessionTerminate(
1313 Reason.SECURITY_ERROR,
1314 String.format(
1315 "Your session-included included media %s but our session-initiate was %s",
1316 this.proposedMedia, contentMap.getMedia()));
1317 return;
1318 }
1319 Log.d(
1320 Config.LOGTAG,
1321 "processing session-accept with " + contentMap.contents.size() + " contents");
1322 if (transition(State.SESSION_ACCEPTED)) {
1323 respondOk(jinglePacket);
1324 receiveSessionAccept(contentMap);
1325 } else {
1326 Log.d(
1327 Config.LOGTAG,
1328 String.format(
1329 "%s: received session-accept while in state %s",
1330 id.account.getJid().asBareJid(), state));
1331 respondOk(jinglePacket);
1332 }
1333 }
1334
1335 private void receiveSessionAccept(final RtpContentMap contentMap) {
1336 this.responderRtpContentMap = contentMap;
1337 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1338 final SessionDescription sessionDescription;
1339 try {
1340 sessionDescription = SessionDescription.of(contentMap, false);
1341 } catch (final IllegalArgumentException | NullPointerException e) {
1342 Log.d(
1343 Config.LOGTAG,
1344 id.account.getJid().asBareJid()
1345 + ": unable convert offer from session-accept to SDP",
1346 e);
1347 webRTCWrapper.close();
1348 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1349 return;
1350 }
1351 final org.webrtc.SessionDescription answer =
1352 new org.webrtc.SessionDescription(
1353 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1354 try {
1355 this.webRTCWrapper.setRemoteDescription(answer).get();
1356 } catch (final Exception e) {
1357 Log.d(
1358 Config.LOGTAG,
1359 id.account.getJid().asBareJid()
1360 + ": unable to set remote description after receiving session-accept",
1361 Throwables.getRootCause(e));
1362 webRTCWrapper.close();
1363 sendSessionTerminate(
1364 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1365 return;
1366 }
1367 processCandidates(contentMap.contents.entrySet());
1368 }
1369
1370 private void sendSessionAccept() {
1371 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1372 if (rtpContentMap == null) {
1373 throw new IllegalStateException("initiator RTP Content Map has not been set");
1374 }
1375 final SessionDescription offer;
1376 try {
1377 offer = SessionDescription.of(rtpContentMap, true);
1378 } catch (final IllegalArgumentException | NullPointerException e) {
1379 Log.d(
1380 Config.LOGTAG,
1381 id.account.getJid().asBareJid()
1382 + ": unable convert offer from session-initiate to SDP",
1383 e);
1384 webRTCWrapper.close();
1385 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1386 return;
1387 }
1388 sendSessionAccept(rtpContentMap.getMedia(), offer);
1389 }
1390
1391 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1392 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1393 }
1394
1395 private synchronized void sendSessionAccept(
1396 final Set<Media> media,
1397 final SessionDescription offer,
1398 final List<PeerConnection.IceServer> iceServers) {
1399 if (isTerminated()) {
1400 Log.w(
1401 Config.LOGTAG,
1402 id.account.getJid().asBareJid()
1403 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1404 return;
1405 }
1406 try {
1407 setupWebRTC(media, iceServers);
1408 } catch (final WebRTCWrapper.InitializationException e) {
1409 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1410 webRTCWrapper.close();
1411 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1412 return;
1413 }
1414 final org.webrtc.SessionDescription sdp =
1415 new org.webrtc.SessionDescription(
1416 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1417 try {
1418 this.webRTCWrapper.setRemoteDescription(sdp).get();
1419 addIceCandidatesFromBlackLog();
1420 org.webrtc.SessionDescription webRTCSessionDescription =
1421 this.webRTCWrapper.setLocalDescription().get();
1422 prepareSessionAccept(webRTCSessionDescription);
1423 } catch (final Exception e) {
1424 failureToAcceptSession(e);
1425 }
1426 }
1427
1428 private void failureToAcceptSession(final Throwable throwable) {
1429 if (isTerminated()) {
1430 return;
1431 }
1432 final Throwable rootCause = Throwables.getRootCause(throwable);
1433 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1434 webRTCWrapper.close();
1435 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1436 }
1437
1438 private void failureToPerformAction(final JinglePacket.Action action, final Throwable throwable) {
1439 if (isTerminated()) {
1440 return;
1441 }
1442 final Throwable rootCause = Throwables.getRootCause(throwable);
1443 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1444 webRTCWrapper.close();
1445 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1446 }
1447
1448 private void addIceCandidatesFromBlackLog() {
1449 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1450 while ((foo = this.pendingIceCandidates.poll()) != null) {
1451 processCandidate(foo);
1452 Log.d(
1453 Config.LOGTAG,
1454 id.account.getJid().asBareJid() + ": added candidate from back log");
1455 }
1456 }
1457
1458 private void prepareSessionAccept(
1459 final org.webrtc.SessionDescription webRTCSessionDescription) {
1460 final SessionDescription sessionDescription =
1461 SessionDescription.parse(webRTCSessionDescription.description);
1462 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1463 this.responderRtpContentMap = respondingRtpContentMap;
1464 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1465 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1466 prepareOutgoingContentMap(respondingRtpContentMap);
1467 Futures.addCallback(
1468 outgoingContentMapFuture,
1469 new FutureCallback<RtpContentMap>() {
1470 @Override
1471 public void onSuccess(final RtpContentMap outgoingContentMap) {
1472 sendSessionAccept(outgoingContentMap);
1473 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1474 }
1475
1476 @Override
1477 public void onFailure(@NonNull Throwable throwable) {
1478 failureToAcceptSession(throwable);
1479 }
1480 },
1481 MoreExecutors.directExecutor());
1482 }
1483
1484 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1485 if (isTerminated()) {
1486 Log.w(
1487 Config.LOGTAG,
1488 id.account.getJid().asBareJid()
1489 + ": preparing session accept was too slow. already terminated. nothing to do.");
1490 return;
1491 }
1492 transitionOrThrow(State.SESSION_ACCEPTED);
1493 final JinglePacket sessionAccept =
1494 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1495 send(sessionAccept);
1496 }
1497
1498 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1499 final RtpContentMap rtpContentMap) {
1500 if (this.omemoVerification.hasDeviceId()) {
1501 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1502 verifiedPayloadFuture =
1503 id.account
1504 .getAxolotlService()
1505 .encrypt(
1506 rtpContentMap,
1507 id.with,
1508 omemoVerification.getDeviceId());
1509 return Futures.transform(
1510 verifiedPayloadFuture,
1511 verifiedPayload -> {
1512 omemoVerification.setOrEnsureEqual(verifiedPayload);
1513 return verifiedPayload.getPayload();
1514 },
1515 MoreExecutors.directExecutor());
1516 } else {
1517 return Futures.immediateFuture(rtpContentMap);
1518 }
1519 }
1520
1521 synchronized void deliveryMessage(
1522 final Jid from,
1523 final Element message,
1524 final String serverMessageId,
1525 final long timestamp) {
1526 Log.d(
1527 Config.LOGTAG,
1528 id.account.getJid().asBareJid()
1529 + ": delivered message to JingleRtpConnection "
1530 + message);
1531 switch (message.getName()) {
1532 case "propose":
1533 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1534 break;
1535 case "proceed":
1536 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1537 break;
1538 case "retract":
1539 receiveRetract(from, serverMessageId, timestamp);
1540 break;
1541 case "reject":
1542 receiveReject(from, serverMessageId, timestamp);
1543 break;
1544 case "accept":
1545 receiveAccept(from, serverMessageId, timestamp);
1546 break;
1547 default:
1548 break;
1549 }
1550 }
1551
1552 void deliverFailedProceed(final String message) {
1553 Log.d(
1554 Config.LOGTAG,
1555 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1556 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1557 webRTCWrapper.close();
1558 Log.d(
1559 Config.LOGTAG,
1560 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1561 this.finish();
1562 }
1563 }
1564
1565 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1566 final boolean originatedFromMyself =
1567 from.asBareJid().equals(id.account.getJid().asBareJid());
1568 if (originatedFromMyself) {
1569 if (transition(State.ACCEPTED)) {
1570 acceptedOnOtherDevice(serverMsgId, timestamp);
1571 } else {
1572 Log.d(
1573 Config.LOGTAG,
1574 id.account.getJid().asBareJid()
1575 + ": unable to transition to accept because already in state="
1576 + this.state);
1577 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1578 }
1579 } else {
1580 Log.d(
1581 Config.LOGTAG,
1582 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1583 }
1584 }
1585
1586 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1587 if (serverMsgId != null) {
1588 this.message.setServerMsgId(serverMsgId);
1589 }
1590 this.message.setTime(timestamp);
1591 this.message.setCarbon(true); // indicate that call was accepted on other device
1592 this.writeLogMessageSuccess(0);
1593 this.xmppConnectionService
1594 .getNotificationService()
1595 .cancelIncomingCallNotification();
1596 this.finish();
1597 }
1598
1599 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1600 final boolean originatedFromMyself =
1601 from.asBareJid().equals(id.account.getJid().asBareJid());
1602 // reject from another one of my clients
1603 if (originatedFromMyself) {
1604 receiveRejectFromMyself(serverMsgId, timestamp);
1605 } else if (isInitiator()) {
1606 if (from.equals(id.with)) {
1607 receiveRejectFromResponder();
1608 } else {
1609 Log.d(
1610 Config.LOGTAG,
1611 id.account.getJid()
1612 + ": ignoring reject from "
1613 + from
1614 + " for session with "
1615 + id.with);
1616 }
1617 } else {
1618 Log.d(
1619 Config.LOGTAG,
1620 id.account.getJid()
1621 + ": ignoring reject from "
1622 + from
1623 + " for session with "
1624 + id.with);
1625 }
1626 }
1627
1628 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1629 if (transition(State.REJECTED)) {
1630 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1631 this.finish();
1632 if (serverMsgId != null) {
1633 this.message.setServerMsgId(serverMsgId);
1634 }
1635 this.message.setTime(timestamp);
1636 this.message.setCarbon(true); // indicate that call was rejected on other device
1637 writeLogMessageMissed();
1638 } else {
1639 Log.d(
1640 Config.LOGTAG,
1641 "not able to transition into REJECTED because already in " + this.state);
1642 }
1643 }
1644
1645 private void receiveRejectFromResponder() {
1646 if (isInState(State.PROCEED)) {
1647 Log.d(
1648 Config.LOGTAG,
1649 id.account.getJid()
1650 + ": received reject while still in proceed. callee reconsidered");
1651 closeTransitionLogFinish(State.REJECTED_RACED);
1652 return;
1653 }
1654 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1655 Log.d(
1656 Config.LOGTAG,
1657 id.account.getJid()
1658 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1659 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1660 return;
1661 }
1662 Log.d(
1663 Config.LOGTAG,
1664 id.account.getJid()
1665 + ": ignoring reject from responder because already in state "
1666 + this.state);
1667 }
1668
1669 private void receivePropose(
1670 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1671 final boolean originatedFromMyself =
1672 from.asBareJid().equals(id.account.getJid().asBareJid());
1673 if (originatedFromMyself) {
1674 Log.d(
1675 Config.LOGTAG,
1676 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1677 } else if (transition(
1678 State.PROPOSED,
1679 () -> {
1680 final Collection<RtpDescription> descriptions =
1681 Collections2.transform(
1682 Collections2.filter(
1683 propose.getDescriptions(),
1684 d -> d instanceof RtpDescription),
1685 input -> (RtpDescription) input);
1686 final Collection<Media> media =
1687 Collections2.transform(descriptions, RtpDescription::getMedia);
1688 Preconditions.checkState(
1689 !media.contains(Media.UNKNOWN),
1690 "RTP descriptions contain unknown media");
1691 Log.d(
1692 Config.LOGTAG,
1693 id.account.getJid().asBareJid()
1694 + ": received session proposal from "
1695 + from
1696 + " for "
1697 + media);
1698 this.proposedMedia = Sets.newHashSet(media);
1699 })) {
1700 if (serverMsgId != null) {
1701 this.message.setServerMsgId(serverMsgId);
1702 }
1703 this.message.setTime(timestamp);
1704 startRinging();
1705 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1706 sendJingleMessage("ringing");
1707 }
1708 } else {
1709 Log.d(
1710 Config.LOGTAG,
1711 id.account.getJid()
1712 + ": ignoring session proposal because already in "
1713 + state);
1714 }
1715 }
1716
1717 private void startRinging() {
1718 Log.d(
1719 Config.LOGTAG,
1720 id.account.getJid().asBareJid()
1721 + ": received call from "
1722 + id.with
1723 + ". start ringing");
1724 ringingTimeoutFuture =
1725 jingleConnectionManager.schedule(
1726 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1727 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1728 }
1729
1730 private synchronized void ringingTimeout() {
1731 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1732 switch (this.state) {
1733 case PROPOSED:
1734 message.markUnread();
1735 rejectCallFromProposed();
1736 break;
1737 case SESSION_INITIALIZED:
1738 message.markUnread();
1739 rejectCallFromSessionInitiate();
1740 break;
1741 }
1742 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1743 }
1744
1745 private void cancelRingingTimeout() {
1746 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1747 if (future != null && !future.isCancelled()) {
1748 future.cancel(false);
1749 }
1750 }
1751
1752 private void receiveProceed(
1753 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1754 final Set<Media> media =
1755 Preconditions.checkNotNull(
1756 this.proposedMedia, "Proposed media has to be set before handling proceed");
1757 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1758 if (from.equals(id.with)) {
1759 if (isInitiator()) {
1760 if (transition(State.PROCEED)) {
1761 if (serverMsgId != null) {
1762 this.message.setServerMsgId(serverMsgId);
1763 }
1764 this.message.setTime(timestamp);
1765 final Integer remoteDeviceId = proceed.getDeviceId();
1766 if (isOmemoEnabled()) {
1767 this.omemoVerification.setDeviceId(remoteDeviceId);
1768 } else {
1769 if (remoteDeviceId != null) {
1770 Log.d(
1771 Config.LOGTAG,
1772 id.account.getJid().asBareJid()
1773 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1774 }
1775 this.omemoVerification.setDeviceId(null);
1776 }
1777 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1778 } else {
1779 Log.d(
1780 Config.LOGTAG,
1781 String.format(
1782 "%s: ignoring proceed because already in %s",
1783 id.account.getJid().asBareJid(), this.state));
1784 }
1785 } else {
1786 Log.d(
1787 Config.LOGTAG,
1788 String.format(
1789 "%s: ignoring proceed because we were not initializing",
1790 id.account.getJid().asBareJid()));
1791 }
1792 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1793 if (transition(State.ACCEPTED)) {
1794 Log.d(
1795 Config.LOGTAG,
1796 id.account.getJid().asBareJid()
1797 + ": moved session with "
1798 + id.with
1799 + " into state accepted after received carbon copied proceed");
1800 acceptedOnOtherDevice(serverMsgId, timestamp);
1801 }
1802 } else {
1803 Log.d(
1804 Config.LOGTAG,
1805 String.format(
1806 "%s: ignoring proceed from %s. was expected from %s",
1807 id.account.getJid().asBareJid(), from, id.with));
1808 }
1809 }
1810
1811 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1812 if (from.equals(id.with)) {
1813 final State target =
1814 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1815 if (transition(target)) {
1816 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1817 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1818 Log.d(
1819 Config.LOGTAG,
1820 id.account.getJid().asBareJid()
1821 + ": session with "
1822 + id.with
1823 + " has been retracted (serverMsgId="
1824 + serverMsgId
1825 + ")");
1826 if (serverMsgId != null) {
1827 this.message.setServerMsgId(serverMsgId);
1828 }
1829 this.message.setTime(timestamp);
1830 if (target == State.RETRACTED) {
1831 this.message.markUnread();
1832 }
1833 writeLogMessageMissed();
1834 finish();
1835 } else {
1836 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1837 }
1838 } else {
1839 // TODO parse retract from self
1840 Log.d(
1841 Config.LOGTAG,
1842 id.account.getJid().asBareJid()
1843 + ": received retract from "
1844 + from
1845 + ". expected retract from"
1846 + id.with
1847 + ". ignoring");
1848 }
1849 }
1850
1851 public void sendSessionInitiate() {
1852 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1853 }
1854
1855 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1856 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1857 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1858 }
1859
1860 private synchronized void sendSessionInitiate(
1861 final Set<Media> media,
1862 final State targetState,
1863 final List<PeerConnection.IceServer> iceServers) {
1864 if (isTerminated()) {
1865 Log.w(
1866 Config.LOGTAG,
1867 id.account.getJid().asBareJid()
1868 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1869 return;
1870 }
1871 try {
1872 setupWebRTC(media, iceServers);
1873 } catch (final WebRTCWrapper.InitializationException e) {
1874 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1875 webRTCWrapper.close();
1876 sendRetract(Reason.ofThrowable(e));
1877 return;
1878 }
1879 try {
1880 org.webrtc.SessionDescription webRTCSessionDescription =
1881 this.webRTCWrapper.setLocalDescription().get();
1882 prepareSessionInitiate(webRTCSessionDescription, targetState);
1883 } catch (final Exception e) {
1884 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1885 // exceptions
1886 failureToInitiateSession(e, targetState);
1887 }
1888 }
1889
1890 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1891 if (isTerminated()) {
1892 return;
1893 }
1894 Log.d(
1895 Config.LOGTAG,
1896 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1897 Throwables.getRootCause(throwable));
1898 webRTCWrapper.close();
1899 final Reason reason = Reason.ofThrowable(throwable);
1900 if (isInState(targetState)) {
1901 sendSessionTerminate(reason, throwable.getMessage());
1902 } else {
1903 sendRetract(reason);
1904 }
1905 }
1906
1907 private void sendRetract(final Reason reason) {
1908 // TODO embed reason into retract
1909 sendJingleMessage("retract", id.with.asBareJid());
1910 transitionOrThrow(reasonToState(reason));
1911 this.finish();
1912 }
1913
1914 private void prepareSessionInitiate(
1915 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1916 final SessionDescription sessionDescription =
1917 SessionDescription.parse(webRTCSessionDescription.description);
1918 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1919 this.initiatorRtpContentMap = rtpContentMap;
1920 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1921 encryptSessionInitiate(rtpContentMap);
1922 Futures.addCallback(
1923 outgoingContentMapFuture,
1924 new FutureCallback<RtpContentMap>() {
1925 @Override
1926 public void onSuccess(final RtpContentMap outgoingContentMap) {
1927 sendSessionInitiate(outgoingContentMap, targetState);
1928 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1929 }
1930
1931 @Override
1932 public void onFailure(@NonNull final Throwable throwable) {
1933 failureToInitiateSession(throwable, targetState);
1934 }
1935 },
1936 MoreExecutors.directExecutor());
1937 }
1938
1939 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1940 if (isTerminated()) {
1941 Log.w(
1942 Config.LOGTAG,
1943 id.account.getJid().asBareJid()
1944 + ": preparing session was too slow. already terminated. nothing to do.");
1945 return;
1946 }
1947 this.transitionOrThrow(targetState);
1948 final JinglePacket sessionInitiate =
1949 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1950 send(sessionInitiate);
1951 }
1952
1953 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1954 final RtpContentMap rtpContentMap) {
1955 if (this.omemoVerification.hasDeviceId()) {
1956 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1957 verifiedPayloadFuture =
1958 id.account
1959 .getAxolotlService()
1960 .encrypt(
1961 rtpContentMap,
1962 id.with,
1963 omemoVerification.getDeviceId());
1964 final ListenableFuture<RtpContentMap> future =
1965 Futures.transform(
1966 verifiedPayloadFuture,
1967 verifiedPayload -> {
1968 omemoVerification.setSessionFingerprint(
1969 verifiedPayload.getFingerprint());
1970 return verifiedPayload.getPayload();
1971 },
1972 MoreExecutors.directExecutor());
1973 if (Config.REQUIRE_RTP_VERIFICATION) {
1974 return future;
1975 }
1976 return Futures.catching(
1977 future,
1978 CryptoFailedException.class,
1979 e -> {
1980 Log.w(
1981 Config.LOGTAG,
1982 id.account.getJid().asBareJid()
1983 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1984 e);
1985 return rtpContentMap;
1986 },
1987 MoreExecutors.directExecutor());
1988 } else {
1989 return Futures.immediateFuture(rtpContentMap);
1990 }
1991 }
1992
1993 private void sendSessionTerminate(final Reason reason) {
1994 sendSessionTerminate(reason, null);
1995 }
1996
1997 private void sendSessionTerminate(final Reason reason, final String text) {
1998 final State previous = this.state;
1999 final State target = reasonToState(reason);
2000 transitionOrThrow(target);
2001 if (previous != State.NULL) {
2002 writeLogMessage(target);
2003 }
2004 final JinglePacket jinglePacket =
2005 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
2006 jinglePacket.setReason(reason, text);
2007 Log.d(Config.LOGTAG, jinglePacket.toString());
2008 send(jinglePacket);
2009 finish();
2010 }
2011
2012 private void sendTransportInfo(
2013 final String contentName, IceUdpTransportInfo.Candidate candidate) {
2014 final RtpContentMap transportInfo;
2015 try {
2016 final RtpContentMap rtpContentMap =
2017 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2018 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
2019 } catch (final Exception e) {
2020 Log.d(
2021 Config.LOGTAG,
2022 id.account.getJid().asBareJid()
2023 + ": unable to prepare transport-info from candidate for content="
2024 + contentName);
2025 return;
2026 }
2027 final JinglePacket jinglePacket =
2028 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2029 send(jinglePacket);
2030 }
2031
2032 private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
2033 // TODO send all candidates in one transport-info
2034 }
2035
2036
2037 private void send(final JinglePacket jinglePacket) {
2038 jinglePacket.setTo(id.with);
2039 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
2040 }
2041
2042 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
2043 if (response.getType() == IqPacket.TYPE.ERROR) {
2044 handleIqErrorResponse(response);
2045 return;
2046 }
2047 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2048 handleIqTimeoutResponse(response);
2049 }
2050 }
2051
2052 private void handleIqErrorResponse(final IqPacket response) {
2053 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
2054 final String errorCondition = response.getErrorCondition();
2055 Log.d(
2056 Config.LOGTAG,
2057 id.account.getJid().asBareJid()
2058 + ": received IQ-error from "
2059 + response.getFrom()
2060 + " in RTP session. "
2061 + errorCondition);
2062 if (isTerminated()) {
2063 Log.i(
2064 Config.LOGTAG,
2065 id.account.getJid().asBareJid()
2066 + ": ignoring error because session was already terminated");
2067 return;
2068 }
2069 this.webRTCWrapper.close();
2070 final State target;
2071 if (Arrays.asList(
2072 "service-unavailable",
2073 "recipient-unavailable",
2074 "remote-server-not-found",
2075 "remote-server-timeout")
2076 .contains(errorCondition)) {
2077 target = State.TERMINATED_CONNECTIVITY_ERROR;
2078 } else {
2079 target = State.TERMINATED_APPLICATION_FAILURE;
2080 }
2081 transitionOrThrow(target);
2082 this.finish();
2083 }
2084
2085 private void handleIqTimeoutResponse(final IqPacket response) {
2086 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
2087 Log.d(
2088 Config.LOGTAG,
2089 id.account.getJid().asBareJid()
2090 + ": received IQ timeout in RTP session with "
2091 + id.with
2092 + ". terminating with connectivity error");
2093 if (isTerminated()) {
2094 Log.i(
2095 Config.LOGTAG,
2096 id.account.getJid().asBareJid()
2097 + ": ignoring error because session was already terminated");
2098 return;
2099 }
2100 this.webRTCWrapper.close();
2101 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
2102 this.finish();
2103 }
2104
2105 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
2106 Log.d(
2107 Config.LOGTAG,
2108 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
2109 this.webRTCWrapper.close();
2110 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
2111 respondWithOutOfOrder(jinglePacket);
2112 this.finish();
2113 }
2114
2115 private void respondWithTieBreak(final JinglePacket jinglePacket) {
2116 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
2117 }
2118
2119 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
2120 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
2121 }
2122
2123 private void respondWithItemNotFound(final JinglePacket jinglePacket) {
2124 respondWithJingleError(jinglePacket, null, "item-not-found", "cancel");
2125 }
2126
2127 void respondWithJingleError(
2128 final IqPacket original,
2129 String jingleCondition,
2130 String condition,
2131 String conditionType) {
2132 jingleConnectionManager.respondWithJingleError(
2133 id.account, original, jingleCondition, condition, conditionType);
2134 }
2135
2136 private void respondOk(final JinglePacket jinglePacket) {
2137 xmppConnectionService.sendIqPacket(
2138 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
2139 }
2140
2141 public RtpEndUserState getEndUserState() {
2142 switch (this.state) {
2143 case NULL:
2144 case PROPOSED:
2145 case SESSION_INITIALIZED:
2146 if (isInitiator()) {
2147 return RtpEndUserState.RINGING;
2148 } else {
2149 return RtpEndUserState.INCOMING_CALL;
2150 }
2151 case PROCEED:
2152 if (isInitiator()) {
2153 return RtpEndUserState.RINGING;
2154 } else {
2155 return RtpEndUserState.ACCEPTING_CALL;
2156 }
2157 case SESSION_INITIALIZED_PRE_APPROVED:
2158 if (isInitiator()) {
2159 return RtpEndUserState.RINGING;
2160 } else {
2161 return RtpEndUserState.CONNECTING;
2162 }
2163 case SESSION_ACCEPTED:
2164 final ContentAddition ca = getPendingContentAddition();
2165 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2166 return RtpEndUserState.INCOMING_CONTENT_ADD;
2167 }
2168 return getPeerConnectionStateAsEndUserState();
2169 case REJECTED:
2170 case REJECTED_RACED:
2171 case TERMINATED_DECLINED_OR_BUSY:
2172 if (isInitiator()) {
2173 return RtpEndUserState.DECLINED_OR_BUSY;
2174 } else {
2175 return RtpEndUserState.ENDED;
2176 }
2177 case TERMINATED_SUCCESS:
2178 case ACCEPTED:
2179 case RETRACTED:
2180 case TERMINATED_CANCEL_OR_TIMEOUT:
2181 return RtpEndUserState.ENDED;
2182 case RETRACTED_RACED:
2183 if (isInitiator()) {
2184 return RtpEndUserState.ENDED;
2185 } else {
2186 return RtpEndUserState.RETRACTED;
2187 }
2188 case TERMINATED_CONNECTIVITY_ERROR:
2189 return zeroDuration()
2190 ? RtpEndUserState.CONNECTIVITY_ERROR
2191 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2192 case TERMINATED_APPLICATION_FAILURE:
2193 return RtpEndUserState.APPLICATION_ERROR;
2194 case TERMINATED_SECURITY_ERROR:
2195 return RtpEndUserState.SECURITY_ERROR;
2196 }
2197 throw new IllegalStateException(
2198 String.format("%s has no equivalent EndUserState", this.state));
2199 }
2200
2201 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2202 final PeerConnection.PeerConnectionState state;
2203 try {
2204 state = webRTCWrapper.getState();
2205 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2206 // We usually close the WebRTCWrapper *before* transitioning so we might still
2207 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2208 return RtpEndUserState.ENDING_CALL;
2209 }
2210 switch (state) {
2211 case CONNECTED:
2212 return RtpEndUserState.CONNECTED;
2213 case NEW:
2214 case CONNECTING:
2215 return RtpEndUserState.CONNECTING;
2216 case CLOSED:
2217 return RtpEndUserState.ENDING_CALL;
2218 default:
2219 return zeroDuration()
2220 ? RtpEndUserState.CONNECTIVITY_ERROR
2221 : RtpEndUserState.RECONNECTING;
2222 }
2223 }
2224
2225 public ContentAddition getPendingContentAddition() {
2226 final RtpContentMap in = this.incomingContentAdd;
2227 final RtpContentMap out = this.outgoingContentAdd;
2228 if (out != null) {
2229 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2230 } else if (in != null) {
2231 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2232 } else {
2233 return null;
2234 }
2235 }
2236
2237 public Set<Media> getMedia() {
2238 final State current = getState();
2239 if (current == State.NULL) {
2240 if (isInitiator()) {
2241 return Preconditions.checkNotNull(
2242 this.proposedMedia, "RTP connection has not been initialized properly");
2243 }
2244 throw new IllegalStateException("RTP connection has not been initialized yet");
2245 }
2246 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2247 return Preconditions.checkNotNull(
2248 this.proposedMedia, "RTP connection has not been initialized properly");
2249 }
2250 final RtpContentMap localContentMap = getLocalContentMap();
2251 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2252 if (localContentMap != null) {
2253 return localContentMap.getMedia();
2254 } else if (initiatorContentMap != null) {
2255 return initiatorContentMap.getMedia();
2256 } else if (isTerminated()) {
2257 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2258 } else {
2259 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2260 }
2261 }
2262
2263 public boolean isVerified() {
2264 final String fingerprint = this.omemoVerification.getFingerprint();
2265 if (fingerprint == null) {
2266 return false;
2267 }
2268 final FingerprintStatus status =
2269 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2270 return status != null && status.isVerified();
2271 }
2272
2273 public boolean addMedia(final Media media) {
2274 final Set<Media> currentMedia = getMedia();
2275 if (currentMedia.contains(media)) {
2276 throw new IllegalStateException(String.format("%s has already been proposed", media));
2277 }
2278 // TODO add state protection - can only add while ACCEPTED or so
2279 Log.d(Config.LOGTAG,"adding media: "+media);
2280 return webRTCWrapper.addTrack(media);
2281 }
2282
2283 public synchronized void acceptCall() {
2284 switch (this.state) {
2285 case PROPOSED:
2286 cancelRingingTimeout();
2287 acceptCallFromProposed();
2288 break;
2289 case SESSION_INITIALIZED:
2290 cancelRingingTimeout();
2291 acceptCallFromSessionInitialized();
2292 break;
2293 case ACCEPTED:
2294 Log.w(
2295 Config.LOGTAG,
2296 id.account.getJid().asBareJid()
2297 + ": the call has already been accepted with another client. UI was just lagging behind");
2298 break;
2299 case PROCEED:
2300 case SESSION_ACCEPTED:
2301 Log.w(
2302 Config.LOGTAG,
2303 id.account.getJid().asBareJid()
2304 + ": the call has already been accepted. user probably double tapped the UI");
2305 break;
2306 default:
2307 throw new IllegalStateException("Can not accept call from " + this.state);
2308 }
2309 }
2310
2311 public void notifyPhoneCall() {
2312 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2313 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2314 rejectCall();
2315 } else {
2316 endCall();
2317 }
2318 }
2319
2320 public synchronized void rejectCall() {
2321 if (isTerminated()) {
2322 Log.w(
2323 Config.LOGTAG,
2324 id.account.getJid().asBareJid()
2325 + ": received rejectCall() when session has already been terminated. nothing to do");
2326 return;
2327 }
2328 switch (this.state) {
2329 case PROPOSED:
2330 rejectCallFromProposed();
2331 break;
2332 case SESSION_INITIALIZED:
2333 rejectCallFromSessionInitiate();
2334 break;
2335 default:
2336 throw new IllegalStateException("Can not reject call from " + this.state);
2337 }
2338 }
2339
2340 public synchronized void endCall() {
2341 if (isTerminated()) {
2342 Log.w(
2343 Config.LOGTAG,
2344 id.account.getJid().asBareJid()
2345 + ": received endCall() when session has already been terminated. nothing to do");
2346 return;
2347 }
2348 if (isInState(State.PROPOSED) && !isInitiator()) {
2349 rejectCallFromProposed();
2350 return;
2351 }
2352 if (isInState(State.PROCEED)) {
2353 if (isInitiator()) {
2354 retractFromProceed();
2355 } else {
2356 rejectCallFromProceed();
2357 }
2358 return;
2359 }
2360 if (isInitiator()
2361 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2362 this.webRTCWrapper.close();
2363 sendSessionTerminate(Reason.CANCEL);
2364 return;
2365 }
2366 if (isInState(State.SESSION_INITIALIZED)) {
2367 rejectCallFromSessionInitiate();
2368 return;
2369 }
2370 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2371 this.webRTCWrapper.close();
2372 sendSessionTerminate(Reason.SUCCESS);
2373 return;
2374 }
2375 if (isInState(
2376 State.TERMINATED_APPLICATION_FAILURE,
2377 State.TERMINATED_CONNECTIVITY_ERROR,
2378 State.TERMINATED_DECLINED_OR_BUSY)) {
2379 Log.d(
2380 Config.LOGTAG,
2381 "ignoring request to end call because already in state " + this.state);
2382 return;
2383 }
2384 throw new IllegalStateException(
2385 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2386 }
2387
2388 private void retractFromProceed() {
2389 Log.d(Config.LOGTAG, "retract from proceed");
2390 this.sendJingleMessage("retract");
2391 closeTransitionLogFinish(State.RETRACTED_RACED);
2392 }
2393
2394 private void closeTransitionLogFinish(final State state) {
2395 this.webRTCWrapper.close();
2396 transitionOrThrow(state);
2397 writeLogMessage(state);
2398 finish();
2399 }
2400
2401 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2402 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2403 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2404 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2405 }
2406
2407 private void acceptCallFromProposed() {
2408 transitionOrThrow(State.PROCEED);
2409 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2410 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2411 this.sendJingleMessage("proceed");
2412 }
2413
2414 private void rejectCallFromProposed() {
2415 transitionOrThrow(State.REJECTED);
2416 writeLogMessageMissed();
2417 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2418 this.sendJingleMessage("reject");
2419 finish();
2420 }
2421
2422 private void rejectCallFromProceed() {
2423 this.sendJingleMessage("reject");
2424 closeTransitionLogFinish(State.REJECTED_RACED);
2425 }
2426
2427 private void rejectCallFromSessionInitiate() {
2428 webRTCWrapper.close();
2429 sendSessionTerminate(Reason.DECLINE);
2430 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2431 }
2432
2433 private void sendJingleMessage(final String action) {
2434 sendJingleMessage(action, id.with);
2435 }
2436
2437 private void sendJingleMessage(final String action, final Jid to) {
2438 final MessagePacket messagePacket = new MessagePacket();
2439 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2440 messagePacket.setTo(to);
2441 final Element intent =
2442 messagePacket
2443 .addChild(action, Namespace.JINGLE_MESSAGE)
2444 .setAttribute("id", id.sessionId);
2445 if ("proceed".equals(action)) {
2446 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2447 if (isOmemoEnabled()) {
2448 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2449 final Element device =
2450 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2451 device.setAttribute("id", deviceId);
2452 }
2453 }
2454 messagePacket.addChild("store", "urn:xmpp:hints");
2455 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2456 }
2457
2458 private boolean isOmemoEnabled() {
2459 final Conversational conversational = message.getConversation();
2460 if (conversational instanceof Conversation) {
2461 return ((Conversation) conversational).getNextEncryption()
2462 == Message.ENCRYPTION_AXOLOTL;
2463 }
2464 return false;
2465 }
2466
2467 private void acceptCallFromSessionInitialized() {
2468 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2469 sendSessionAccept();
2470 }
2471
2472 private synchronized boolean isInState(State... state) {
2473 return Arrays.asList(state).contains(this.state);
2474 }
2475
2476 private boolean transition(final State target) {
2477 return transition(target, null);
2478 }
2479
2480 private synchronized boolean transition(final State target, final Runnable runnable) {
2481 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2482 if (validTransitions != null && validTransitions.contains(target)) {
2483 this.state = target;
2484 if (runnable != null) {
2485 runnable.run();
2486 }
2487 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2488 updateEndUserState();
2489 updateOngoingCallNotification();
2490 return true;
2491 } else {
2492 return false;
2493 }
2494 }
2495
2496 void transitionOrThrow(final State target) {
2497 if (!transition(target)) {
2498 throw new IllegalStateException(
2499 String.format("Unable to transition from %s to %s", this.state, target));
2500 }
2501 }
2502
2503 @Override
2504 public void onIceCandidate(final IceCandidate iceCandidate) {
2505 final RtpContentMap rtpContentMap =
2506 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2507 final IceUdpTransportInfo.Credentials credentials;
2508 try {
2509 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2510 } catch (final IllegalArgumentException e) {
2511 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2512 return;
2513 }
2514 final String uFrag = credentials.ufrag;
2515 final IceUdpTransportInfo.Candidate candidate =
2516 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2517 if (candidate == null) {
2518 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2519 return;
2520 }
2521 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2522 sendTransportInfo(iceCandidate.sdpMid, candidate);
2523 }
2524
2525 @Override
2526 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2527 Log.d(
2528 Config.LOGTAG,
2529 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2530 this.stateHistory.add(newState);
2531 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2532 this.sessionDuration.start();
2533 updateOngoingCallNotification();
2534 } else if (this.sessionDuration.isRunning()) {
2535 this.sessionDuration.stop();
2536 updateOngoingCallNotification();
2537 }
2538
2539 final boolean neverConnected =
2540 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2541
2542 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2543 if (neverConnected) {
2544 if (isTerminated()) {
2545 Log.d(
2546 Config.LOGTAG,
2547 id.account.getJid().asBareJid()
2548 + ": not sending session-terminate after connectivity error because session is already in state "
2549 + this.state);
2550 return;
2551 }
2552 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2553 return;
2554 } else {
2555 this.restartIce();
2556 }
2557 }
2558 updateEndUserState();
2559 }
2560
2561 private void restartIce() {
2562 this.stateHistory.clear();
2563 this.webRTCWrapper.restartIceAsync();
2564 }
2565
2566 @Override
2567 public void onRenegotiationNeeded() {
2568 this.webRTCWrapper.execute(this::renegotiate);
2569 }
2570
2571 private void renegotiate() {
2572 final SessionDescription sessionDescription;
2573 try {
2574 sessionDescription = setLocalSessionDescription();
2575 } catch (final Exception e) {
2576 final Throwable cause = Throwables.getRootCause(e);
2577 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2578 webRTCWrapper.close();
2579 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2580 return;
2581 }
2582 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2583 final RtpContentMap currentContentMap = getLocalContentMap();
2584 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2585 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2586
2587 Log.d(
2588 Config.LOGTAG,
2589 id.getAccount().getJid().asBareJid()
2590 + ": renegotiate. iceRestart="
2591 + iceRestart
2592 + " content id diff="
2593 + diff);
2594
2595 if (diff.hasModifications() && iceRestart) {
2596 webRTCWrapper.close();
2597 sendSessionTerminate(
2598 Reason.FAILED_APPLICATION,
2599 "WebRTC unexpectedly tried to modify content and transport at once");
2600 return;
2601 }
2602
2603 if (iceRestart) {
2604 initiateIceRestart(rtpContentMap);
2605 return;
2606 } else if (diff.isEmpty()) {
2607 Log.d(
2608 Config.LOGTAG,
2609 "renegotiation. nothing to do. SignalingState="
2610 + this.webRTCWrapper.getSignalingState());
2611 }
2612
2613 if (diff.added.size() > 0) {
2614 modifyLocalContentMap(rtpContentMap);
2615 sendContentAdd(rtpContentMap, diff.added);
2616 }
2617 }
2618
2619 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2620 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2621 final JinglePacket jinglePacket =
2622 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2623 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2624 jinglePacket.setTo(id.with);
2625 xmppConnectionService.sendIqPacket(
2626 id.account,
2627 jinglePacket,
2628 (account, response) -> {
2629 if (response.getType() == IqPacket.TYPE.RESULT) {
2630 Log.d(Config.LOGTAG, "received success to our ice restart");
2631 setLocalContentMap(rtpContentMap);
2632 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2633 return;
2634 }
2635 if (response.getType() == IqPacket.TYPE.ERROR) {
2636 if (isTieBreak(response)) {
2637 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2638 return;
2639 }
2640 handleIqErrorResponse(response);
2641 }
2642 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2643 handleIqTimeoutResponse(response);
2644 }
2645 });
2646 }
2647
2648 private boolean isTieBreak(final IqPacket response) {
2649 final Element error = response.findChild("error");
2650 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2651 }
2652
2653 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2654 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2655 this.outgoingContentAdd = contentAdd;
2656 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2657 prepareOutgoingContentMap(contentAdd);
2658 Futures.addCallback(
2659 outgoingContentMapFuture,
2660 new FutureCallback<RtpContentMap>() {
2661 @Override
2662 public void onSuccess(final RtpContentMap outgoingContentMap) {
2663 sendContentAdd(outgoingContentMap);
2664 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2665 }
2666
2667 @Override
2668 public void onFailure(@NonNull Throwable throwable) {
2669 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2670 }
2671 },
2672 MoreExecutors.directExecutor());
2673 }
2674
2675 private void sendContentAdd(final RtpContentMap contentAdd) {
2676
2677 final JinglePacket jinglePacket =
2678 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2679 jinglePacket.setTo(id.with);
2680 xmppConnectionService.sendIqPacket(
2681 id.account,
2682 jinglePacket,
2683 (connection, response) -> {
2684 if (response.getType() == IqPacket.TYPE.RESULT) {
2685 Log.d(
2686 Config.LOGTAG,
2687 id.getAccount().getJid().asBareJid()
2688 + ": received ACK to our content-add");
2689 return;
2690 }
2691 if (response.getType() == IqPacket.TYPE.ERROR) {
2692 if (isTieBreak(response)) {
2693 this.outgoingContentAdd = null;
2694 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2695 return;
2696 }
2697 handleIqErrorResponse(response);
2698 }
2699 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2700 handleIqTimeoutResponse(response);
2701 }
2702 });
2703 }
2704
2705 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2706 if (isInitiator()) {
2707 this.initiatorRtpContentMap = rtpContentMap;
2708 } else {
2709 this.responderRtpContentMap = rtpContentMap;
2710 }
2711 }
2712
2713 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2714 if (isInitiator()) {
2715 this.responderRtpContentMap = rtpContentMap;
2716 } else {
2717 this.initiatorRtpContentMap = rtpContentMap;
2718 }
2719 }
2720
2721 // this method is to be used for content map modifications that modify media
2722 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2723 final RtpContentMap activeContents = rtpContentMap.activeContents();
2724 setLocalContentMap(activeContents);
2725 this.webRTCWrapper.switchSpeakerPhonePreference(
2726 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2727 updateEndUserState();
2728 }
2729
2730 private SessionDescription setLocalSessionDescription()
2731 throws ExecutionException, InterruptedException {
2732 final org.webrtc.SessionDescription sessionDescription =
2733 this.webRTCWrapper.setLocalDescription().get();
2734 return SessionDescription.parse(sessionDescription.description);
2735 }
2736
2737 private void closeWebRTCSessionAfterFailedConnection() {
2738 this.webRTCWrapper.close();
2739 synchronized (this) {
2740 if (isTerminated()) {
2741 Log.d(
2742 Config.LOGTAG,
2743 id.account.getJid().asBareJid()
2744 + ": no need to send session-terminate after failed connection. Other party already did");
2745 return;
2746 }
2747 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2748 }
2749 }
2750
2751 public boolean zeroDuration() {
2752 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2753 }
2754
2755 public long getCallDuration() {
2756 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2757 }
2758
2759 public AppRTCAudioManager getAudioManager() {
2760 return webRTCWrapper.getAudioManager();
2761 }
2762
2763 public boolean isMicrophoneEnabled() {
2764 return webRTCWrapper.isMicrophoneEnabled();
2765 }
2766
2767 public boolean setMicrophoneEnabled(final boolean enabled) {
2768 return webRTCWrapper.setMicrophoneEnabled(enabled);
2769 }
2770
2771 public boolean isVideoEnabled() {
2772 return webRTCWrapper.isVideoEnabled();
2773 }
2774
2775 public void setVideoEnabled(final boolean enabled) {
2776 webRTCWrapper.setVideoEnabled(enabled);
2777 }
2778
2779 public boolean isCameraSwitchable() {
2780 return webRTCWrapper.isCameraSwitchable();
2781 }
2782
2783 public boolean isFrontCamera() {
2784 return webRTCWrapper.isFrontCamera();
2785 }
2786
2787 public ListenableFuture<Boolean> switchCamera() {
2788 return webRTCWrapper.switchCamera();
2789 }
2790
2791 @Override
2792 public void onAudioDeviceChanged(
2793 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2794 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2795 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2796 selectedAudioDevice, availableAudioDevices);
2797 }
2798
2799 private void updateEndUserState() {
2800 final RtpEndUserState endUserState = getEndUserState();
2801 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2802 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2803 id.account, id.with, id.sessionId, endUserState);
2804 }
2805
2806 private void updateOngoingCallNotification() {
2807 final State state = this.state;
2808 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2809 final boolean reconnecting;
2810 if (state == State.SESSION_ACCEPTED) {
2811 reconnecting =
2812 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2813 } else {
2814 reconnecting = false;
2815 }
2816 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2817 } else {
2818 xmppConnectionService.removeOngoingCall();
2819 }
2820 }
2821
2822 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2823 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2824 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2825 request.setTo(id.account.getDomain());
2826 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2827 xmppConnectionService.sendIqPacket(
2828 id.account,
2829 request,
2830 (account, response) -> {
2831 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2832 new ImmutableList.Builder<>();
2833 if (response.getType() == IqPacket.TYPE.RESULT) {
2834 final Element services =
2835 response.findChild(
2836 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2837 final List<Element> children =
2838 services == null
2839 ? Collections.emptyList()
2840 : services.getChildren();
2841 for (final Element child : children) {
2842 if ("service".equals(child.getName())) {
2843 final String type = child.getAttribute("type");
2844 final String host = child.getAttribute("host");
2845 final String sport = child.getAttribute("port");
2846 final Integer port =
2847 sport == null ? null : Ints.tryParse(sport);
2848 final String transport = child.getAttribute("transport");
2849 final String username = child.getAttribute("username");
2850 final String password = child.getAttribute("password");
2851 if (Strings.isNullOrEmpty(host) || port == null) {
2852 continue;
2853 }
2854 if (port < 0 || port > 65535) {
2855 continue;
2856 }
2857
2858
2859
2860
2861 if (Arrays.asList("stun", "stuns", "turn", "turns")
2862 .contains(type)
2863 && Arrays.asList("udp", "tcp").contains(transport)) {
2864 if (Arrays.asList("stuns", "turns").contains(type)
2865 && "udp".equals(transport)) {
2866 Log.d(
2867 Config.LOGTAG,
2868 id.account.getJid().asBareJid()
2869 + ": skipping invalid combination of udp/tls in external services");
2870 continue;
2871 }
2872
2873 // STUN URLs do not support a query section since M110
2874 final String uri;
2875 if (Arrays.asList("stun","stuns").contains(type)) {
2876 uri = String.format("%s:%s:%s", type, IP.wrapIPv6(host),port);
2877 } else {
2878 uri = String.format(
2879 "%s:%s:%s?transport=%s",
2880 type,
2881 IP.wrapIPv6(host),
2882 port,
2883 transport);
2884 }
2885
2886 final PeerConnection.IceServer.Builder iceServerBuilder =
2887 PeerConnection.IceServer.builder(uri);
2888 iceServerBuilder.setTlsCertPolicy(
2889 PeerConnection.TlsCertPolicy
2890 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2891 if (username != null && password != null) {
2892 iceServerBuilder.setUsername(username);
2893 iceServerBuilder.setPassword(password);
2894 } else if (Arrays.asList("turn", "turns").contains(type)) {
2895 // The WebRTC spec requires throwing an
2896 // InvalidAccessError when username (from libwebrtc
2897 // source coder)
2898 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2899 Log.d(
2900 Config.LOGTAG,
2901 id.account.getJid().asBareJid()
2902 + ": skipping "
2903 + type
2904 + "/"
2905 + transport
2906 + " without username and password");
2907 continue;
2908 }
2909 final PeerConnection.IceServer iceServer =
2910 iceServerBuilder.createIceServer();
2911 Log.d(
2912 Config.LOGTAG,
2913 id.account.getJid().asBareJid()
2914 + ": discovered ICE Server: "
2915 + iceServer);
2916 listBuilder.add(iceServer);
2917 }
2918 }
2919 }
2920 }
2921 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2922 if (iceServers.size() == 0) {
2923 Log.w(
2924 Config.LOGTAG,
2925 id.account.getJid().asBareJid()
2926 + ": no ICE server found "
2927 + response);
2928 }
2929 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2930 });
2931 } else {
2932 Log.w(
2933 Config.LOGTAG,
2934 id.account.getJid().asBareJid() + ": has no external service discovery");
2935 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2936 }
2937 }
2938
2939 private void finish() {
2940 if (isTerminated()) {
2941 this.cancelRingingTimeout();
2942 this.webRTCWrapper.verifyClosed();
2943 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2944 this.jingleConnectionManager.finishConnectionOrThrow(this);
2945 } else {
2946 throw new IllegalStateException(
2947 String.format("Unable to call finish from %s", this.state));
2948 }
2949 }
2950
2951 private void writeLogMessage(final State state) {
2952 final long duration = getCallDuration();
2953 if (state == State.TERMINATED_SUCCESS
2954 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2955 writeLogMessageSuccess(duration);
2956 } else {
2957 writeLogMessageMissed();
2958 }
2959 }
2960
2961 private void writeLogMessageSuccess(final long duration) {
2962 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2963 this.writeMessage();
2964 }
2965
2966 private void writeLogMessageMissed() {
2967 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2968 this.writeMessage();
2969 }
2970
2971 private void writeMessage() {
2972 final Conversational conversational = message.getConversation();
2973 if (conversational instanceof Conversation) {
2974 ((Conversation) conversational).add(this.message);
2975 xmppConnectionService.createMessageAsync(message);
2976 xmppConnectionService.updateConversationUi();
2977 } else {
2978 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2979 }
2980 }
2981
2982 public State getState() {
2983 return this.state;
2984 }
2985
2986 boolean isTerminated() {
2987 return TERMINATED.contains(this.state);
2988 }
2989
2990 public Optional<VideoTrack> getLocalVideoTrack() {
2991 return webRTCWrapper.getLocalVideoTrack();
2992 }
2993
2994 public Optional<VideoTrack> getRemoteVideoTrack() {
2995 return webRTCWrapper.getRemoteVideoTrack();
2996 }
2997
2998 public EglBase.Context getEglBaseContext() {
2999 return webRTCWrapper.getEglBaseContext();
3000 }
3001
3002 void setProposedMedia(final Set<Media> media) {
3003 this.proposedMedia = media;
3004 }
3005
3006 public void fireStateUpdate() {
3007 final RtpEndUserState endUserState = getEndUserState();
3008 xmppConnectionService.notifyJingleRtpConnectionUpdate(
3009 id.account, id.with, id.sessionId, endUserState);
3010 }
3011
3012 public boolean isSwitchToVideoAvailable() {
3013 final boolean prerequisite =
3014 Media.audioOnly(getMedia())
3015 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
3016 .contains(getEndUserState());
3017 return prerequisite && remoteHasVideoFeature();
3018 }
3019
3020 private boolean remoteHasVideoFeature() {
3021 final Contact contact = id.getContact();
3022 final Presence presence =
3023 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
3024 final ServiceDiscoveryResult serviceDiscoveryResult =
3025 presence == null ? null : presence.getServiceDiscoveryResult();
3026 final List<String> features =
3027 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
3028 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
3029 }
3030
3031 private interface OnIceServersDiscovered {
3032 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
3033 }
3034}