1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Joiner;
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.base.Stopwatch;
12import com.google.common.base.Strings;
13import com.google.common.base.Throwables;
14import com.google.common.collect.Collections2;
15import com.google.common.collect.ImmutableList;
16import com.google.common.collect.ImmutableMap;
17import com.google.common.collect.ImmutableSet;
18import com.google.common.collect.Maps;
19import com.google.common.collect.Sets;
20import com.google.common.primitives.Ints;
21import com.google.common.util.concurrent.FutureCallback;
22import com.google.common.util.concurrent.Futures;
23import com.google.common.util.concurrent.ListenableFuture;
24import com.google.common.util.concurrent.MoreExecutors;
25
26import org.webrtc.EglBase;
27import org.webrtc.IceCandidate;
28import org.webrtc.PeerConnection;
29import org.webrtc.VideoTrack;
30
31import java.util.Arrays;
32import java.util.Collection;
33import java.util.Collections;
34import java.util.LinkedList;
35import java.util.List;
36import java.util.Map;
37import java.util.Queue;
38import java.util.Set;
39import java.util.concurrent.ExecutionException;
40import java.util.concurrent.ScheduledFuture;
41import java.util.concurrent.TimeUnit;
42
43import eu.siacs.conversations.BuildConfig;
44import eu.siacs.conversations.Config;
45import eu.siacs.conversations.crypto.axolotl.AxolotlService;
46import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
47import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
48import eu.siacs.conversations.entities.Account;
49import eu.siacs.conversations.entities.Contact;
50import eu.siacs.conversations.entities.Conversation;
51import eu.siacs.conversations.entities.Conversational;
52import eu.siacs.conversations.entities.Message;
53import eu.siacs.conversations.entities.Presence;
54import eu.siacs.conversations.entities.RtpSessionStatus;
55import eu.siacs.conversations.entities.ServiceDiscoveryResult;
56import eu.siacs.conversations.services.AppRTCAudioManager;
57import eu.siacs.conversations.utils.IP;
58import eu.siacs.conversations.xml.Element;
59import eu.siacs.conversations.xml.Namespace;
60import eu.siacs.conversations.xmpp.Jid;
61import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
62import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
63import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
64import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
65import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
66import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
67import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
68import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
69import eu.siacs.conversations.xmpp.stanzas.IqPacket;
70import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
71
72public class JingleRtpConnection extends AbstractJingleConnection
73 implements WebRTCWrapper.EventCallback {
74
75 public static final List<State> STATES_SHOWING_ONGOING_CALL =
76 Arrays.asList(
77 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
78 private static final long BUSY_TIME_OUT = 30;
79 private static final List<State> TERMINATED =
80 Arrays.asList(
81 State.ACCEPTED,
82 State.REJECTED,
83 State.REJECTED_RACED,
84 State.RETRACTED,
85 State.RETRACTED_RACED,
86 State.TERMINATED_SUCCESS,
87 State.TERMINATED_DECLINED_OR_BUSY,
88 State.TERMINATED_CONNECTIVITY_ERROR,
89 State.TERMINATED_CANCEL_OR_TIMEOUT,
90 State.TERMINATED_APPLICATION_FAILURE,
91 State.TERMINATED_SECURITY_ERROR);
92
93 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
94
95 static {
96 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
97 new ImmutableMap.Builder<>();
98 transitionBuilder.put(
99 State.NULL,
100 ImmutableList.of(
101 State.PROPOSED,
102 State.SESSION_INITIALIZED,
103 State.TERMINATED_APPLICATION_FAILURE,
104 State.TERMINATED_SECURITY_ERROR));
105 transitionBuilder.put(
106 State.PROPOSED,
107 ImmutableList.of(
108 State.ACCEPTED,
109 State.PROCEED,
110 State.REJECTED,
111 State.RETRACTED,
112 State.TERMINATED_APPLICATION_FAILURE,
113 State.TERMINATED_SECURITY_ERROR,
114 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
115 // rebinds
116 ));
117 transitionBuilder.put(
118 State.PROCEED,
119 ImmutableList.of(
120 State.REJECTED_RACED,
121 State.RETRACTED_RACED,
122 State.SESSION_INITIALIZED_PRE_APPROVED,
123 State.TERMINATED_SUCCESS,
124 State.TERMINATED_APPLICATION_FAILURE,
125 State.TERMINATED_SECURITY_ERROR,
126 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
127 // bounces of the proceed message
128 ));
129 transitionBuilder.put(
130 State.SESSION_INITIALIZED,
131 ImmutableList.of(
132 State.SESSION_ACCEPTED,
133 State.TERMINATED_SUCCESS,
134 State.TERMINATED_DECLINED_OR_BUSY,
135 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
136 // and IQ timeouts
137 State.TERMINATED_CANCEL_OR_TIMEOUT,
138 State.TERMINATED_APPLICATION_FAILURE,
139 State.TERMINATED_SECURITY_ERROR));
140 transitionBuilder.put(
141 State.SESSION_INITIALIZED_PRE_APPROVED,
142 ImmutableList.of(
143 State.SESSION_ACCEPTED,
144 State.TERMINATED_SUCCESS,
145 State.TERMINATED_DECLINED_OR_BUSY,
146 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
147 // and IQ timeouts
148 State.TERMINATED_CANCEL_OR_TIMEOUT,
149 State.TERMINATED_APPLICATION_FAILURE,
150 State.TERMINATED_SECURITY_ERROR));
151 transitionBuilder.put(
152 State.SESSION_ACCEPTED,
153 ImmutableList.of(
154 State.TERMINATED_SUCCESS,
155 State.TERMINATED_DECLINED_OR_BUSY,
156 State.TERMINATED_CONNECTIVITY_ERROR,
157 State.TERMINATED_CANCEL_OR_TIMEOUT,
158 State.TERMINATED_APPLICATION_FAILURE,
159 State.TERMINATED_SECURITY_ERROR));
160 VALID_TRANSITIONS = transitionBuilder.build();
161 }
162
163 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
164 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
165 pendingIceCandidates = new LinkedList<>();
166 private final OmemoVerification omemoVerification = new OmemoVerification();
167 private final Message message;
168 private State state = State.NULL;
169 private Set<Media> proposedMedia;
170 private RtpContentMap initiatorRtpContentMap;
171 private RtpContentMap responderRtpContentMap;
172 private RtpContentMap incomingContentAdd;
173 private RtpContentMap outgoingContentAdd;
174 private IceUdpTransportInfo.Setup peerDtlsSetup;
175 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
176 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
177 private ScheduledFuture<?> ringingTimeoutFuture;
178
179 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
180 super(jingleConnectionManager, id, initiator);
181 final Conversation conversation =
182 jingleConnectionManager
183 .getXmppConnectionService()
184 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
185 this.message =
186 new Message(
187 conversation,
188 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
189 Message.TYPE_RTP_SESSION,
190 id.sessionId);
191 }
192
193 private static State reasonToState(Reason reason) {
194 switch (reason) {
195 case SUCCESS:
196 return State.TERMINATED_SUCCESS;
197 case DECLINE:
198 case BUSY:
199 return State.TERMINATED_DECLINED_OR_BUSY;
200 case CANCEL:
201 case TIMEOUT:
202 return State.TERMINATED_CANCEL_OR_TIMEOUT;
203 case SECURITY_ERROR:
204 return State.TERMINATED_SECURITY_ERROR;
205 case FAILED_APPLICATION:
206 case UNSUPPORTED_TRANSPORTS:
207 case UNSUPPORTED_APPLICATIONS:
208 return State.TERMINATED_APPLICATION_FAILURE;
209 default:
210 return State.TERMINATED_CONNECTIVITY_ERROR;
211 }
212 }
213
214 @Override
215 synchronized void deliverPacket(final JinglePacket jinglePacket) {
216 switch (jinglePacket.getAction()) {
217 case SESSION_INITIATE:
218 receiveSessionInitiate(jinglePacket);
219 break;
220 case TRANSPORT_INFO:
221 receiveTransportInfo(jinglePacket);
222 break;
223 case SESSION_ACCEPT:
224 receiveSessionAccept(jinglePacket);
225 break;
226 case SESSION_TERMINATE:
227 receiveSessionTerminate(jinglePacket);
228 break;
229 case CONTENT_ADD:
230 receiveContentAdd(jinglePacket);
231 break;
232 case CONTENT_ACCEPT:
233 receiveContentAccept(jinglePacket);
234 break;
235 case CONTENT_REJECT:
236 receiveContentReject(jinglePacket);
237 break;
238 case CONTENT_REMOVE:
239 receiveContentRemove(jinglePacket);
240 break;
241 case CONTENT_MODIFY:
242 receiveContentModify(jinglePacket);
243 break;
244 default:
245 respondOk(jinglePacket);
246 Log.d(
247 Config.LOGTAG,
248 String.format(
249 "%s: received unhandled jingle action %s",
250 id.account.getJid().asBareJid(), jinglePacket.getAction()));
251 break;
252 }
253 }
254
255 @Override
256 synchronized void notifyRebound() {
257 if (isTerminated()) {
258 return;
259 }
260 webRTCWrapper.close();
261 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
262 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
263 }
264 if (isInState(
265 State.SESSION_INITIALIZED,
266 State.SESSION_INITIALIZED_PRE_APPROVED,
267 State.SESSION_ACCEPTED)) {
268 // we might have already changed resources (full jid) at this point; so this might not
269 // even reach the other party
270 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
271 } else {
272 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
273 finish();
274 }
275 }
276
277 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
278 respondOk(jinglePacket);
279 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
280 final State previous = this.state;
281 Log.d(
282 Config.LOGTAG,
283 id.account.getJid().asBareJid()
284 + ": received session terminate reason="
285 + wrapper.reason
286 + "("
287 + Strings.nullToEmpty(wrapper.text)
288 + ") while in state "
289 + previous);
290 if (TERMINATED.contains(previous)) {
291 Log.d(
292 Config.LOGTAG,
293 id.account.getJid().asBareJid()
294 + ": ignoring session terminate because already in "
295 + previous);
296 return;
297 }
298 webRTCWrapper.close();
299 final State target = reasonToState(wrapper.reason);
300 transitionOrThrow(target);
301 writeLogMessage(target);
302 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
303 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
304 }
305 finish();
306 }
307
308 private void receiveTransportInfo(final JinglePacket jinglePacket) {
309 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
310 // INITIALIZED only after transport-info has been received
311 if (isInState(
312 State.NULL,
313 State.PROCEED,
314 State.SESSION_INITIALIZED,
315 State.SESSION_INITIALIZED_PRE_APPROVED,
316 State.SESSION_ACCEPTED)) {
317 final RtpContentMap contentMap;
318 try {
319 contentMap = RtpContentMap.of(jinglePacket);
320 } catch (final IllegalArgumentException | NullPointerException e) {
321 Log.d(
322 Config.LOGTAG,
323 id.account.getJid().asBareJid()
324 + ": improperly formatted contents; ignoring",
325 e);
326 respondOk(jinglePacket);
327 return;
328 }
329 receiveTransportInfo(jinglePacket, contentMap);
330 } else {
331 if (isTerminated()) {
332 respondOk(jinglePacket);
333 Log.d(
334 Config.LOGTAG,
335 id.account.getJid().asBareJid()
336 + ": ignoring out-of-order transport info; we where already terminated");
337 } else {
338 Log.d(
339 Config.LOGTAG,
340 id.account.getJid().asBareJid()
341 + ": received transport info while in state="
342 + this.state);
343 terminateWithOutOfOrder(jinglePacket);
344 }
345 }
346 }
347
348 private void receiveTransportInfo(
349 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
350 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
351 contentMap.contents.entrySet();
352 if (this.state == State.SESSION_ACCEPTED) {
353 // zero candidates + modified credentials are an ICE restart offer
354 if (checkForIceRestart(jinglePacket, contentMap)) {
355 return;
356 }
357 respondOk(jinglePacket);
358 try {
359 processCandidates(candidates);
360 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
361 Log.w(
362 Config.LOGTAG,
363 id.account.getJid().asBareJid()
364 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
365 }
366 } else {
367 respondOk(jinglePacket);
368 pendingIceCandidates.addAll(candidates);
369 }
370 }
371
372 private void receiveContentAdd(final JinglePacket jinglePacket) {
373 // TODO check if in session accepted
374 final RtpContentMap modification;
375 try {
376 modification = RtpContentMap.of(jinglePacket);
377 modification.requireContentDescriptions();
378 } catch (final RuntimeException e) {
379 Log.d(
380 Config.LOGTAG,
381 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
382 Throwables.getRootCause(e));
383 respondOk(jinglePacket);
384 webRTCWrapper.close();
385 sendSessionTerminate(Reason.of(e), e.getMessage());
386 return;
387 }
388 if (isInState(State.SESSION_ACCEPTED)) {
389 final boolean hasFullTransportInfo = modification.hasFullTransportInfo();
390 final ListenableFuture<RtpContentMap> future =
391 receiveRtpContentMap(
392 modification, this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
393 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
394 @Override
395 public void onSuccess(final RtpContentMap rtpContentMap) {
396 receiveContentAdd(jinglePacket, rtpContentMap);
397 }
398
399 @Override
400 public void onFailure(@NonNull Throwable throwable) {
401 respondOk(jinglePacket);
402 final Throwable rootCause = Throwables.getRootCause(throwable);
403 Log.d(
404 Config.LOGTAG,
405 id.account.getJid().asBareJid()
406 + ": improperly formatted contents in content-add",
407 throwable);
408 webRTCWrapper.close();
409 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
410 }
411 }, MoreExecutors.directExecutor());
412 } else {
413 terminateWithOutOfOrder(jinglePacket);
414 }
415 }
416
417 private void receiveContentAdd(
418 final JinglePacket jinglePacket, final RtpContentMap modification) {
419 final RtpContentMap remote = getRemoteContentMap();
420 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
421 respondOk(jinglePacket);
422 this.webRTCWrapper.close();
423 sendSessionTerminate(
424 Reason.FAILED_APPLICATION,
425 String.format(
426 "contents with names %s already exists",
427 Joiner.on(", ").join(modification.getNames())));
428 return;
429 }
430 final ContentAddition contentAddition =
431 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
432
433 final RtpContentMap outgoing = this.outgoingContentAdd;
434 final Set<ContentAddition.Summary> outgoingContentAddSummary =
435 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
436
437 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
438 if (isInitiator()) {
439 Log.d(
440 Config.LOGTAG,
441 id.getAccount().getJid().asBareJid()
442 + ": respond with tie break to matching content-add offer");
443 respondWithTieBreak(jinglePacket);
444 } else {
445 Log.d(
446 Config.LOGTAG,
447 id.getAccount().getJid().asBareJid()
448 + ": automatically accept matching content-add offer");
449 acceptContentAdd(contentAddition.summary, modification);
450 }
451 return;
452 }
453
454 // once we can display multiple video tracks we can be more loose with this condition
455 // theoretically it should also be fine to automatically accept audio only contents
456 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
457 Log.d(
458 Config.LOGTAG,
459 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
460 this.incomingContentAdd = modification;
461 respondOk(jinglePacket);
462 updateEndUserState();
463 } else {
464 respondOk(jinglePacket);
465 // TODO do we want to add a reason?
466 rejectContentAdd(modification);
467 }
468 }
469
470 private void receiveContentAccept(final JinglePacket jinglePacket) {
471 final RtpContentMap receivedContentAccept;
472 try {
473 receivedContentAccept = RtpContentMap.of(jinglePacket);
474 receivedContentAccept.requireContentDescriptions();
475 } catch (final RuntimeException e) {
476 Log.d(
477 Config.LOGTAG,
478 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
479 Throwables.getRootCause(e));
480 respondOk(jinglePacket);
481 webRTCWrapper.close();
482 sendSessionTerminate(Reason.of(e), e.getMessage());
483 return;
484 }
485
486 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
487 if (outgoingContentAdd == null) {
488 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
489 terminateWithOutOfOrder(jinglePacket);
490 return;
491 }
492 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
493 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
494 this.outgoingContentAdd = null;
495 respondOk(jinglePacket);
496 final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo();
497 final ListenableFuture<RtpContentMap> future =
498 receiveRtpContentMap(
499 receivedContentAccept, this.omemoVerification.hasFingerprint() && hasFullTransportInfo);
500 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
501 @Override
502 public void onSuccess(final RtpContentMap result) {
503 receiveContentAccept(result);
504 }
505
506 @Override
507 public void onFailure(@NonNull final Throwable throwable) {
508 webRTCWrapper.close();
509 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
510 }
511 }, MoreExecutors.directExecutor());
512 } else {
513 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
514 terminateWithOutOfOrder(jinglePacket);
515 }
516 }
517
518 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
519 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
520 final RtpContentMap modifiedContentMap =
521 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
522
523 setRemoteContentMap(modifiedContentMap);
524
525 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
526
527 final org.webrtc.SessionDescription sdp =
528 new org.webrtc.SessionDescription(
529 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
530
531 try {
532 this.webRTCWrapper.setRemoteDescription(sdp).get();
533 } catch (final Exception e) {
534 final Throwable cause = Throwables.getRootCause(e);
535 Log.d(
536 Config.LOGTAG,
537 id.getAccount().getJid().asBareJid()
538 + ": unable to set remote description after receiving content-accept",
539 cause);
540 webRTCWrapper.close();
541 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
542 return;
543 }
544 processCandidates(receivedContentAccept.contents.entrySet());
545 updateEndUserState();
546 Log.d(
547 Config.LOGTAG,
548 id.getAccount().getJid().asBareJid()
549 + ": remote has accepted content-add "
550 + ContentAddition.summary(receivedContentAccept));
551 }
552
553 private void receiveContentModify(final JinglePacket jinglePacket) {
554 final Map<String, Content.Senders> modification =
555 Maps.transformEntries(
556 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
557 respondOk(jinglePacket);
558 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
559 final Set<String> currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet();
560 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
561 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
562 final boolean isInitiator = isInitiator();
563 final RtpContentMap modifiedContentMap;
564 try {
565 modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification);
566 } catch (final IllegalArgumentException e) {
567 webRTCWrapper.close();
568 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
569 return;
570 }
571 this.outgoingContentAdd = modifiedContentMap;
572 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add");
573 } else {
574 webRTCWrapper.close();
575 sendSessionTerminate(
576 Reason.FAILED_APPLICATION,
577 String.format(
578 "%s only supports %s as a means to modify a not yet accepted %s",
579 BuildConfig.APP_NAME,
580 JinglePacket.Action.CONTENT_MODIFY,
581 JinglePacket.Action.CONTENT_ADD));
582 }
583 }
584
585 private void receiveContentReject(final JinglePacket jinglePacket) {
586 final RtpContentMap receivedContentReject;
587 try {
588 receivedContentReject = RtpContentMap.of(jinglePacket);
589 } catch (final RuntimeException e) {
590 Log.d(
591 Config.LOGTAG,
592 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
593 Throwables.getRootCause(e));
594 respondOk(jinglePacket);
595 this.webRTCWrapper.close();
596 sendSessionTerminate(Reason.of(e), e.getMessage());
597 return;
598 }
599
600 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
601 if (outgoingContentAdd == null) {
602 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
603 terminateWithOutOfOrder(jinglePacket);
604 return;
605 }
606 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
607 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
608 this.outgoingContentAdd = null;
609 respondOk(jinglePacket);
610 Log.d(Config.LOGTAG,jinglePacket.toString());
611 receiveContentReject(ourSummary);
612 } else {
613 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
614 terminateWithOutOfOrder(jinglePacket);
615 }
616 }
617
618 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
619 try {
620 this.webRTCWrapper.removeTrack(Media.VIDEO);
621 final RtpContentMap localContentMap = customRollback();
622 modifyLocalContentMap(localContentMap);
623 } catch (final Exception e) {
624 final Throwable cause = Throwables.getRootCause(e);
625 Log.d(
626 Config.LOGTAG,
627 id.getAccount().getJid().asBareJid()
628 + ": unable to rollback local description after receiving content-reject",
629 cause);
630 webRTCWrapper.close();
631 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
632 return;
633 }
634 Log.d(
635 Config.LOGTAG,
636 id.getAccount().getJid().asBareJid()
637 + ": remote has rejected our content-add "
638 + summary);
639 }
640
641 private void receiveContentRemove(final JinglePacket jinglePacket) {
642 final RtpContentMap receivedContentRemove;
643 try {
644 receivedContentRemove = RtpContentMap.of(jinglePacket);
645 receivedContentRemove.requireContentDescriptions();
646 } catch (final RuntimeException e) {
647 Log.d(
648 Config.LOGTAG,
649 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
650 Throwables.getRootCause(e));
651 respondOk(jinglePacket);
652 this.webRTCWrapper.close();
653 sendSessionTerminate(Reason.of(e), e.getMessage());
654 return;
655 }
656 respondOk(jinglePacket);
657 receiveContentRemove(receivedContentRemove);
658 }
659
660 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
661 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
662 final Set<ContentAddition.Summary> contentAddSummary =
663 incomingContentAdd == null
664 ? Collections.emptySet()
665 : ContentAddition.summary(incomingContentAdd);
666 final Set<ContentAddition.Summary> removeSummary =
667 ContentAddition.summary(receivedContentRemove);
668 if (contentAddSummary.equals(removeSummary)) {
669 this.incomingContentAdd = null;
670 updateEndUserState();
671 } else {
672 webRTCWrapper.close();
673 sendSessionTerminate(
674 Reason.FAILED_APPLICATION,
675 String.format(
676 "%s only supports %s as a means to retract a not yet accepted %s",
677 BuildConfig.APP_NAME,
678 JinglePacket.Action.CONTENT_REMOVE,
679 JinglePacket.Action.CONTENT_ADD));
680 }
681 }
682
683 public synchronized void retractContentAdd() {
684 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
685 if (outgoingContentAdd == null) {
686 throw new IllegalStateException("Not outgoing content add");
687 }
688 try {
689 webRTCWrapper.removeTrack(Media.VIDEO);
690 final RtpContentMap localContentMap = customRollback();
691 modifyLocalContentMap(localContentMap);
692 } catch (final Exception e) {
693 final Throwable cause = Throwables.getRootCause(e);
694 Log.d(
695 Config.LOGTAG,
696 id.getAccount().getJid().asBareJid()
697 + ": unable to rollback local description after trying to retract content-add",
698 cause);
699 webRTCWrapper.close();
700 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
701 return;
702 }
703 this.outgoingContentAdd = null;
704 final JinglePacket retract =
705 outgoingContentAdd
706 .toStub()
707 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
708 this.send(retract);
709 Log.d(
710 Config.LOGTAG,
711 id.getAccount().getJid()
712 + ": retract content-add "
713 + ContentAddition.summary(outgoingContentAdd));
714 }
715
716 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
717 final SessionDescription sdp = setLocalSessionDescription();
718 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
719 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
720 this.webRTCWrapper
721 .setRemoteDescription(
722 new org.webrtc.SessionDescription(
723 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
724 .get();
725 return localRtpContentMap;
726 }
727
728 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
729 final RtpContentMap currentRemote = getRemoteContentMap();
730 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
731 if (diff.isEmpty()) {
732 throw new IllegalStateException(
733 "Unexpected rollback condition. No difference between local and remote");
734 }
735 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
736 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
737 final RtpContentMap nextRemote =
738 currentRemote.addContent(
739 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
740 return SessionDescription.of(nextRemote, !isInitiator());
741 }
742 throw new IllegalStateException(
743 "Unexpected rollback condition. Senders were not uniformly none");
744 }
745
746 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
747 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
748 if (incomingContentAdd == null) {
749 throw new IllegalStateException("No incoming content add");
750 }
751
752 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
753 this.incomingContentAdd = null;
754 acceptContentAdd(contentAddition, incomingContentAdd);
755 } else {
756 throw new IllegalStateException("Accepted content add does not match pending content-add");
757 }
758 }
759
760 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
761 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
762 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
763 this.setRemoteContentMap(modifiedContentMap);
764
765 final SessionDescription offer;
766 try {
767 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
768 } catch (final IllegalArgumentException | NullPointerException e) {
769 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
770 webRTCWrapper.close();
771 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
772 return;
773 }
774 this.incomingContentAdd = null;
775 acceptContentAdd(contentAddition, offer);
776 }
777
778 private void acceptContentAdd(
779 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
780 final org.webrtc.SessionDescription sdp =
781 new org.webrtc.SessionDescription(
782 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
783 try {
784 this.webRTCWrapper.setRemoteDescription(sdp).get();
785
786 // TODO add tracks for 'media' where contentAddition.senders matches
787
788 // TODO if senders.sending(isInitiator())
789
790 this.webRTCWrapper.addTrack(Media.VIDEO);
791
792 // TODO add additional transceivers for recv only cases
793
794 final SessionDescription answer = setLocalSessionDescription();
795 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
796
797 final RtpContentMap contentAcceptMap =
798 rtpContentMap.toContentModification(
799 Collections2.transform(contentAddition, ca -> ca.name));
800
801 Log.d(
802 Config.LOGTAG,
803 id.getAccount().getJid().asBareJid()
804 + ": sending content-accept "
805 + ContentAddition.summary(contentAcceptMap));
806 modifyLocalContentMap(rtpContentMap);
807 final ListenableFuture<RtpContentMap> future = prepareOutgoingContentMap(contentAcceptMap);
808 Futures.addCallback(
809 future,
810 new FutureCallback<RtpContentMap>() {
811 @Override
812 public void onSuccess(final RtpContentMap rtpContentMap) {
813 sendContentAccept(rtpContentMap);
814 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
815 }
816
817 @Override
818 public void onFailure(@NonNull final Throwable throwable) {
819 failureToPerformAction(JinglePacket.Action.CONTENT_ACCEPT, throwable);
820 }
821 },
822 MoreExecutors.directExecutor());
823 } catch (final Exception e) {
824 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
825 webRTCWrapper.close();
826 sendSessionTerminate(Reason.FAILED_APPLICATION);
827 }
828 }
829
830 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
831 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
832 send(jinglePacket);
833 }
834
835 public synchronized void rejectContentAdd() {
836 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
837 if (incomingContentAdd == null) {
838 throw new IllegalStateException("No incoming content add");
839 }
840 this.incomingContentAdd = null;
841 updateEndUserState();
842 rejectContentAdd(incomingContentAdd);
843 }
844
845 private void rejectContentAdd(final RtpContentMap contentMap) {
846 final JinglePacket jinglePacket =
847 contentMap
848 .toStub()
849 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
850 Log.d(
851 Config.LOGTAG,
852 id.getAccount().getJid().asBareJid()
853 + ": rejecting content "
854 + ContentAddition.summary(contentMap));
855 send(jinglePacket);
856 }
857
858 private boolean checkForIceRestart(
859 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
860 final RtpContentMap existing = getRemoteContentMap();
861 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
862 final IceUdpTransportInfo.Credentials newCredentials;
863 try {
864 existingCredentials = existing.getCredentials();
865 newCredentials = rtpContentMap.getDistinctCredentials();
866 } catch (final IllegalStateException e) {
867 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
868 return false;
869 }
870 if (existingCredentials.contains(newCredentials)) {
871 return false;
872 }
873 // TODO an alternative approach is to check if we already got an iq result to our
874 // ICE-restart
875 // and if that's the case we are seeing an answer.
876 // This might be more spec compliant but also more error prone potentially
877 final boolean isSignalStateStable = this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
878 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
879 final boolean isOffer = rtpContentMap.emptyCandidates();
880 final RtpContentMap restartContentMap;
881 try {
882 if (isOffer) {
883 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
884 restartContentMap =
885 existing.modifiedCredentials(
886 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
887 } else {
888 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
889 Log.d(
890 Config.LOGTAG,
891 "received confirmation of ICE restart"
892 + newCredentials
893 + " peer_setup="
894 + setup);
895 // DTLS setup attribute needs to be rewritten to reflect current peer state
896 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
897 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
898 }
899 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
900 return isOffer;
901 } else {
902 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
903 respondWithTieBreak(jinglePacket);
904 return true;
905 }
906 } catch (final Exception exception) {
907 respondOk(jinglePacket);
908 final Throwable rootCause = Throwables.getRootCause(exception);
909 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
910 // If this happens a termination is already in progress
911 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
912 return true;
913 }
914 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
915 webRTCWrapper.close();
916 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
917 return true;
918 }
919 }
920
921 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
922 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
923 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
924 throw new IllegalStateException("Invalid peer setup");
925 }
926 return peerSetup;
927 }
928
929 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
930 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
931 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
932 }
933 this.peerDtlsSetup = setup;
934 }
935
936 private boolean applyIceRestart(
937 final JinglePacket jinglePacket,
938 final RtpContentMap restartContentMap,
939 final boolean isOffer)
940 throws ExecutionException, InterruptedException {
941 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
942 final org.webrtc.SessionDescription.Type type =
943 isOffer
944 ? org.webrtc.SessionDescription.Type.OFFER
945 : org.webrtc.SessionDescription.Type.ANSWER;
946 org.webrtc.SessionDescription sdp =
947 new org.webrtc.SessionDescription(type, sessionDescription.toString());
948 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
949 if (isInitiator()) {
950 // We ignore the offer and respond with tie-break. This will clause the responder
951 // not to apply the content map
952 return false;
953 }
954 }
955 webRTCWrapper.setRemoteDescription(sdp).get();
956 setRemoteContentMap(restartContentMap);
957 if (isOffer) {
958 final SessionDescription localSessionDescription = setLocalSessionDescription();
959 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
960 // We need to respond OK before sending any candidates
961 respondOk(jinglePacket);
962 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
963 } else {
964 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
965 }
966 return true;
967 }
968
969 private void processCandidates(
970 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
971 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
972 processCandidate(content);
973 }
974 }
975
976 private void processCandidate(
977 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
978 final RtpContentMap rtpContentMap = getRemoteContentMap();
979 final List<String> indices = toIdentificationTags(rtpContentMap);
980 final String sdpMid = content.getKey(); // aka content name
981 final IceUdpTransportInfo transport = content.getValue().transport;
982 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
983
984 // TODO check that credentials remained the same
985
986 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
987 final String sdp;
988 try {
989 sdp = candidate.toSdpAttribute(credentials.ufrag);
990 } catch (final IllegalArgumentException e) {
991 Log.d(
992 Config.LOGTAG,
993 id.account.getJid().asBareJid()
994 + ": ignoring invalid ICE candidate "
995 + e.getMessage());
996 continue;
997 }
998 final int mLineIndex = indices.indexOf(sdpMid);
999 if (mLineIndex < 0) {
1000 Log.w(
1001 Config.LOGTAG,
1002 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1003 }
1004 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1005 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1006 this.webRTCWrapper.addIceCandidate(iceCandidate);
1007 }
1008 }
1009
1010 private RtpContentMap getRemoteContentMap() {
1011 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1012 }
1013
1014 private RtpContentMap getLocalContentMap() {
1015 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1016 }
1017
1018 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1019 final Group originalGroup = rtpContentMap.group;
1020 final List<String> identificationTags =
1021 originalGroup == null
1022 ? rtpContentMap.getNames()
1023 : originalGroup.getIdentificationTags();
1024 if (identificationTags.size() == 0) {
1025 Log.w(
1026 Config.LOGTAG,
1027 id.account.getJid().asBareJid()
1028 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1029 }
1030 return identificationTags;
1031 }
1032
1033 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1034 final JinglePacket jinglePacket, final boolean expectVerification) {
1035 try {
1036 return receiveRtpContentMap(RtpContentMap.of(jinglePacket), expectVerification);
1037 } catch (final Exception e) {
1038 return Futures.immediateFailedFuture(e);
1039 }
1040 }
1041 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final RtpContentMap receivedContentMap, final boolean expectVerification) {
1042 Log.d(
1043 Config.LOGTAG,
1044 "receiveRtpContentMap("
1045 + receivedContentMap.getClass().getSimpleName()
1046 + ",expectVerification="
1047 + expectVerification
1048 + ")");
1049 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1050 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1051 id.account
1052 .getAxolotlService()
1053 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1054 return Futures.transform(
1055 future,
1056 omemoVerifiedPayload -> {
1057 // TODO test if an exception here triggers a correct abort
1058 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1059 Log.d(
1060 Config.LOGTAG,
1061 id.account.getJid().asBareJid()
1062 + ": received verifiable DTLS fingerprint via "
1063 + omemoVerification);
1064 return omemoVerifiedPayload.getPayload();
1065 },
1066 MoreExecutors.directExecutor());
1067 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1068 return Futures.immediateFailedFuture(
1069 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1070 } else {
1071 return Futures.immediateFuture(receivedContentMap);
1072 }
1073 }
1074
1075 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1076 if (isInitiator()) {
1077 Log.d(
1078 Config.LOGTAG,
1079 String.format(
1080 "%s: received session-initiate even though we were initiating",
1081 id.account.getJid().asBareJid()));
1082 if (isTerminated()) {
1083 Log.d(
1084 Config.LOGTAG,
1085 String.format(
1086 "%s: got a reason to terminate with out-of-order. but already in state %s",
1087 id.account.getJid().asBareJid(), getState()));
1088 respondWithOutOfOrder(jinglePacket);
1089 } else {
1090 terminateWithOutOfOrder(jinglePacket);
1091 }
1092 return;
1093 }
1094 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1095 Futures.addCallback(
1096 future,
1097 new FutureCallback<RtpContentMap>() {
1098 @Override
1099 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1100 receiveSessionInitiate(jinglePacket, rtpContentMap);
1101 }
1102
1103 @Override
1104 public void onFailure(@NonNull final Throwable throwable) {
1105 respondOk(jinglePacket);
1106 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1107 }
1108 },
1109 MoreExecutors.directExecutor());
1110 }
1111
1112 private void receiveSessionInitiate(
1113 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1114 try {
1115 contentMap.requireContentDescriptions();
1116 contentMap.requireDTLSFingerprint(true);
1117 } catch (final RuntimeException e) {
1118 Log.d(
1119 Config.LOGTAG,
1120 id.account.getJid().asBareJid() + ": improperly formatted contents",
1121 Throwables.getRootCause(e));
1122 respondOk(jinglePacket);
1123 sendSessionTerminate(Reason.of(e), e.getMessage());
1124 return;
1125 }
1126 Log.d(
1127 Config.LOGTAG,
1128 "processing session-init with " + contentMap.contents.size() + " contents");
1129 final State target;
1130 if (this.state == State.PROCEED) {
1131 Preconditions.checkState(
1132 proposedMedia != null && proposedMedia.size() > 0,
1133 "proposed media must be set when processing pre-approved session-initiate");
1134 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1135 sendSessionTerminate(
1136 Reason.SECURITY_ERROR,
1137 String.format(
1138 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1139 this.proposedMedia, contentMap.getMedia()));
1140 return;
1141 }
1142 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1143 } else {
1144 target = State.SESSION_INITIALIZED;
1145 }
1146 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1147 respondOk(jinglePacket);
1148 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1149 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1150 Log.d(
1151 Config.LOGTAG,
1152 id.account.getJid().asBareJid()
1153 + ": automatically accepting session-initiate");
1154 sendSessionAccept();
1155 } else {
1156 Log.d(
1157 Config.LOGTAG,
1158 id.account.getJid().asBareJid()
1159 + ": received not pre-approved session-initiate. start ringing");
1160 startRinging();
1161 }
1162 } else {
1163 Log.d(
1164 Config.LOGTAG,
1165 String.format(
1166 "%s: received session-initiate while in state %s",
1167 id.account.getJid().asBareJid(), state));
1168 terminateWithOutOfOrder(jinglePacket);
1169 }
1170 }
1171
1172 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1173 if (!isInitiator()) {
1174 Log.d(
1175 Config.LOGTAG,
1176 String.format(
1177 "%s: received session-accept even though we were responding",
1178 id.account.getJid().asBareJid()));
1179 terminateWithOutOfOrder(jinglePacket);
1180 return;
1181 }
1182 final ListenableFuture<RtpContentMap> future =
1183 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1184 Futures.addCallback(
1185 future,
1186 new FutureCallback<RtpContentMap>() {
1187 @Override
1188 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1189 receiveSessionAccept(jinglePacket, rtpContentMap);
1190 }
1191
1192 @Override
1193 public void onFailure(@NonNull final Throwable throwable) {
1194 respondOk(jinglePacket);
1195 Log.d(
1196 Config.LOGTAG,
1197 id.account.getJid().asBareJid()
1198 + ": improperly formatted contents in session-accept",
1199 throwable);
1200 webRTCWrapper.close();
1201 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1202 }
1203 },
1204 MoreExecutors.directExecutor());
1205 }
1206
1207 private void receiveSessionAccept(
1208 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1209 try {
1210 contentMap.requireContentDescriptions();
1211 contentMap.requireDTLSFingerprint();
1212 } catch (final RuntimeException e) {
1213 respondOk(jinglePacket);
1214 Log.d(
1215 Config.LOGTAG,
1216 id.account.getJid().asBareJid()
1217 + ": improperly formatted contents in session-accept",
1218 e);
1219 webRTCWrapper.close();
1220 sendSessionTerminate(Reason.of(e), e.getMessage());
1221 return;
1222 }
1223 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1224 if (!initiatorMedia.equals(contentMap.getMedia())) {
1225 sendSessionTerminate(
1226 Reason.SECURITY_ERROR,
1227 String.format(
1228 "Your session-included included media %s but our session-initiate was %s",
1229 this.proposedMedia, contentMap.getMedia()));
1230 return;
1231 }
1232 Log.d(
1233 Config.LOGTAG,
1234 "processing session-accept with " + contentMap.contents.size() + " contents");
1235 if (transition(State.SESSION_ACCEPTED)) {
1236 respondOk(jinglePacket);
1237 receiveSessionAccept(contentMap);
1238 } else {
1239 Log.d(
1240 Config.LOGTAG,
1241 String.format(
1242 "%s: received session-accept while in state %s",
1243 id.account.getJid().asBareJid(), state));
1244 respondOk(jinglePacket);
1245 }
1246 }
1247
1248 private void receiveSessionAccept(final RtpContentMap contentMap) {
1249 this.responderRtpContentMap = contentMap;
1250 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1251 final SessionDescription sessionDescription;
1252 try {
1253 sessionDescription = SessionDescription.of(contentMap, false);
1254 } catch (final IllegalArgumentException | NullPointerException e) {
1255 Log.d(
1256 Config.LOGTAG,
1257 id.account.getJid().asBareJid()
1258 + ": unable convert offer from session-accept to SDP",
1259 e);
1260 webRTCWrapper.close();
1261 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1262 return;
1263 }
1264 final org.webrtc.SessionDescription answer =
1265 new org.webrtc.SessionDescription(
1266 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1267 try {
1268 this.webRTCWrapper.setRemoteDescription(answer).get();
1269 } catch (final Exception e) {
1270 Log.d(
1271 Config.LOGTAG,
1272 id.account.getJid().asBareJid()
1273 + ": unable to set remote description after receiving session-accept",
1274 Throwables.getRootCause(e));
1275 webRTCWrapper.close();
1276 sendSessionTerminate(
1277 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1278 return;
1279 }
1280 processCandidates(contentMap.contents.entrySet());
1281 }
1282
1283 private void sendSessionAccept() {
1284 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1285 if (rtpContentMap == null) {
1286 throw new IllegalStateException("initiator RTP Content Map has not been set");
1287 }
1288 final SessionDescription offer;
1289 try {
1290 offer = SessionDescription.of(rtpContentMap, true);
1291 } catch (final IllegalArgumentException | NullPointerException e) {
1292 Log.d(
1293 Config.LOGTAG,
1294 id.account.getJid().asBareJid()
1295 + ": unable convert offer from session-initiate to SDP",
1296 e);
1297 webRTCWrapper.close();
1298 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1299 return;
1300 }
1301 sendSessionAccept(rtpContentMap.getMedia(), offer);
1302 }
1303
1304 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1305 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1306 }
1307
1308 private synchronized void sendSessionAccept(
1309 final Set<Media> media,
1310 final SessionDescription offer,
1311 final List<PeerConnection.IceServer> iceServers) {
1312 if (isTerminated()) {
1313 Log.w(
1314 Config.LOGTAG,
1315 id.account.getJid().asBareJid()
1316 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1317 return;
1318 }
1319 try {
1320 setupWebRTC(media, iceServers);
1321 } catch (final WebRTCWrapper.InitializationException e) {
1322 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1323 webRTCWrapper.close();
1324 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1325 return;
1326 }
1327 final org.webrtc.SessionDescription sdp =
1328 new org.webrtc.SessionDescription(
1329 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1330 try {
1331 this.webRTCWrapper.setRemoteDescription(sdp).get();
1332 addIceCandidatesFromBlackLog();
1333 org.webrtc.SessionDescription webRTCSessionDescription =
1334 this.webRTCWrapper.setLocalDescription().get();
1335 prepareSessionAccept(webRTCSessionDescription);
1336 } catch (final Exception e) {
1337 failureToAcceptSession(e);
1338 }
1339 }
1340
1341 private void failureToAcceptSession(final Throwable throwable) {
1342 if (isTerminated()) {
1343 return;
1344 }
1345 final Throwable rootCause = Throwables.getRootCause(throwable);
1346 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1347 webRTCWrapper.close();
1348 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1349 }
1350
1351 private void failureToPerformAction(final JinglePacket.Action action, final Throwable throwable) {
1352 if (isTerminated()) {
1353 return;
1354 }
1355 final Throwable rootCause = Throwables.getRootCause(throwable);
1356 Log.d(Config.LOGTAG, "unable to send " + action, rootCause);
1357 webRTCWrapper.close();
1358 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1359 }
1360
1361 private void addIceCandidatesFromBlackLog() {
1362 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1363 while ((foo = this.pendingIceCandidates.poll()) != null) {
1364 processCandidate(foo);
1365 Log.d(
1366 Config.LOGTAG,
1367 id.account.getJid().asBareJid() + ": added candidate from back log");
1368 }
1369 }
1370
1371 private void prepareSessionAccept(
1372 final org.webrtc.SessionDescription webRTCSessionDescription) {
1373 final SessionDescription sessionDescription =
1374 SessionDescription.parse(webRTCSessionDescription.description);
1375 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1376 this.responderRtpContentMap = respondingRtpContentMap;
1377 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1378 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1379 prepareOutgoingContentMap(respondingRtpContentMap);
1380 Futures.addCallback(
1381 outgoingContentMapFuture,
1382 new FutureCallback<RtpContentMap>() {
1383 @Override
1384 public void onSuccess(final RtpContentMap outgoingContentMap) {
1385 sendSessionAccept(outgoingContentMap);
1386 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1387 }
1388
1389 @Override
1390 public void onFailure(@NonNull Throwable throwable) {
1391 failureToAcceptSession(throwable);
1392 }
1393 },
1394 MoreExecutors.directExecutor());
1395 }
1396
1397 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1398 if (isTerminated()) {
1399 Log.w(
1400 Config.LOGTAG,
1401 id.account.getJid().asBareJid()
1402 + ": preparing session accept was too slow. already terminated. nothing to do.");
1403 return;
1404 }
1405 transitionOrThrow(State.SESSION_ACCEPTED);
1406 final JinglePacket sessionAccept =
1407 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1408 send(sessionAccept);
1409 }
1410
1411 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1412 final RtpContentMap rtpContentMap) {
1413 if (this.omemoVerification.hasDeviceId()) {
1414 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1415 verifiedPayloadFuture =
1416 id.account
1417 .getAxolotlService()
1418 .encrypt(
1419 rtpContentMap,
1420 id.with,
1421 omemoVerification.getDeviceId());
1422 return Futures.transform(
1423 verifiedPayloadFuture,
1424 verifiedPayload -> {
1425 omemoVerification.setOrEnsureEqual(verifiedPayload);
1426 return verifiedPayload.getPayload();
1427 },
1428 MoreExecutors.directExecutor());
1429 } else {
1430 return Futures.immediateFuture(rtpContentMap);
1431 }
1432 }
1433
1434 synchronized void deliveryMessage(
1435 final Jid from,
1436 final Element message,
1437 final String serverMessageId,
1438 final long timestamp) {
1439 Log.d(
1440 Config.LOGTAG,
1441 id.account.getJid().asBareJid()
1442 + ": delivered message to JingleRtpConnection "
1443 + message);
1444 switch (message.getName()) {
1445 case "propose":
1446 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1447 break;
1448 case "proceed":
1449 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1450 break;
1451 case "retract":
1452 receiveRetract(from, serverMessageId, timestamp);
1453 break;
1454 case "reject":
1455 receiveReject(from, serverMessageId, timestamp);
1456 break;
1457 case "accept":
1458 receiveAccept(from, serverMessageId, timestamp);
1459 break;
1460 default:
1461 break;
1462 }
1463 }
1464
1465 void deliverFailedProceed(final String message) {
1466 Log.d(
1467 Config.LOGTAG,
1468 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1469 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1470 webRTCWrapper.close();
1471 Log.d(
1472 Config.LOGTAG,
1473 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1474 this.finish();
1475 }
1476 }
1477
1478 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1479 final boolean originatedFromMyself =
1480 from.asBareJid().equals(id.account.getJid().asBareJid());
1481 if (originatedFromMyself) {
1482 if (transition(State.ACCEPTED)) {
1483 acceptedOnOtherDevice(serverMsgId, timestamp);
1484 } else {
1485 Log.d(
1486 Config.LOGTAG,
1487 id.account.getJid().asBareJid()
1488 + ": unable to transition to accept because already in state="
1489 + this.state);
1490 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1491 }
1492 } else {
1493 Log.d(
1494 Config.LOGTAG,
1495 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1496 }
1497 }
1498
1499 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1500 if (serverMsgId != null) {
1501 this.message.setServerMsgId(serverMsgId);
1502 }
1503 this.message.setTime(timestamp);
1504 this.message.setCarbon(true); // indicate that call was accepted on other device
1505 this.writeLogMessageSuccess(0);
1506 this.xmppConnectionService
1507 .getNotificationService()
1508 .cancelIncomingCallNotification();
1509 this.finish();
1510 }
1511
1512 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1513 final boolean originatedFromMyself =
1514 from.asBareJid().equals(id.account.getJid().asBareJid());
1515 // reject from another one of my clients
1516 if (originatedFromMyself) {
1517 receiveRejectFromMyself(serverMsgId, timestamp);
1518 } else if (isInitiator()) {
1519 if (from.equals(id.with)) {
1520 receiveRejectFromResponder();
1521 } else {
1522 Log.d(
1523 Config.LOGTAG,
1524 id.account.getJid()
1525 + ": ignoring reject from "
1526 + from
1527 + " for session with "
1528 + id.with);
1529 }
1530 } else {
1531 Log.d(
1532 Config.LOGTAG,
1533 id.account.getJid()
1534 + ": ignoring reject from "
1535 + from
1536 + " for session with "
1537 + id.with);
1538 }
1539 }
1540
1541 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1542 if (transition(State.REJECTED)) {
1543 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1544 this.finish();
1545 if (serverMsgId != null) {
1546 this.message.setServerMsgId(serverMsgId);
1547 }
1548 this.message.setTime(timestamp);
1549 this.message.setCarbon(true); // indicate that call was rejected on other device
1550 writeLogMessageMissed();
1551 } else {
1552 Log.d(
1553 Config.LOGTAG,
1554 "not able to transition into REJECTED because already in " + this.state);
1555 }
1556 }
1557
1558 private void receiveRejectFromResponder() {
1559 if (isInState(State.PROCEED)) {
1560 Log.d(
1561 Config.LOGTAG,
1562 id.account.getJid()
1563 + ": received reject while still in proceed. callee reconsidered");
1564 closeTransitionLogFinish(State.REJECTED_RACED);
1565 return;
1566 }
1567 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1568 Log.d(
1569 Config.LOGTAG,
1570 id.account.getJid()
1571 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1572 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1573 return;
1574 }
1575 Log.d(
1576 Config.LOGTAG,
1577 id.account.getJid()
1578 + ": ignoring reject from responder because already in state "
1579 + this.state);
1580 }
1581
1582 private void receivePropose(
1583 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1584 final boolean originatedFromMyself =
1585 from.asBareJid().equals(id.account.getJid().asBareJid());
1586 if (originatedFromMyself) {
1587 Log.d(
1588 Config.LOGTAG,
1589 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1590 } else if (transition(
1591 State.PROPOSED,
1592 () -> {
1593 final Collection<RtpDescription> descriptions =
1594 Collections2.transform(
1595 Collections2.filter(
1596 propose.getDescriptions(),
1597 d -> d instanceof RtpDescription),
1598 input -> (RtpDescription) input);
1599 final Collection<Media> media =
1600 Collections2.transform(descriptions, RtpDescription::getMedia);
1601 Preconditions.checkState(
1602 !media.contains(Media.UNKNOWN),
1603 "RTP descriptions contain unknown media");
1604 Log.d(
1605 Config.LOGTAG,
1606 id.account.getJid().asBareJid()
1607 + ": received session proposal from "
1608 + from
1609 + " for "
1610 + media);
1611 this.proposedMedia = Sets.newHashSet(media);
1612 })) {
1613 if (serverMsgId != null) {
1614 this.message.setServerMsgId(serverMsgId);
1615 }
1616 this.message.setTime(timestamp);
1617 startRinging();
1618 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1619 sendJingleMessage("ringing");
1620 }
1621 } else {
1622 Log.d(
1623 Config.LOGTAG,
1624 id.account.getJid()
1625 + ": ignoring session proposal because already in "
1626 + state);
1627 }
1628 }
1629
1630 private void startRinging() {
1631 Log.d(
1632 Config.LOGTAG,
1633 id.account.getJid().asBareJid()
1634 + ": received call from "
1635 + id.with
1636 + ". start ringing");
1637 ringingTimeoutFuture =
1638 jingleConnectionManager.schedule(
1639 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1640 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1641 }
1642
1643 private synchronized void ringingTimeout() {
1644 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1645 switch (this.state) {
1646 case PROPOSED:
1647 message.markUnread();
1648 rejectCallFromProposed();
1649 break;
1650 case SESSION_INITIALIZED:
1651 message.markUnread();
1652 rejectCallFromSessionInitiate();
1653 break;
1654 }
1655 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1656 }
1657
1658 private void cancelRingingTimeout() {
1659 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1660 if (future != null && !future.isCancelled()) {
1661 future.cancel(false);
1662 }
1663 }
1664
1665 private void receiveProceed(
1666 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1667 final Set<Media> media =
1668 Preconditions.checkNotNull(
1669 this.proposedMedia, "Proposed media has to be set before handling proceed");
1670 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1671 if (from.equals(id.with)) {
1672 if (isInitiator()) {
1673 if (transition(State.PROCEED)) {
1674 if (serverMsgId != null) {
1675 this.message.setServerMsgId(serverMsgId);
1676 }
1677 this.message.setTime(timestamp);
1678 final Integer remoteDeviceId = proceed.getDeviceId();
1679 if (isOmemoEnabled()) {
1680 this.omemoVerification.setDeviceId(remoteDeviceId);
1681 } else {
1682 if (remoteDeviceId != null) {
1683 Log.d(
1684 Config.LOGTAG,
1685 id.account.getJid().asBareJid()
1686 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1687 }
1688 this.omemoVerification.setDeviceId(null);
1689 }
1690 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1691 } else {
1692 Log.d(
1693 Config.LOGTAG,
1694 String.format(
1695 "%s: ignoring proceed because already in %s",
1696 id.account.getJid().asBareJid(), this.state));
1697 }
1698 } else {
1699 Log.d(
1700 Config.LOGTAG,
1701 String.format(
1702 "%s: ignoring proceed because we were not initializing",
1703 id.account.getJid().asBareJid()));
1704 }
1705 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1706 if (transition(State.ACCEPTED)) {
1707 Log.d(
1708 Config.LOGTAG,
1709 id.account.getJid().asBareJid()
1710 + ": moved session with "
1711 + id.with
1712 + " into state accepted after received carbon copied proceed");
1713 acceptedOnOtherDevice(serverMsgId, timestamp);
1714 }
1715 } else {
1716 Log.d(
1717 Config.LOGTAG,
1718 String.format(
1719 "%s: ignoring proceed from %s. was expected from %s",
1720 id.account.getJid().asBareJid(), from, id.with));
1721 }
1722 }
1723
1724 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1725 if (from.equals(id.with)) {
1726 final State target =
1727 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1728 if (transition(target)) {
1729 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1730 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1731 Log.d(
1732 Config.LOGTAG,
1733 id.account.getJid().asBareJid()
1734 + ": session with "
1735 + id.with
1736 + " has been retracted (serverMsgId="
1737 + serverMsgId
1738 + ")");
1739 if (serverMsgId != null) {
1740 this.message.setServerMsgId(serverMsgId);
1741 }
1742 this.message.setTime(timestamp);
1743 if (target == State.RETRACTED) {
1744 this.message.markUnread();
1745 }
1746 writeLogMessageMissed();
1747 finish();
1748 } else {
1749 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1750 }
1751 } else {
1752 // TODO parse retract from self
1753 Log.d(
1754 Config.LOGTAG,
1755 id.account.getJid().asBareJid()
1756 + ": received retract from "
1757 + from
1758 + ". expected retract from"
1759 + id.with
1760 + ". ignoring");
1761 }
1762 }
1763
1764 public void sendSessionInitiate() {
1765 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1766 }
1767
1768 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1769 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1770 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1771 }
1772
1773 private synchronized void sendSessionInitiate(
1774 final Set<Media> media,
1775 final State targetState,
1776 final List<PeerConnection.IceServer> iceServers) {
1777 if (isTerminated()) {
1778 Log.w(
1779 Config.LOGTAG,
1780 id.account.getJid().asBareJid()
1781 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1782 return;
1783 }
1784 try {
1785 setupWebRTC(media, iceServers);
1786 } catch (final WebRTCWrapper.InitializationException e) {
1787 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1788 webRTCWrapper.close();
1789 sendRetract(Reason.ofThrowable(e));
1790 return;
1791 }
1792 try {
1793 org.webrtc.SessionDescription webRTCSessionDescription =
1794 this.webRTCWrapper.setLocalDescription().get();
1795 prepareSessionInitiate(webRTCSessionDescription, targetState);
1796 } catch (final Exception e) {
1797 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1798 // exceptions
1799 failureToInitiateSession(e, targetState);
1800 }
1801 }
1802
1803 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1804 if (isTerminated()) {
1805 return;
1806 }
1807 Log.d(
1808 Config.LOGTAG,
1809 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1810 Throwables.getRootCause(throwable));
1811 webRTCWrapper.close();
1812 final Reason reason = Reason.ofThrowable(throwable);
1813 if (isInState(targetState)) {
1814 sendSessionTerminate(reason, throwable.getMessage());
1815 } else {
1816 sendRetract(reason);
1817 }
1818 }
1819
1820 private void sendRetract(final Reason reason) {
1821 // TODO embed reason into retract
1822 sendJingleMessage("retract", id.with.asBareJid());
1823 transitionOrThrow(reasonToState(reason));
1824 this.finish();
1825 }
1826
1827 private void prepareSessionInitiate(
1828 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1829 final SessionDescription sessionDescription =
1830 SessionDescription.parse(webRTCSessionDescription.description);
1831 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1832 this.initiatorRtpContentMap = rtpContentMap;
1833 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1834 encryptSessionInitiate(rtpContentMap);
1835 Futures.addCallback(
1836 outgoingContentMapFuture,
1837 new FutureCallback<RtpContentMap>() {
1838 @Override
1839 public void onSuccess(final RtpContentMap outgoingContentMap) {
1840 sendSessionInitiate(outgoingContentMap, targetState);
1841 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1842 }
1843
1844 @Override
1845 public void onFailure(@NonNull final Throwable throwable) {
1846 failureToInitiateSession(throwable, targetState);
1847 }
1848 },
1849 MoreExecutors.directExecutor());
1850 }
1851
1852 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1853 if (isTerminated()) {
1854 Log.w(
1855 Config.LOGTAG,
1856 id.account.getJid().asBareJid()
1857 + ": preparing session was too slow. already terminated. nothing to do.");
1858 return;
1859 }
1860 this.transitionOrThrow(targetState);
1861 final JinglePacket sessionInitiate =
1862 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1863 send(sessionInitiate);
1864 }
1865
1866 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1867 final RtpContentMap rtpContentMap) {
1868 if (this.omemoVerification.hasDeviceId()) {
1869 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1870 verifiedPayloadFuture =
1871 id.account
1872 .getAxolotlService()
1873 .encrypt(
1874 rtpContentMap,
1875 id.with,
1876 omemoVerification.getDeviceId());
1877 final ListenableFuture<RtpContentMap> future =
1878 Futures.transform(
1879 verifiedPayloadFuture,
1880 verifiedPayload -> {
1881 omemoVerification.setSessionFingerprint(
1882 verifiedPayload.getFingerprint());
1883 return verifiedPayload.getPayload();
1884 },
1885 MoreExecutors.directExecutor());
1886 if (Config.REQUIRE_RTP_VERIFICATION) {
1887 return future;
1888 }
1889 return Futures.catching(
1890 future,
1891 CryptoFailedException.class,
1892 e -> {
1893 Log.w(
1894 Config.LOGTAG,
1895 id.account.getJid().asBareJid()
1896 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1897 e);
1898 return rtpContentMap;
1899 },
1900 MoreExecutors.directExecutor());
1901 } else {
1902 return Futures.immediateFuture(rtpContentMap);
1903 }
1904 }
1905
1906 private void sendSessionTerminate(final Reason reason) {
1907 sendSessionTerminate(reason, null);
1908 }
1909
1910 private void sendSessionTerminate(final Reason reason, final String text) {
1911 final State previous = this.state;
1912 final State target = reasonToState(reason);
1913 transitionOrThrow(target);
1914 if (previous != State.NULL) {
1915 writeLogMessage(target);
1916 }
1917 final JinglePacket jinglePacket =
1918 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1919 jinglePacket.setReason(reason, text);
1920 Log.d(Config.LOGTAG, jinglePacket.toString());
1921 send(jinglePacket);
1922 finish();
1923 }
1924
1925 private void sendTransportInfo(
1926 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1927 final RtpContentMap transportInfo;
1928 try {
1929 final RtpContentMap rtpContentMap =
1930 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1931 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1932 } catch (final Exception e) {
1933 Log.d(
1934 Config.LOGTAG,
1935 id.account.getJid().asBareJid()
1936 + ": unable to prepare transport-info from candidate for content="
1937 + contentName);
1938 return;
1939 }
1940 final JinglePacket jinglePacket =
1941 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1942 send(jinglePacket);
1943 }
1944
1945 private void send(final JinglePacket jinglePacket) {
1946 jinglePacket.setTo(id.with);
1947 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1948 }
1949
1950 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1951 if (response.getType() == IqPacket.TYPE.ERROR) {
1952 handleIqErrorResponse(response);
1953 return;
1954 }
1955 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1956 handleIqTimeoutResponse(response);
1957 }
1958 }
1959
1960 private void handleIqErrorResponse(final IqPacket response) {
1961 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1962 final String errorCondition = response.getErrorCondition();
1963 Log.d(
1964 Config.LOGTAG,
1965 id.account.getJid().asBareJid()
1966 + ": received IQ-error from "
1967 + response.getFrom()
1968 + " in RTP session. "
1969 + errorCondition);
1970 if (isTerminated()) {
1971 Log.i(
1972 Config.LOGTAG,
1973 id.account.getJid().asBareJid()
1974 + ": ignoring error because session was already terminated");
1975 return;
1976 }
1977 this.webRTCWrapper.close();
1978 final State target;
1979 if (Arrays.asList(
1980 "service-unavailable",
1981 "recipient-unavailable",
1982 "remote-server-not-found",
1983 "remote-server-timeout")
1984 .contains(errorCondition)) {
1985 target = State.TERMINATED_CONNECTIVITY_ERROR;
1986 } else {
1987 target = State.TERMINATED_APPLICATION_FAILURE;
1988 }
1989 transitionOrThrow(target);
1990 this.finish();
1991 }
1992
1993 private void handleIqTimeoutResponse(final IqPacket response) {
1994 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1995 Log.d(
1996 Config.LOGTAG,
1997 id.account.getJid().asBareJid()
1998 + ": received IQ timeout in RTP session with "
1999 + id.with
2000 + ". terminating with connectivity error");
2001 if (isTerminated()) {
2002 Log.i(
2003 Config.LOGTAG,
2004 id.account.getJid().asBareJid()
2005 + ": ignoring error because session was already terminated");
2006 return;
2007 }
2008 this.webRTCWrapper.close();
2009 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
2010 this.finish();
2011 }
2012
2013 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
2014 Log.d(
2015 Config.LOGTAG,
2016 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
2017 this.webRTCWrapper.close();
2018 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
2019 respondWithOutOfOrder(jinglePacket);
2020 this.finish();
2021 }
2022
2023 private void respondWithTieBreak(final JinglePacket jinglePacket) {
2024 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
2025 }
2026
2027 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
2028 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
2029 }
2030
2031 void respondWithJingleError(
2032 final IqPacket original,
2033 String jingleCondition,
2034 String condition,
2035 String conditionType) {
2036 jingleConnectionManager.respondWithJingleError(
2037 id.account, original, jingleCondition, condition, conditionType);
2038 }
2039
2040 private void respondOk(final JinglePacket jinglePacket) {
2041 xmppConnectionService.sendIqPacket(
2042 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
2043 }
2044
2045 public RtpEndUserState getEndUserState() {
2046 switch (this.state) {
2047 case NULL:
2048 case PROPOSED:
2049 case SESSION_INITIALIZED:
2050 if (isInitiator()) {
2051 return RtpEndUserState.RINGING;
2052 } else {
2053 return RtpEndUserState.INCOMING_CALL;
2054 }
2055 case PROCEED:
2056 if (isInitiator()) {
2057 return RtpEndUserState.RINGING;
2058 } else {
2059 return RtpEndUserState.ACCEPTING_CALL;
2060 }
2061 case SESSION_INITIALIZED_PRE_APPROVED:
2062 if (isInitiator()) {
2063 return RtpEndUserState.RINGING;
2064 } else {
2065 return RtpEndUserState.CONNECTING;
2066 }
2067 case SESSION_ACCEPTED:
2068 final ContentAddition ca = getPendingContentAddition();
2069 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2070 return RtpEndUserState.INCOMING_CONTENT_ADD;
2071 }
2072 return getPeerConnectionStateAsEndUserState();
2073 case REJECTED:
2074 case REJECTED_RACED:
2075 case TERMINATED_DECLINED_OR_BUSY:
2076 if (isInitiator()) {
2077 return RtpEndUserState.DECLINED_OR_BUSY;
2078 } else {
2079 return RtpEndUserState.ENDED;
2080 }
2081 case TERMINATED_SUCCESS:
2082 case ACCEPTED:
2083 case RETRACTED:
2084 case TERMINATED_CANCEL_OR_TIMEOUT:
2085 return RtpEndUserState.ENDED;
2086 case RETRACTED_RACED:
2087 if (isInitiator()) {
2088 return RtpEndUserState.ENDED;
2089 } else {
2090 return RtpEndUserState.RETRACTED;
2091 }
2092 case TERMINATED_CONNECTIVITY_ERROR:
2093 return zeroDuration()
2094 ? RtpEndUserState.CONNECTIVITY_ERROR
2095 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2096 case TERMINATED_APPLICATION_FAILURE:
2097 return RtpEndUserState.APPLICATION_ERROR;
2098 case TERMINATED_SECURITY_ERROR:
2099 return RtpEndUserState.SECURITY_ERROR;
2100 }
2101 throw new IllegalStateException(
2102 String.format("%s has no equivalent EndUserState", this.state));
2103 }
2104
2105 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2106 final PeerConnection.PeerConnectionState state;
2107 try {
2108 state = webRTCWrapper.getState();
2109 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2110 // We usually close the WebRTCWrapper *before* transitioning so we might still
2111 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2112 return RtpEndUserState.ENDING_CALL;
2113 }
2114 switch (state) {
2115 case CONNECTED:
2116 return RtpEndUserState.CONNECTED;
2117 case NEW:
2118 case CONNECTING:
2119 return RtpEndUserState.CONNECTING;
2120 case CLOSED:
2121 return RtpEndUserState.ENDING_CALL;
2122 default:
2123 return zeroDuration()
2124 ? RtpEndUserState.CONNECTIVITY_ERROR
2125 : RtpEndUserState.RECONNECTING;
2126 }
2127 }
2128
2129 public ContentAddition getPendingContentAddition() {
2130 final RtpContentMap in = this.incomingContentAdd;
2131 final RtpContentMap out = this.outgoingContentAdd;
2132 if (out != null) {
2133 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2134 } else if (in != null) {
2135 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2136 } else {
2137 return null;
2138 }
2139 }
2140
2141 public Set<Media> getMedia() {
2142 final State current = getState();
2143 if (current == State.NULL) {
2144 if (isInitiator()) {
2145 return Preconditions.checkNotNull(
2146 this.proposedMedia, "RTP connection has not been initialized properly");
2147 }
2148 throw new IllegalStateException("RTP connection has not been initialized yet");
2149 }
2150 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2151 return Preconditions.checkNotNull(
2152 this.proposedMedia, "RTP connection has not been initialized properly");
2153 }
2154 final RtpContentMap localContentMap = getLocalContentMap();
2155 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2156 if (localContentMap != null) {
2157 return localContentMap.getMedia();
2158 } else if (initiatorContentMap != null) {
2159 return initiatorContentMap.getMedia();
2160 } else if (isTerminated()) {
2161 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2162 } else {
2163 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2164 }
2165 }
2166
2167 public boolean isVerified() {
2168 final String fingerprint = this.omemoVerification.getFingerprint();
2169 if (fingerprint == null) {
2170 return false;
2171 }
2172 final FingerprintStatus status =
2173 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2174 return status != null && status.isVerified();
2175 }
2176
2177 public boolean addMedia(final Media media) {
2178 final Set<Media> currentMedia = getMedia();
2179 if (currentMedia.contains(media)) {
2180 throw new IllegalStateException(String.format("%s has already been proposed", media));
2181 }
2182 // TODO add state protection - can only add while ACCEPTED or so
2183 Log.d(Config.LOGTAG,"adding media: "+media);
2184 return webRTCWrapper.addTrack(media);
2185 }
2186
2187 public synchronized void acceptCall() {
2188 switch (this.state) {
2189 case PROPOSED:
2190 cancelRingingTimeout();
2191 acceptCallFromProposed();
2192 break;
2193 case SESSION_INITIALIZED:
2194 cancelRingingTimeout();
2195 acceptCallFromSessionInitialized();
2196 break;
2197 case ACCEPTED:
2198 Log.w(
2199 Config.LOGTAG,
2200 id.account.getJid().asBareJid()
2201 + ": the call has already been accepted with another client. UI was just lagging behind");
2202 break;
2203 case PROCEED:
2204 case SESSION_ACCEPTED:
2205 Log.w(
2206 Config.LOGTAG,
2207 id.account.getJid().asBareJid()
2208 + ": the call has already been accepted. user probably double tapped the UI");
2209 break;
2210 default:
2211 throw new IllegalStateException("Can not accept call from " + this.state);
2212 }
2213 }
2214
2215 public void notifyPhoneCall() {
2216 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2217 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2218 rejectCall();
2219 } else {
2220 endCall();
2221 }
2222 }
2223
2224 public synchronized void rejectCall() {
2225 if (isTerminated()) {
2226 Log.w(
2227 Config.LOGTAG,
2228 id.account.getJid().asBareJid()
2229 + ": received rejectCall() when session has already been terminated. nothing to do");
2230 return;
2231 }
2232 switch (this.state) {
2233 case PROPOSED:
2234 rejectCallFromProposed();
2235 break;
2236 case SESSION_INITIALIZED:
2237 rejectCallFromSessionInitiate();
2238 break;
2239 default:
2240 throw new IllegalStateException("Can not reject call from " + this.state);
2241 }
2242 }
2243
2244 public synchronized void endCall() {
2245 if (isTerminated()) {
2246 Log.w(
2247 Config.LOGTAG,
2248 id.account.getJid().asBareJid()
2249 + ": received endCall() when session has already been terminated. nothing to do");
2250 return;
2251 }
2252 if (isInState(State.PROPOSED) && !isInitiator()) {
2253 rejectCallFromProposed();
2254 return;
2255 }
2256 if (isInState(State.PROCEED)) {
2257 if (isInitiator()) {
2258 retractFromProceed();
2259 } else {
2260 rejectCallFromProceed();
2261 }
2262 return;
2263 }
2264 if (isInitiator()
2265 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2266 this.webRTCWrapper.close();
2267 sendSessionTerminate(Reason.CANCEL);
2268 return;
2269 }
2270 if (isInState(State.SESSION_INITIALIZED)) {
2271 rejectCallFromSessionInitiate();
2272 return;
2273 }
2274 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2275 this.webRTCWrapper.close();
2276 sendSessionTerminate(Reason.SUCCESS);
2277 return;
2278 }
2279 if (isInState(
2280 State.TERMINATED_APPLICATION_FAILURE,
2281 State.TERMINATED_CONNECTIVITY_ERROR,
2282 State.TERMINATED_DECLINED_OR_BUSY)) {
2283 Log.d(
2284 Config.LOGTAG,
2285 "ignoring request to end call because already in state " + this.state);
2286 return;
2287 }
2288 throw new IllegalStateException(
2289 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2290 }
2291
2292 private void retractFromProceed() {
2293 Log.d(Config.LOGTAG, "retract from proceed");
2294 this.sendJingleMessage("retract");
2295 closeTransitionLogFinish(State.RETRACTED_RACED);
2296 }
2297
2298 private void closeTransitionLogFinish(final State state) {
2299 this.webRTCWrapper.close();
2300 transitionOrThrow(state);
2301 writeLogMessage(state);
2302 finish();
2303 }
2304
2305 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2306 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2307 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2308 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2309 }
2310
2311 private void acceptCallFromProposed() {
2312 transitionOrThrow(State.PROCEED);
2313 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2314 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2315 this.sendJingleMessage("proceed");
2316 }
2317
2318 private void rejectCallFromProposed() {
2319 transitionOrThrow(State.REJECTED);
2320 writeLogMessageMissed();
2321 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2322 this.sendJingleMessage("reject");
2323 finish();
2324 }
2325
2326 private void rejectCallFromProceed() {
2327 this.sendJingleMessage("reject");
2328 closeTransitionLogFinish(State.REJECTED_RACED);
2329 }
2330
2331 private void rejectCallFromSessionInitiate() {
2332 webRTCWrapper.close();
2333 sendSessionTerminate(Reason.DECLINE);
2334 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2335 }
2336
2337 private void sendJingleMessage(final String action) {
2338 sendJingleMessage(action, id.with);
2339 }
2340
2341 private void sendJingleMessage(final String action, final Jid to) {
2342 final MessagePacket messagePacket = new MessagePacket();
2343 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2344 messagePacket.setTo(to);
2345 final Element intent =
2346 messagePacket
2347 .addChild(action, Namespace.JINGLE_MESSAGE)
2348 .setAttribute("id", id.sessionId);
2349 if ("proceed".equals(action)) {
2350 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2351 if (isOmemoEnabled()) {
2352 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2353 final Element device =
2354 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2355 device.setAttribute("id", deviceId);
2356 }
2357 }
2358 messagePacket.addChild("store", "urn:xmpp:hints");
2359 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2360 }
2361
2362 private boolean isOmemoEnabled() {
2363 final Conversational conversational = message.getConversation();
2364 if (conversational instanceof Conversation) {
2365 return ((Conversation) conversational).getNextEncryption()
2366 == Message.ENCRYPTION_AXOLOTL;
2367 }
2368 return false;
2369 }
2370
2371 private void acceptCallFromSessionInitialized() {
2372 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2373 sendSessionAccept();
2374 }
2375
2376 private synchronized boolean isInState(State... state) {
2377 return Arrays.asList(state).contains(this.state);
2378 }
2379
2380 private boolean transition(final State target) {
2381 return transition(target, null);
2382 }
2383
2384 private synchronized boolean transition(final State target, final Runnable runnable) {
2385 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2386 if (validTransitions != null && validTransitions.contains(target)) {
2387 this.state = target;
2388 if (runnable != null) {
2389 runnable.run();
2390 }
2391 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2392 updateEndUserState();
2393 updateOngoingCallNotification();
2394 return true;
2395 } else {
2396 return false;
2397 }
2398 }
2399
2400 void transitionOrThrow(final State target) {
2401 if (!transition(target)) {
2402 throw new IllegalStateException(
2403 String.format("Unable to transition from %s to %s", this.state, target));
2404 }
2405 }
2406
2407 @Override
2408 public void onIceCandidate(final IceCandidate iceCandidate) {
2409 final RtpContentMap rtpContentMap =
2410 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2411 final IceUdpTransportInfo.Credentials credentials;
2412 try {
2413 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2414 } catch (final IllegalArgumentException e) {
2415 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2416 return;
2417 }
2418 final String uFrag = credentials.ufrag;
2419 final IceUdpTransportInfo.Candidate candidate =
2420 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2421 if (candidate == null) {
2422 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2423 return;
2424 }
2425 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2426 sendTransportInfo(iceCandidate.sdpMid, candidate);
2427 }
2428
2429 @Override
2430 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2431 Log.d(
2432 Config.LOGTAG,
2433 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2434 this.stateHistory.add(newState);
2435 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2436 this.sessionDuration.start();
2437 updateOngoingCallNotification();
2438 } else if (this.sessionDuration.isRunning()) {
2439 this.sessionDuration.stop();
2440 updateOngoingCallNotification();
2441 }
2442
2443 final boolean neverConnected =
2444 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2445
2446 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2447 if (neverConnected) {
2448 if (isTerminated()) {
2449 Log.d(
2450 Config.LOGTAG,
2451 id.account.getJid().asBareJid()
2452 + ": not sending session-terminate after connectivity error because session is already in state "
2453 + this.state);
2454 return;
2455 }
2456 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2457 return;
2458 } else {
2459 this.restartIce();
2460 }
2461 }
2462 updateEndUserState();
2463 }
2464
2465 private void restartIce() {
2466 this.stateHistory.clear();
2467 this.webRTCWrapper.restartIceAsync();
2468 }
2469
2470 @Override
2471 public void onRenegotiationNeeded() {
2472 this.webRTCWrapper.execute(this::renegotiate);
2473 }
2474
2475 private void renegotiate() {
2476 final SessionDescription sessionDescription;
2477 try {
2478 sessionDescription = setLocalSessionDescription();
2479 } catch (final Exception e) {
2480 final Throwable cause = Throwables.getRootCause(e);
2481 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2482 webRTCWrapper.close();
2483 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2484 return;
2485 }
2486 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2487 final RtpContentMap currentContentMap = getLocalContentMap();
2488 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2489 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2490
2491 Log.d(
2492 Config.LOGTAG,
2493 id.getAccount().getJid().asBareJid()
2494 + ": renegotiate. iceRestart="
2495 + iceRestart
2496 + " content id diff="
2497 + diff);
2498
2499 if (diff.hasModifications() && iceRestart) {
2500 webRTCWrapper.close();
2501 sendSessionTerminate(
2502 Reason.FAILED_APPLICATION,
2503 "WebRTC unexpectedly tried to modify content and transport at once");
2504 return;
2505 }
2506
2507 if (iceRestart) {
2508 initiateIceRestart(rtpContentMap);
2509 return;
2510 } else if (diff.isEmpty()) {
2511 Log.d(
2512 Config.LOGTAG,
2513 "renegotiation. nothing to do. SignalingState="
2514 + this.webRTCWrapper.getSignalingState());
2515 }
2516
2517 if (diff.added.size() > 0) {
2518 modifyLocalContentMap(rtpContentMap);
2519 sendContentAdd(rtpContentMap, diff.added);
2520 }
2521 }
2522
2523 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2524 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2525 final JinglePacket jinglePacket =
2526 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2527 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2528 jinglePacket.setTo(id.with);
2529 xmppConnectionService.sendIqPacket(
2530 id.account,
2531 jinglePacket,
2532 (account, response) -> {
2533 if (response.getType() == IqPacket.TYPE.RESULT) {
2534 Log.d(Config.LOGTAG, "received success to our ice restart");
2535 setLocalContentMap(rtpContentMap);
2536 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2537 return;
2538 }
2539 if (response.getType() == IqPacket.TYPE.ERROR) {
2540 if (isTieBreak(response)) {
2541 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2542 return;
2543 }
2544 handleIqErrorResponse(response);
2545 }
2546 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2547 handleIqTimeoutResponse(response);
2548 }
2549 });
2550 }
2551
2552 private boolean isTieBreak(final IqPacket response) {
2553 final Element error = response.findChild("error");
2554 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2555 }
2556
2557 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2558 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2559 this.outgoingContentAdd = contentAdd;
2560 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
2561 prepareOutgoingContentMap(contentAdd);
2562 Futures.addCallback(
2563 outgoingContentMapFuture,
2564 new FutureCallback<RtpContentMap>() {
2565 @Override
2566 public void onSuccess(final RtpContentMap outgoingContentMap) {
2567 sendContentAdd(outgoingContentMap);
2568 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2569 }
2570
2571 @Override
2572 public void onFailure(@NonNull Throwable throwable) {
2573 failureToPerformAction(JinglePacket.Action.CONTENT_ADD, throwable);
2574 }
2575 },
2576 MoreExecutors.directExecutor());
2577 }
2578
2579 private void sendContentAdd(final RtpContentMap contentAdd) {
2580
2581 final JinglePacket jinglePacket =
2582 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2583 jinglePacket.setTo(id.with);
2584 xmppConnectionService.sendIqPacket(
2585 id.account,
2586 jinglePacket,
2587 (connection, response) -> {
2588 if (response.getType() == IqPacket.TYPE.RESULT) {
2589 Log.d(
2590 Config.LOGTAG,
2591 id.getAccount().getJid().asBareJid()
2592 + ": received ACK to our content-add");
2593 return;
2594 }
2595 if (response.getType() == IqPacket.TYPE.ERROR) {
2596 if (isTieBreak(response)) {
2597 this.outgoingContentAdd = null;
2598 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2599 return;
2600 }
2601 handleIqErrorResponse(response);
2602 }
2603 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2604 handleIqTimeoutResponse(response);
2605 }
2606 });
2607 }
2608
2609 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2610 if (isInitiator()) {
2611 this.initiatorRtpContentMap = rtpContentMap;
2612 } else {
2613 this.responderRtpContentMap = rtpContentMap;
2614 }
2615 }
2616
2617 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2618 if (isInitiator()) {
2619 this.responderRtpContentMap = rtpContentMap;
2620 } else {
2621 this.initiatorRtpContentMap = rtpContentMap;
2622 }
2623 }
2624
2625 // this method is to be used for content map modifications that modify media
2626 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2627 final RtpContentMap activeContents = rtpContentMap.activeContents();
2628 setLocalContentMap(activeContents);
2629 this.webRTCWrapper.switchSpeakerPhonePreference(
2630 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2631 updateEndUserState();
2632 }
2633
2634 private SessionDescription setLocalSessionDescription()
2635 throws ExecutionException, InterruptedException {
2636 final org.webrtc.SessionDescription sessionDescription =
2637 this.webRTCWrapper.setLocalDescription().get();
2638 return SessionDescription.parse(sessionDescription.description);
2639 }
2640
2641 private void closeWebRTCSessionAfterFailedConnection() {
2642 this.webRTCWrapper.close();
2643 synchronized (this) {
2644 if (isTerminated()) {
2645 Log.d(
2646 Config.LOGTAG,
2647 id.account.getJid().asBareJid()
2648 + ": no need to send session-terminate after failed connection. Other party already did");
2649 return;
2650 }
2651 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2652 }
2653 }
2654
2655 public boolean zeroDuration() {
2656 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2657 }
2658
2659 public long getCallDuration() {
2660 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2661 }
2662
2663 public AppRTCAudioManager getAudioManager() {
2664 return webRTCWrapper.getAudioManager();
2665 }
2666
2667 public boolean isMicrophoneEnabled() {
2668 return webRTCWrapper.isMicrophoneEnabled();
2669 }
2670
2671 public boolean setMicrophoneEnabled(final boolean enabled) {
2672 return webRTCWrapper.setMicrophoneEnabled(enabled);
2673 }
2674
2675 public boolean isVideoEnabled() {
2676 return webRTCWrapper.isVideoEnabled();
2677 }
2678
2679 public void setVideoEnabled(final boolean enabled) {
2680 webRTCWrapper.setVideoEnabled(enabled);
2681 }
2682
2683 public boolean isCameraSwitchable() {
2684 return webRTCWrapper.isCameraSwitchable();
2685 }
2686
2687 public boolean isFrontCamera() {
2688 return webRTCWrapper.isFrontCamera();
2689 }
2690
2691 public ListenableFuture<Boolean> switchCamera() {
2692 return webRTCWrapper.switchCamera();
2693 }
2694
2695 @Override
2696 public void onAudioDeviceChanged(
2697 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2698 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2699 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2700 selectedAudioDevice, availableAudioDevices);
2701 }
2702
2703 private void updateEndUserState() {
2704 final RtpEndUserState endUserState = getEndUserState();
2705 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2706 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2707 id.account, id.with, id.sessionId, endUserState);
2708 }
2709
2710 private void updateOngoingCallNotification() {
2711 final State state = this.state;
2712 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2713 final boolean reconnecting;
2714 if (state == State.SESSION_ACCEPTED) {
2715 reconnecting =
2716 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2717 } else {
2718 reconnecting = false;
2719 }
2720 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2721 } else {
2722 xmppConnectionService.removeOngoingCall();
2723 }
2724 }
2725
2726 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2727 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2728 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2729 request.setTo(id.account.getDomain());
2730 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2731 xmppConnectionService.sendIqPacket(
2732 id.account,
2733 request,
2734 (account, response) -> {
2735 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2736 new ImmutableList.Builder<>();
2737 if (response.getType() == IqPacket.TYPE.RESULT) {
2738 final Element services =
2739 response.findChild(
2740 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2741 final List<Element> children =
2742 services == null
2743 ? Collections.emptyList()
2744 : services.getChildren();
2745 for (final Element child : children) {
2746 if ("service".equals(child.getName())) {
2747 final String type = child.getAttribute("type");
2748 final String host = child.getAttribute("host");
2749 final String sport = child.getAttribute("port");
2750 final Integer port =
2751 sport == null ? null : Ints.tryParse(sport);
2752 final String transport = child.getAttribute("transport");
2753 final String username = child.getAttribute("username");
2754 final String password = child.getAttribute("password");
2755 if (Strings.isNullOrEmpty(host) || port == null) {
2756 continue;
2757 }
2758 if (port < 0 || port > 65535) {
2759 continue;
2760 }
2761
2762
2763
2764
2765 if (Arrays.asList("stun", "stuns", "turn", "turns")
2766 .contains(type)
2767 && Arrays.asList("udp", "tcp").contains(transport)) {
2768 if (Arrays.asList("stuns", "turns").contains(type)
2769 && "udp".equals(transport)) {
2770 Log.d(
2771 Config.LOGTAG,
2772 id.account.getJid().asBareJid()
2773 + ": skipping invalid combination of udp/tls in external services");
2774 continue;
2775 }
2776
2777 // STUN URLs do not support a query section since M110
2778 final String uri;
2779 if (Arrays.asList("stun","stuns").contains(type)) {
2780 uri = String.format("%s:%s:%s", type, IP.wrapIPv6(host),port);
2781 } else {
2782 uri = String.format(
2783 "%s:%s:%s?transport=%s",
2784 type,
2785 IP.wrapIPv6(host),
2786 port,
2787 transport);
2788 }
2789
2790 final PeerConnection.IceServer.Builder iceServerBuilder =
2791 PeerConnection.IceServer.builder(uri);
2792 iceServerBuilder.setTlsCertPolicy(
2793 PeerConnection.TlsCertPolicy
2794 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2795 if (username != null && password != null) {
2796 iceServerBuilder.setUsername(username);
2797 iceServerBuilder.setPassword(password);
2798 } else if (Arrays.asList("turn", "turns").contains(type)) {
2799 // The WebRTC spec requires throwing an
2800 // InvalidAccessError when username (from libwebrtc
2801 // source coder)
2802 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2803 Log.d(
2804 Config.LOGTAG,
2805 id.account.getJid().asBareJid()
2806 + ": skipping "
2807 + type
2808 + "/"
2809 + transport
2810 + " without username and password");
2811 continue;
2812 }
2813 final PeerConnection.IceServer iceServer =
2814 iceServerBuilder.createIceServer();
2815 Log.d(
2816 Config.LOGTAG,
2817 id.account.getJid().asBareJid()
2818 + ": discovered ICE Server: "
2819 + iceServer);
2820 listBuilder.add(iceServer);
2821 }
2822 }
2823 }
2824 }
2825 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2826 if (iceServers.size() == 0) {
2827 Log.w(
2828 Config.LOGTAG,
2829 id.account.getJid().asBareJid()
2830 + ": no ICE server found "
2831 + response);
2832 }
2833 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2834 });
2835 } else {
2836 Log.w(
2837 Config.LOGTAG,
2838 id.account.getJid().asBareJid() + ": has no external service discovery");
2839 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2840 }
2841 }
2842
2843 private void finish() {
2844 if (isTerminated()) {
2845 this.cancelRingingTimeout();
2846 this.webRTCWrapper.verifyClosed();
2847 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2848 this.jingleConnectionManager.finishConnectionOrThrow(this);
2849 } else {
2850 throw new IllegalStateException(
2851 String.format("Unable to call finish from %s", this.state));
2852 }
2853 }
2854
2855 private void writeLogMessage(final State state) {
2856 final long duration = getCallDuration();
2857 if (state == State.TERMINATED_SUCCESS
2858 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2859 writeLogMessageSuccess(duration);
2860 } else {
2861 writeLogMessageMissed();
2862 }
2863 }
2864
2865 private void writeLogMessageSuccess(final long duration) {
2866 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2867 this.writeMessage();
2868 }
2869
2870 private void writeLogMessageMissed() {
2871 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2872 this.writeMessage();
2873 }
2874
2875 private void writeMessage() {
2876 final Conversational conversational = message.getConversation();
2877 if (conversational instanceof Conversation) {
2878 ((Conversation) conversational).add(this.message);
2879 xmppConnectionService.createMessageAsync(message);
2880 xmppConnectionService.updateConversationUi();
2881 } else {
2882 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2883 }
2884 }
2885
2886 public State getState() {
2887 return this.state;
2888 }
2889
2890 boolean isTerminated() {
2891 return TERMINATED.contains(this.state);
2892 }
2893
2894 public Optional<VideoTrack> getLocalVideoTrack() {
2895 return webRTCWrapper.getLocalVideoTrack();
2896 }
2897
2898 public Optional<VideoTrack> getRemoteVideoTrack() {
2899 return webRTCWrapper.getRemoteVideoTrack();
2900 }
2901
2902 public EglBase.Context getEglBaseContext() {
2903 return webRTCWrapper.getEglBaseContext();
2904 }
2905
2906 void setProposedMedia(final Set<Media> media) {
2907 this.proposedMedia = media;
2908 }
2909
2910 public void fireStateUpdate() {
2911 final RtpEndUserState endUserState = getEndUserState();
2912 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2913 id.account, id.with, id.sessionId, endUserState);
2914 }
2915
2916 public boolean isSwitchToVideoAvailable() {
2917 final boolean prerequisite =
2918 Media.audioOnly(getMedia())
2919 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2920 .contains(getEndUserState());
2921 return prerequisite && remoteHasVideoFeature();
2922 }
2923
2924 private boolean remoteHasVideoFeature() {
2925 final Contact contact = id.getContact();
2926 final Presence presence =
2927 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2928 final ServiceDiscoveryResult serviceDiscoveryResult =
2929 presence == null ? null : presence.getServiceDiscoveryResult();
2930 final List<String> features =
2931 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
2932 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
2933 }
2934
2935 private interface OnIceServersDiscovered {
2936 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2937 }
2938}