1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Joiner;
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.base.Stopwatch;
12import com.google.common.base.Strings;
13import com.google.common.base.Throwables;
14import com.google.common.collect.Collections2;
15import com.google.common.collect.ImmutableList;
16import com.google.common.collect.ImmutableMap;
17import com.google.common.collect.ImmutableSet;
18import com.google.common.collect.Sets;
19import com.google.common.primitives.Ints;
20import com.google.common.util.concurrent.FutureCallback;
21import com.google.common.util.concurrent.Futures;
22import com.google.common.util.concurrent.ListenableFuture;
23import com.google.common.util.concurrent.MoreExecutors;
24
25import org.webrtc.EglBase;
26import org.webrtc.IceCandidate;
27import org.webrtc.PeerConnection;
28import org.webrtc.VideoTrack;
29
30import java.util.Arrays;
31import java.util.Collection;
32import java.util.Collections;
33import java.util.LinkedList;
34import java.util.List;
35import java.util.Map;
36import java.util.Queue;
37import java.util.Set;
38import java.util.concurrent.ExecutionException;
39import java.util.concurrent.ScheduledFuture;
40import java.util.concurrent.TimeUnit;
41
42import eu.siacs.conversations.BuildConfig;
43import eu.siacs.conversations.Config;
44import eu.siacs.conversations.crypto.axolotl.AxolotlService;
45import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
46import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
47import eu.siacs.conversations.entities.Account;
48import eu.siacs.conversations.entities.Contact;
49import eu.siacs.conversations.entities.Conversation;
50import eu.siacs.conversations.entities.Conversational;
51import eu.siacs.conversations.entities.Message;
52import eu.siacs.conversations.entities.Presence;
53import eu.siacs.conversations.entities.RtpSessionStatus;
54import eu.siacs.conversations.entities.ServiceDiscoveryResult;
55import eu.siacs.conversations.services.AppRTCAudioManager;
56import eu.siacs.conversations.utils.IP;
57import eu.siacs.conversations.xml.Element;
58import eu.siacs.conversations.xml.Namespace;
59import eu.siacs.conversations.xmpp.Jid;
60import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
61import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
62import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
63import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
64import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
65import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
66import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
67import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
68import eu.siacs.conversations.xmpp.stanzas.IqPacket;
69import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
70
71public class JingleRtpConnection extends AbstractJingleConnection
72 implements WebRTCWrapper.EventCallback {
73
74 public static final List<State> STATES_SHOWING_ONGOING_CALL =
75 Arrays.asList(
76 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
77 private static final long BUSY_TIME_OUT = 30;
78 private static final List<State> TERMINATED =
79 Arrays.asList(
80 State.ACCEPTED,
81 State.REJECTED,
82 State.REJECTED_RACED,
83 State.RETRACTED,
84 State.RETRACTED_RACED,
85 State.TERMINATED_SUCCESS,
86 State.TERMINATED_DECLINED_OR_BUSY,
87 State.TERMINATED_CONNECTIVITY_ERROR,
88 State.TERMINATED_CANCEL_OR_TIMEOUT,
89 State.TERMINATED_APPLICATION_FAILURE,
90 State.TERMINATED_SECURITY_ERROR);
91
92 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
93
94 static {
95 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
96 new ImmutableMap.Builder<>();
97 transitionBuilder.put(
98 State.NULL,
99 ImmutableList.of(
100 State.PROPOSED,
101 State.SESSION_INITIALIZED,
102 State.TERMINATED_APPLICATION_FAILURE,
103 State.TERMINATED_SECURITY_ERROR));
104 transitionBuilder.put(
105 State.PROPOSED,
106 ImmutableList.of(
107 State.ACCEPTED,
108 State.PROCEED,
109 State.REJECTED,
110 State.RETRACTED,
111 State.TERMINATED_APPLICATION_FAILURE,
112 State.TERMINATED_SECURITY_ERROR,
113 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
114 // rebinds
115 ));
116 transitionBuilder.put(
117 State.PROCEED,
118 ImmutableList.of(
119 State.REJECTED_RACED,
120 State.RETRACTED_RACED,
121 State.SESSION_INITIALIZED_PRE_APPROVED,
122 State.TERMINATED_SUCCESS,
123 State.TERMINATED_APPLICATION_FAILURE,
124 State.TERMINATED_SECURITY_ERROR,
125 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
126 // bounces of the proceed message
127 ));
128 transitionBuilder.put(
129 State.SESSION_INITIALIZED,
130 ImmutableList.of(
131 State.SESSION_ACCEPTED,
132 State.TERMINATED_SUCCESS,
133 State.TERMINATED_DECLINED_OR_BUSY,
134 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
135 // and IQ timeouts
136 State.TERMINATED_CANCEL_OR_TIMEOUT,
137 State.TERMINATED_APPLICATION_FAILURE,
138 State.TERMINATED_SECURITY_ERROR));
139 transitionBuilder.put(
140 State.SESSION_INITIALIZED_PRE_APPROVED,
141 ImmutableList.of(
142 State.SESSION_ACCEPTED,
143 State.TERMINATED_SUCCESS,
144 State.TERMINATED_DECLINED_OR_BUSY,
145 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
146 // and IQ timeouts
147 State.TERMINATED_CANCEL_OR_TIMEOUT,
148 State.TERMINATED_APPLICATION_FAILURE,
149 State.TERMINATED_SECURITY_ERROR));
150 transitionBuilder.put(
151 State.SESSION_ACCEPTED,
152 ImmutableList.of(
153 State.TERMINATED_SUCCESS,
154 State.TERMINATED_DECLINED_OR_BUSY,
155 State.TERMINATED_CONNECTIVITY_ERROR,
156 State.TERMINATED_CANCEL_OR_TIMEOUT,
157 State.TERMINATED_APPLICATION_FAILURE,
158 State.TERMINATED_SECURITY_ERROR));
159 VALID_TRANSITIONS = transitionBuilder.build();
160 }
161
162 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
163 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
164 pendingIceCandidates = new LinkedList<>();
165 private final OmemoVerification omemoVerification = new OmemoVerification();
166 private final Message message;
167 private State state = State.NULL;
168 private Set<Media> proposedMedia;
169 private RtpContentMap initiatorRtpContentMap;
170 private RtpContentMap responderRtpContentMap;
171 private RtpContentMap incomingContentAdd;
172 private RtpContentMap outgoingContentAdd;
173 private IceUdpTransportInfo.Setup peerDtlsSetup;
174 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
175 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
176 private ScheduledFuture<?> ringingTimeoutFuture;
177
178 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
179 super(jingleConnectionManager, id, initiator);
180 final Conversation conversation =
181 jingleConnectionManager
182 .getXmppConnectionService()
183 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
184 this.message =
185 new Message(
186 conversation,
187 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
188 Message.TYPE_RTP_SESSION,
189 id.sessionId);
190 }
191
192 private static State reasonToState(Reason reason) {
193 switch (reason) {
194 case SUCCESS:
195 return State.TERMINATED_SUCCESS;
196 case DECLINE:
197 case BUSY:
198 return State.TERMINATED_DECLINED_OR_BUSY;
199 case CANCEL:
200 case TIMEOUT:
201 return State.TERMINATED_CANCEL_OR_TIMEOUT;
202 case SECURITY_ERROR:
203 return State.TERMINATED_SECURITY_ERROR;
204 case FAILED_APPLICATION:
205 case UNSUPPORTED_TRANSPORTS:
206 case UNSUPPORTED_APPLICATIONS:
207 return State.TERMINATED_APPLICATION_FAILURE;
208 default:
209 return State.TERMINATED_CONNECTIVITY_ERROR;
210 }
211 }
212
213 @Override
214 synchronized void deliverPacket(final JinglePacket jinglePacket) {
215 switch (jinglePacket.getAction()) {
216 case SESSION_INITIATE:
217 receiveSessionInitiate(jinglePacket);
218 break;
219 case TRANSPORT_INFO:
220 receiveTransportInfo(jinglePacket);
221 break;
222 case SESSION_ACCEPT:
223 receiveSessionAccept(jinglePacket);
224 break;
225 case SESSION_TERMINATE:
226 receiveSessionTerminate(jinglePacket);
227 break;
228 case CONTENT_ADD:
229 receiveContentAdd(jinglePacket);
230 break;
231 case CONTENT_ACCEPT:
232 receiveContentAccept(jinglePacket);
233 break;
234 case CONTENT_REJECT:
235 receiveContentReject(jinglePacket);
236 break;
237 case CONTENT_REMOVE:
238 receiveContentRemove(jinglePacket);
239 break;
240 default:
241 respondOk(jinglePacket);
242 Log.d(
243 Config.LOGTAG,
244 String.format(
245 "%s: received unhandled jingle action %s",
246 id.account.getJid().asBareJid(), jinglePacket.getAction()));
247 break;
248 }
249 }
250
251 @Override
252 synchronized void notifyRebound() {
253 if (isTerminated()) {
254 return;
255 }
256 webRTCWrapper.close();
257 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
258 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
259 }
260 if (isInState(
261 State.SESSION_INITIALIZED,
262 State.SESSION_INITIALIZED_PRE_APPROVED,
263 State.SESSION_ACCEPTED)) {
264 // we might have already changed resources (full jid) at this point; so this might not
265 // even reach the other party
266 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
267 } else {
268 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
269 finish();
270 }
271 }
272
273 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
274 respondOk(jinglePacket);
275 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
276 final State previous = this.state;
277 Log.d(
278 Config.LOGTAG,
279 id.account.getJid().asBareJid()
280 + ": received session terminate reason="
281 + wrapper.reason
282 + "("
283 + Strings.nullToEmpty(wrapper.text)
284 + ") while in state "
285 + previous);
286 if (TERMINATED.contains(previous)) {
287 Log.d(
288 Config.LOGTAG,
289 id.account.getJid().asBareJid()
290 + ": ignoring session terminate because already in "
291 + previous);
292 return;
293 }
294 webRTCWrapper.close();
295 final State target = reasonToState(wrapper.reason);
296 transitionOrThrow(target);
297 writeLogMessage(target);
298 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
299 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
300 }
301 finish();
302 }
303
304 private void receiveTransportInfo(final JinglePacket jinglePacket) {
305 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
306 // INITIALIZED only after transport-info has been received
307 if (isInState(
308 State.NULL,
309 State.PROCEED,
310 State.SESSION_INITIALIZED,
311 State.SESSION_INITIALIZED_PRE_APPROVED,
312 State.SESSION_ACCEPTED)) {
313 final RtpContentMap contentMap;
314 try {
315 contentMap = RtpContentMap.of(jinglePacket);
316 } catch (final IllegalArgumentException | NullPointerException e) {
317 Log.d(
318 Config.LOGTAG,
319 id.account.getJid().asBareJid()
320 + ": improperly formatted contents; ignoring",
321 e);
322 respondOk(jinglePacket);
323 return;
324 }
325 receiveTransportInfo(jinglePacket, contentMap);
326 } else {
327 if (isTerminated()) {
328 respondOk(jinglePacket);
329 Log.d(
330 Config.LOGTAG,
331 id.account.getJid().asBareJid()
332 + ": ignoring out-of-order transport info; we where already terminated");
333 } else {
334 Log.d(
335 Config.LOGTAG,
336 id.account.getJid().asBareJid()
337 + ": received transport info while in state="
338 + this.state);
339 terminateWithOutOfOrder(jinglePacket);
340 }
341 }
342 }
343
344 private void receiveTransportInfo(
345 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
346 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
347 contentMap.contents.entrySet();
348 if (this.state == State.SESSION_ACCEPTED) {
349 // zero candidates + modified credentials are an ICE restart offer
350 if (checkForIceRestart(jinglePacket, contentMap)) {
351 return;
352 }
353 respondOk(jinglePacket);
354 try {
355 processCandidates(candidates);
356 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
357 Log.w(
358 Config.LOGTAG,
359 id.account.getJid().asBareJid()
360 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
361 }
362 } else {
363 respondOk(jinglePacket);
364 pendingIceCandidates.addAll(candidates);
365 }
366 }
367
368 private void receiveContentAdd(final JinglePacket jinglePacket) {
369 final RtpContentMap modification;
370 try {
371 modification = RtpContentMap.of(jinglePacket);
372 modification.requireContentDescriptions();
373 } catch (final RuntimeException e) {
374 Log.d(
375 Config.LOGTAG,
376 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
377 Throwables.getRootCause(e));
378 respondOk(jinglePacket);
379 webRTCWrapper.close();
380 sendSessionTerminate(Reason.of(e), e.getMessage());
381 return;
382 }
383 if (isInState(State.SESSION_ACCEPTED)) {
384 receiveContentAdd(jinglePacket, modification);
385 } else {
386 terminateWithOutOfOrder(jinglePacket);
387 }
388 }
389
390 private void receiveContentAdd(
391 final JinglePacket jinglePacket, final RtpContentMap modification) {
392 final RtpContentMap remote = getRemoteContentMap();
393 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
394 respondOk(jinglePacket);
395 this.webRTCWrapper.close();
396 sendSessionTerminate(
397 Reason.FAILED_APPLICATION,
398 String.format(
399 "contents with names %s already exists",
400 Joiner.on(", ").join(modification.getNames())));
401 return;
402 }
403 final ContentAddition contentAddition =
404 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
405
406 final RtpContentMap outgoing = this.outgoingContentAdd;
407 final Set<ContentAddition.Summary> outgoingContentAddSummary =
408 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
409
410 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
411 if (isInitiator()) {
412 Log.d(
413 Config.LOGTAG,
414 id.getAccount().getJid().asBareJid()
415 + ": respond with tie break to matching content-add offer");
416 respondWithTieBreak(jinglePacket);
417 } else {
418 Log.d(
419 Config.LOGTAG,
420 id.getAccount().getJid().asBareJid()
421 + ": automatically accept matching content-add offer");
422 acceptContentAdd(contentAddition.summary, modification);
423 }
424 return;
425 }
426
427 // once we can display multiple video tracks we can be more loose with this condition
428 // theoretically it should also be fine to automatically accept audio only contents
429 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
430 Log.d(
431 Config.LOGTAG,
432 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
433 this.incomingContentAdd = modification;
434 respondOk(jinglePacket);
435 updateEndUserState();
436 } else {
437 respondOk(jinglePacket);
438 // TODO do we want to add a reason?
439 rejectContentAdd(modification);
440 }
441 }
442
443 private void receiveContentAccept(final JinglePacket jinglePacket) {
444 final RtpContentMap receivedContentAccept;
445 try {
446 receivedContentAccept = RtpContentMap.of(jinglePacket);
447 receivedContentAccept.requireContentDescriptions();
448 } catch (final RuntimeException e) {
449 Log.d(
450 Config.LOGTAG,
451 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
452 Throwables.getRootCause(e));
453 respondOk(jinglePacket);
454 webRTCWrapper.close();
455 sendSessionTerminate(Reason.of(e), e.getMessage());
456 return;
457 }
458
459 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
460 if (outgoingContentAdd == null) {
461 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
462 terminateWithOutOfOrder(jinglePacket);
463 return;
464 }
465 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
466 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
467 this.outgoingContentAdd = null;
468 respondOk(jinglePacket);
469 receiveContentAccept(receivedContentAccept);
470 } else {
471 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
472 terminateWithOutOfOrder(jinglePacket);
473 }
474 }
475
476 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
477 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
478 final RtpContentMap modifiedContentMap =
479 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
480
481 setRemoteContentMap(modifiedContentMap);
482
483 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
484
485 final org.webrtc.SessionDescription sdp =
486 new org.webrtc.SessionDescription(
487 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
488
489 try {
490 this.webRTCWrapper.setRemoteDescription(sdp).get();
491 } catch (final Exception e) {
492 final Throwable cause = Throwables.getRootCause(e);
493 Log.d(
494 Config.LOGTAG,
495 id.getAccount().getJid().asBareJid()
496 + ": unable to set remote description after receiving content-accept",
497 cause);
498 webRTCWrapper.close();
499 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
500 return;
501 }
502 updateEndUserState();
503 Log.d(
504 Config.LOGTAG,
505 id.getAccount().getJid().asBareJid()
506 + ": remote has accepted content-add "
507 + ContentAddition.summary(receivedContentAccept));
508 }
509
510 private void receiveContentReject(final JinglePacket jinglePacket) {
511 final RtpContentMap receivedContentReject;
512 try {
513 receivedContentReject = RtpContentMap.of(jinglePacket);
514 } catch (final RuntimeException e) {
515 Log.d(
516 Config.LOGTAG,
517 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
518 Throwables.getRootCause(e));
519 respondOk(jinglePacket);
520 this.webRTCWrapper.close();
521 sendSessionTerminate(Reason.of(e), e.getMessage());
522 return;
523 }
524
525 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
526 if (outgoingContentAdd == null) {
527 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
528 terminateWithOutOfOrder(jinglePacket);
529 return;
530 }
531 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
532 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
533 this.outgoingContentAdd = null;
534 respondOk(jinglePacket);
535 Log.d(Config.LOGTAG,jinglePacket.toString());
536 receiveContentReject(ourSummary);
537 } else {
538 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
539 terminateWithOutOfOrder(jinglePacket);
540 }
541 }
542
543 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
544 try {
545 this.webRTCWrapper.removeTrack(Media.VIDEO);
546 final RtpContentMap localContentMap = customRollback();
547 modifyLocalContentMap(localContentMap);
548 } catch (final Exception e) {
549 final Throwable cause = Throwables.getRootCause(e);
550 Log.d(
551 Config.LOGTAG,
552 id.getAccount().getJid().asBareJid()
553 + ": unable to rollback local description after receiving content-reject",
554 cause);
555 webRTCWrapper.close();
556 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
557 return;
558 }
559 Log.d(
560 Config.LOGTAG,
561 id.getAccount().getJid().asBareJid()
562 + ": remote has rejected our content-add "
563 + summary);
564 }
565
566 private void receiveContentRemove(final JinglePacket jinglePacket) {
567 final RtpContentMap receivedContentRemove;
568 try {
569 receivedContentRemove = RtpContentMap.of(jinglePacket);
570 receivedContentRemove.requireContentDescriptions();
571 } catch (final RuntimeException e) {
572 Log.d(
573 Config.LOGTAG,
574 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
575 Throwables.getRootCause(e));
576 respondOk(jinglePacket);
577 this.webRTCWrapper.close();
578 sendSessionTerminate(Reason.of(e), e.getMessage());
579 return;
580 }
581 respondOk(jinglePacket);
582 receiveContentRemove(receivedContentRemove);
583 }
584
585 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
586 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
587 final Set<ContentAddition.Summary> contentAddSummary =
588 incomingContentAdd == null
589 ? Collections.emptySet()
590 : ContentAddition.summary(incomingContentAdd);
591 final Set<ContentAddition.Summary> removeSummary =
592 ContentAddition.summary(receivedContentRemove);
593 if (contentAddSummary.equals(removeSummary)) {
594 this.incomingContentAdd = null;
595 updateEndUserState();
596 } else {
597 webRTCWrapper.close();
598 sendSessionTerminate(
599 Reason.FAILED_APPLICATION,
600 String.format(
601 "%s only supports %s as a means to retract a not yet accepted %s",
602 BuildConfig.APP_NAME,
603 JinglePacket.Action.CONTENT_REMOVE,
604 JinglePacket.Action.CONTENT_ACCEPT));
605 }
606 }
607
608 public synchronized void retractContentAdd() {
609 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
610 if (outgoingContentAdd == null) {
611 throw new IllegalStateException("Not outgoing content add");
612 }
613 try {
614 webRTCWrapper.removeTrack(Media.VIDEO);
615 final RtpContentMap localContentMap = customRollback();
616 modifyLocalContentMap(localContentMap);
617 } catch (final Exception e) {
618 final Throwable cause = Throwables.getRootCause(e);
619 Log.d(
620 Config.LOGTAG,
621 id.getAccount().getJid().asBareJid()
622 + ": unable to rollback local description after trying to retract content-add",
623 cause);
624 webRTCWrapper.close();
625 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
626 return;
627 }
628 this.outgoingContentAdd = null;
629 final JinglePacket retract =
630 outgoingContentAdd
631 .toStub()
632 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
633 this.send(retract);
634 Log.d(
635 Config.LOGTAG,
636 id.getAccount().getJid()
637 + ": retract content-add "
638 + ContentAddition.summary(outgoingContentAdd));
639 }
640
641 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
642 final SessionDescription sdp = setLocalSessionDescription();
643 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
644 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
645 this.webRTCWrapper
646 .setRemoteDescription(
647 new org.webrtc.SessionDescription(
648 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
649 .get();
650 return localRtpContentMap;
651 }
652
653 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
654 final RtpContentMap currentRemote = getRemoteContentMap();
655 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
656 if (diff.isEmpty()) {
657 throw new IllegalStateException(
658 "Unexpected rollback condition. No difference between local and remote");
659 }
660 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
661 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
662 final RtpContentMap nextRemote =
663 currentRemote.addContent(
664 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
665 return SessionDescription.of(nextRemote, !isInitiator());
666 }
667 throw new IllegalStateException(
668 "Unexpected rollback condition. Senders were not uniformly none");
669 }
670
671 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
672 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
673 if (incomingContentAdd == null) {
674 throw new IllegalStateException("No incoming content add");
675 }
676
677 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
678 this.incomingContentAdd = null;
679 acceptContentAdd(contentAddition, incomingContentAdd);
680 } else {
681 throw new IllegalStateException("Accepted content add does not match pending content-add");
682 }
683 }
684
685 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
686 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
687 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
688 this.setRemoteContentMap(modifiedContentMap);
689
690 final SessionDescription offer;
691 try {
692 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
693 } catch (final IllegalArgumentException | NullPointerException e) {
694 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
695 webRTCWrapper.close();
696 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
697 return;
698 }
699 this.incomingContentAdd = null;
700 acceptContentAdd(contentAddition, offer);
701 }
702
703 private void acceptContentAdd(
704 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
705 final org.webrtc.SessionDescription sdp =
706 new org.webrtc.SessionDescription(
707 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
708 try {
709 this.webRTCWrapper.setRemoteDescription(sdp).get();
710
711 // TODO add tracks for 'media' where contentAddition.senders matches
712
713 // TODO if senders.sending(isInitiator())
714
715 this.webRTCWrapper.addTrack(Media.VIDEO);
716
717 // TODO add additional transceivers for recv only cases
718
719 final SessionDescription answer = setLocalSessionDescription();
720 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
721
722 final RtpContentMap contentAcceptMap =
723 rtpContentMap.toContentModification(
724 Collections2.transform(contentAddition, ca -> ca.name));
725 Log.d(
726 Config.LOGTAG,
727 id.getAccount().getJid().asBareJid()
728 + ": sending content-accept "
729 + ContentAddition.summary(contentAcceptMap));
730 modifyLocalContentMap(rtpContentMap);
731 sendContentAccept(contentAcceptMap);
732 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
733 } catch (final Exception e) {
734 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
735 webRTCWrapper.close();
736 sendSessionTerminate(Reason.FAILED_APPLICATION);
737 }
738 }
739
740 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
741 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
742 send(jinglePacket);
743 }
744
745 public synchronized void rejectContentAdd() {
746 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
747 if (incomingContentAdd == null) {
748 throw new IllegalStateException("No incoming content add");
749 }
750 this.incomingContentAdd = null;
751 updateEndUserState();
752 rejectContentAdd(incomingContentAdd);
753 }
754
755 private void rejectContentAdd(final RtpContentMap contentMap) {
756 final JinglePacket jinglePacket =
757 contentMap
758 .toStub()
759 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
760 Log.d(
761 Config.LOGTAG,
762 id.getAccount().getJid().asBareJid()
763 + ": rejecting content "
764 + ContentAddition.summary(contentMap));
765 send(jinglePacket);
766 }
767
768 private boolean checkForIceRestart(
769 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
770 final RtpContentMap existing = getRemoteContentMap();
771 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
772 final IceUdpTransportInfo.Credentials newCredentials;
773 try {
774 existingCredentials = existing.getCredentials();
775 newCredentials = rtpContentMap.getDistinctCredentials();
776 } catch (final IllegalStateException e) {
777 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
778 return false;
779 }
780 if (existingCredentials.contains(newCredentials)) {
781 return false;
782 }
783 // TODO an alternative approach is to check if we already got an iq result to our
784 // ICE-restart
785 // and if that's the case we are seeing an answer.
786 // This might be more spec compliant but also more error prone potentially
787 final boolean isOffer = rtpContentMap.emptyCandidates();
788 final RtpContentMap restartContentMap;
789 try {
790 if (isOffer) {
791 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
792 restartContentMap =
793 existing.modifiedCredentials(
794 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
795 } else {
796 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
797 Log.d(
798 Config.LOGTAG,
799 "received confirmation of ICE restart"
800 + newCredentials
801 + " peer_setup="
802 + setup);
803 // DTLS setup attribute needs to be rewritten to reflect current peer state
804 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
805 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
806 }
807 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
808 return isOffer;
809 } else {
810 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
811 respondWithTieBreak(jinglePacket);
812 return true;
813 }
814 } catch (final Exception exception) {
815 respondOk(jinglePacket);
816 final Throwable rootCause = Throwables.getRootCause(exception);
817 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
818 // If this happens a termination is already in progress
819 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
820 return true;
821 }
822 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
823 webRTCWrapper.close();
824 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
825 return true;
826 }
827 }
828
829 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
830 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
831 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
832 throw new IllegalStateException("Invalid peer setup");
833 }
834 return peerSetup;
835 }
836
837 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
838 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
839 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
840 }
841 this.peerDtlsSetup = setup;
842 }
843
844 private boolean applyIceRestart(
845 final JinglePacket jinglePacket,
846 final RtpContentMap restartContentMap,
847 final boolean isOffer)
848 throws ExecutionException, InterruptedException {
849 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
850 final org.webrtc.SessionDescription.Type type =
851 isOffer
852 ? org.webrtc.SessionDescription.Type.OFFER
853 : org.webrtc.SessionDescription.Type.ANSWER;
854 org.webrtc.SessionDescription sdp =
855 new org.webrtc.SessionDescription(type, sessionDescription.toString());
856 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
857 if (isInitiator()) {
858 // We ignore the offer and respond with tie-break. This will clause the responder
859 // not to apply the content map
860 return false;
861 }
862 }
863 webRTCWrapper.setRemoteDescription(sdp).get();
864 setRemoteContentMap(restartContentMap);
865 if (isOffer) {
866 final SessionDescription localSessionDescription = setLocalSessionDescription();
867 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
868 // We need to respond OK before sending any candidates
869 respondOk(jinglePacket);
870 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
871 } else {
872 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
873 }
874 return true;
875 }
876
877 private void processCandidates(
878 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
879 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
880 processCandidate(content);
881 }
882 }
883
884 private void processCandidate(
885 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
886 final RtpContentMap rtpContentMap = getRemoteContentMap();
887 final List<String> indices = toIdentificationTags(rtpContentMap);
888 final String sdpMid = content.getKey(); // aka content name
889 final IceUdpTransportInfo transport = content.getValue().transport;
890 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
891
892 // TODO check that credentials remained the same
893
894 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
895 final String sdp;
896 try {
897 sdp = candidate.toSdpAttribute(credentials.ufrag);
898 } catch (final IllegalArgumentException e) {
899 Log.d(
900 Config.LOGTAG,
901 id.account.getJid().asBareJid()
902 + ": ignoring invalid ICE candidate "
903 + e.getMessage());
904 continue;
905 }
906 final int mLineIndex = indices.indexOf(sdpMid);
907 if (mLineIndex < 0) {
908 Log.w(
909 Config.LOGTAG,
910 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
911 }
912 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
913 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
914 this.webRTCWrapper.addIceCandidate(iceCandidate);
915 }
916 }
917
918 private RtpContentMap getRemoteContentMap() {
919 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
920 }
921
922 private RtpContentMap getLocalContentMap() {
923 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
924 }
925
926 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
927 final Group originalGroup = rtpContentMap.group;
928 final List<String> identificationTags =
929 originalGroup == null
930 ? rtpContentMap.getNames()
931 : originalGroup.getIdentificationTags();
932 if (identificationTags.size() == 0) {
933 Log.w(
934 Config.LOGTAG,
935 id.account.getJid().asBareJid()
936 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
937 }
938 return identificationTags;
939 }
940
941 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
942 final JinglePacket jinglePacket, final boolean expectVerification) {
943 final RtpContentMap receivedContentMap;
944 try {
945 receivedContentMap = RtpContentMap.of(jinglePacket);
946 } catch (final Exception e) {
947 return Futures.immediateFailedFuture(e);
948 }
949 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
950 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
951 id.account
952 .getAxolotlService()
953 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
954 return Futures.transform(
955 future,
956 omemoVerifiedPayload -> {
957 // TODO test if an exception here triggers a correct abort
958 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
959 Log.d(
960 Config.LOGTAG,
961 id.account.getJid().asBareJid()
962 + ": received verifiable DTLS fingerprint via "
963 + omemoVerification);
964 return omemoVerifiedPayload.getPayload();
965 },
966 MoreExecutors.directExecutor());
967 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
968 return Futures.immediateFailedFuture(
969 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
970 } else {
971 return Futures.immediateFuture(receivedContentMap);
972 }
973 }
974
975 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
976 if (isInitiator()) {
977 Log.d(
978 Config.LOGTAG,
979 String.format(
980 "%s: received session-initiate even though we were initiating",
981 id.account.getJid().asBareJid()));
982 if (isTerminated()) {
983 Log.d(
984 Config.LOGTAG,
985 String.format(
986 "%s: got a reason to terminate with out-of-order. but already in state %s",
987 id.account.getJid().asBareJid(), getState()));
988 respondWithOutOfOrder(jinglePacket);
989 } else {
990 terminateWithOutOfOrder(jinglePacket);
991 }
992 return;
993 }
994 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
995 Futures.addCallback(
996 future,
997 new FutureCallback<RtpContentMap>() {
998 @Override
999 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1000 receiveSessionInitiate(jinglePacket, rtpContentMap);
1001 }
1002
1003 @Override
1004 public void onFailure(@NonNull final Throwable throwable) {
1005 respondOk(jinglePacket);
1006 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1007 }
1008 },
1009 MoreExecutors.directExecutor());
1010 }
1011
1012 private void receiveSessionInitiate(
1013 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1014 try {
1015 contentMap.requireContentDescriptions();
1016 contentMap.requireDTLSFingerprint(true);
1017 } catch (final RuntimeException e) {
1018 Log.d(
1019 Config.LOGTAG,
1020 id.account.getJid().asBareJid() + ": improperly formatted contents",
1021 Throwables.getRootCause(e));
1022 respondOk(jinglePacket);
1023 sendSessionTerminate(Reason.of(e), e.getMessage());
1024 return;
1025 }
1026 Log.d(
1027 Config.LOGTAG,
1028 "processing session-init with " + contentMap.contents.size() + " contents");
1029 final State target;
1030 if (this.state == State.PROCEED) {
1031 Preconditions.checkState(
1032 proposedMedia != null && proposedMedia.size() > 0,
1033 "proposed media must be set when processing pre-approved session-initiate");
1034 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1035 sendSessionTerminate(
1036 Reason.SECURITY_ERROR,
1037 String.format(
1038 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1039 this.proposedMedia, contentMap.getMedia()));
1040 return;
1041 }
1042 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1043 } else {
1044 target = State.SESSION_INITIALIZED;
1045 }
1046 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1047 respondOk(jinglePacket);
1048 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1049 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1050 Log.d(
1051 Config.LOGTAG,
1052 id.account.getJid().asBareJid()
1053 + ": automatically accepting session-initiate");
1054 sendSessionAccept();
1055 } else {
1056 Log.d(
1057 Config.LOGTAG,
1058 id.account.getJid().asBareJid()
1059 + ": received not pre-approved session-initiate. start ringing");
1060 startRinging();
1061 }
1062 } else {
1063 Log.d(
1064 Config.LOGTAG,
1065 String.format(
1066 "%s: received session-initiate while in state %s",
1067 id.account.getJid().asBareJid(), state));
1068 terminateWithOutOfOrder(jinglePacket);
1069 }
1070 }
1071
1072 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1073 if (!isInitiator()) {
1074 Log.d(
1075 Config.LOGTAG,
1076 String.format(
1077 "%s: received session-accept even though we were responding",
1078 id.account.getJid().asBareJid()));
1079 terminateWithOutOfOrder(jinglePacket);
1080 return;
1081 }
1082 final ListenableFuture<RtpContentMap> future =
1083 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1084 Futures.addCallback(
1085 future,
1086 new FutureCallback<RtpContentMap>() {
1087 @Override
1088 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1089 receiveSessionAccept(jinglePacket, rtpContentMap);
1090 }
1091
1092 @Override
1093 public void onFailure(@NonNull final Throwable throwable) {
1094 respondOk(jinglePacket);
1095 Log.d(
1096 Config.LOGTAG,
1097 id.account.getJid().asBareJid()
1098 + ": improperly formatted contents in session-accept",
1099 throwable);
1100 webRTCWrapper.close();
1101 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1102 }
1103 },
1104 MoreExecutors.directExecutor());
1105 }
1106
1107 private void receiveSessionAccept(
1108 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1109 try {
1110 contentMap.requireContentDescriptions();
1111 contentMap.requireDTLSFingerprint();
1112 } catch (final RuntimeException e) {
1113 respondOk(jinglePacket);
1114 Log.d(
1115 Config.LOGTAG,
1116 id.account.getJid().asBareJid()
1117 + ": improperly formatted contents in session-accept",
1118 e);
1119 webRTCWrapper.close();
1120 sendSessionTerminate(Reason.of(e), e.getMessage());
1121 return;
1122 }
1123 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1124 if (!initiatorMedia.equals(contentMap.getMedia())) {
1125 sendSessionTerminate(
1126 Reason.SECURITY_ERROR,
1127 String.format(
1128 "Your session-included included media %s but our session-initiate was %s",
1129 this.proposedMedia, contentMap.getMedia()));
1130 return;
1131 }
1132 Log.d(
1133 Config.LOGTAG,
1134 "processing session-accept with " + contentMap.contents.size() + " contents");
1135 if (transition(State.SESSION_ACCEPTED)) {
1136 respondOk(jinglePacket);
1137 receiveSessionAccept(contentMap);
1138 } else {
1139 Log.d(
1140 Config.LOGTAG,
1141 String.format(
1142 "%s: received session-accept while in state %s",
1143 id.account.getJid().asBareJid(), state));
1144 respondOk(jinglePacket);
1145 }
1146 }
1147
1148 private void receiveSessionAccept(final RtpContentMap contentMap) {
1149 this.responderRtpContentMap = contentMap;
1150 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1151 final SessionDescription sessionDescription;
1152 try {
1153 sessionDescription = SessionDescription.of(contentMap, false);
1154 } catch (final IllegalArgumentException | NullPointerException e) {
1155 Log.d(
1156 Config.LOGTAG,
1157 id.account.getJid().asBareJid()
1158 + ": unable convert offer from session-accept to SDP",
1159 e);
1160 webRTCWrapper.close();
1161 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1162 return;
1163 }
1164 final org.webrtc.SessionDescription answer =
1165 new org.webrtc.SessionDescription(
1166 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1167 try {
1168 this.webRTCWrapper.setRemoteDescription(answer).get();
1169 } catch (final Exception e) {
1170 Log.d(
1171 Config.LOGTAG,
1172 id.account.getJid().asBareJid()
1173 + ": unable to set remote description after receiving session-accept",
1174 Throwables.getRootCause(e));
1175 webRTCWrapper.close();
1176 sendSessionTerminate(
1177 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1178 return;
1179 }
1180 processCandidates(contentMap.contents.entrySet());
1181 }
1182
1183 private void sendSessionAccept() {
1184 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1185 if (rtpContentMap == null) {
1186 throw new IllegalStateException("initiator RTP Content Map has not been set");
1187 }
1188 final SessionDescription offer;
1189 try {
1190 offer = SessionDescription.of(rtpContentMap, true);
1191 } catch (final IllegalArgumentException | NullPointerException e) {
1192 Log.d(
1193 Config.LOGTAG,
1194 id.account.getJid().asBareJid()
1195 + ": unable convert offer from session-initiate to SDP",
1196 e);
1197 webRTCWrapper.close();
1198 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1199 return;
1200 }
1201 sendSessionAccept(rtpContentMap.getMedia(), offer);
1202 }
1203
1204 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1205 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1206 }
1207
1208 private synchronized void sendSessionAccept(
1209 final Set<Media> media,
1210 final SessionDescription offer,
1211 final List<PeerConnection.IceServer> iceServers) {
1212 if (isTerminated()) {
1213 Log.w(
1214 Config.LOGTAG,
1215 id.account.getJid().asBareJid()
1216 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1217 return;
1218 }
1219 try {
1220 setupWebRTC(media, iceServers);
1221 } catch (final WebRTCWrapper.InitializationException e) {
1222 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1223 webRTCWrapper.close();
1224 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1225 return;
1226 }
1227 final org.webrtc.SessionDescription sdp =
1228 new org.webrtc.SessionDescription(
1229 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1230 try {
1231 this.webRTCWrapper.setRemoteDescription(sdp).get();
1232 addIceCandidatesFromBlackLog();
1233 org.webrtc.SessionDescription webRTCSessionDescription =
1234 this.webRTCWrapper.setLocalDescription().get();
1235 prepareSessionAccept(webRTCSessionDescription);
1236 } catch (final Exception e) {
1237 failureToAcceptSession(e);
1238 }
1239 }
1240
1241 private void failureToAcceptSession(final Throwable throwable) {
1242 if (isTerminated()) {
1243 return;
1244 }
1245 final Throwable rootCause = Throwables.getRootCause(throwable);
1246 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1247 webRTCWrapper.close();
1248 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1249 }
1250
1251 private void addIceCandidatesFromBlackLog() {
1252 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1253 while ((foo = this.pendingIceCandidates.poll()) != null) {
1254 processCandidate(foo);
1255 Log.d(
1256 Config.LOGTAG,
1257 id.account.getJid().asBareJid() + ": added candidate from back log");
1258 }
1259 }
1260
1261 private void prepareSessionAccept(
1262 final org.webrtc.SessionDescription webRTCSessionDescription) {
1263 final SessionDescription sessionDescription =
1264 SessionDescription.parse(webRTCSessionDescription.description);
1265 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1266 this.responderRtpContentMap = respondingRtpContentMap;
1267 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1268 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1269 prepareOutgoingContentMap(respondingRtpContentMap);
1270 Futures.addCallback(
1271 outgoingContentMapFuture,
1272 new FutureCallback<RtpContentMap>() {
1273 @Override
1274 public void onSuccess(final RtpContentMap outgoingContentMap) {
1275 sendSessionAccept(outgoingContentMap);
1276 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1277 }
1278
1279 @Override
1280 public void onFailure(@NonNull Throwable throwable) {
1281 failureToAcceptSession(throwable);
1282 }
1283 },
1284 MoreExecutors.directExecutor());
1285 }
1286
1287 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1288 if (isTerminated()) {
1289 Log.w(
1290 Config.LOGTAG,
1291 id.account.getJid().asBareJid()
1292 + ": preparing session accept was too slow. already terminated. nothing to do.");
1293 return;
1294 }
1295 transitionOrThrow(State.SESSION_ACCEPTED);
1296 final JinglePacket sessionAccept =
1297 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1298 send(sessionAccept);
1299 }
1300
1301 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1302 final RtpContentMap rtpContentMap) {
1303 if (this.omemoVerification.hasDeviceId()) {
1304 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1305 verifiedPayloadFuture =
1306 id.account
1307 .getAxolotlService()
1308 .encrypt(
1309 rtpContentMap,
1310 id.with,
1311 omemoVerification.getDeviceId());
1312 return Futures.transform(
1313 verifiedPayloadFuture,
1314 verifiedPayload -> {
1315 omemoVerification.setOrEnsureEqual(verifiedPayload);
1316 return verifiedPayload.getPayload();
1317 },
1318 MoreExecutors.directExecutor());
1319 } else {
1320 return Futures.immediateFuture(rtpContentMap);
1321 }
1322 }
1323
1324 synchronized void deliveryMessage(
1325 final Jid from,
1326 final Element message,
1327 final String serverMessageId,
1328 final long timestamp) {
1329 Log.d(
1330 Config.LOGTAG,
1331 id.account.getJid().asBareJid()
1332 + ": delivered message to JingleRtpConnection "
1333 + message);
1334 switch (message.getName()) {
1335 case "propose":
1336 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1337 break;
1338 case "proceed":
1339 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1340 break;
1341 case "retract":
1342 receiveRetract(from, serverMessageId, timestamp);
1343 break;
1344 case "reject":
1345 receiveReject(from, serverMessageId, timestamp);
1346 break;
1347 case "accept":
1348 receiveAccept(from, serverMessageId, timestamp);
1349 break;
1350 default:
1351 break;
1352 }
1353 }
1354
1355 void deliverFailedProceed(final String message) {
1356 Log.d(
1357 Config.LOGTAG,
1358 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1359 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1360 webRTCWrapper.close();
1361 Log.d(
1362 Config.LOGTAG,
1363 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1364 this.finish();
1365 }
1366 }
1367
1368 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1369 final boolean originatedFromMyself =
1370 from.asBareJid().equals(id.account.getJid().asBareJid());
1371 if (originatedFromMyself) {
1372 if (transition(State.ACCEPTED)) {
1373 acceptedOnOtherDevice(serverMsgId, timestamp);
1374 } else {
1375 Log.d(
1376 Config.LOGTAG,
1377 id.account.getJid().asBareJid()
1378 + ": unable to transition to accept because already in state="
1379 + this.state);
1380 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1381 }
1382 } else {
1383 Log.d(
1384 Config.LOGTAG,
1385 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1386 }
1387 }
1388
1389 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1390 if (serverMsgId != null) {
1391 this.message.setServerMsgId(serverMsgId);
1392 }
1393 this.message.setTime(timestamp);
1394 this.message.setCarbon(true); // indicate that call was accepted on other device
1395 this.writeLogMessageSuccess(0);
1396 this.xmppConnectionService
1397 .getNotificationService()
1398 .cancelIncomingCallNotification();
1399 this.finish();
1400 }
1401
1402 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1403 final boolean originatedFromMyself =
1404 from.asBareJid().equals(id.account.getJid().asBareJid());
1405 // reject from another one of my clients
1406 if (originatedFromMyself) {
1407 receiveRejectFromMyself(serverMsgId, timestamp);
1408 } else if (isInitiator()) {
1409 if (from.equals(id.with)) {
1410 receiveRejectFromResponder();
1411 } else {
1412 Log.d(
1413 Config.LOGTAG,
1414 id.account.getJid()
1415 + ": ignoring reject from "
1416 + from
1417 + " for session with "
1418 + id.with);
1419 }
1420 } else {
1421 Log.d(
1422 Config.LOGTAG,
1423 id.account.getJid()
1424 + ": ignoring reject from "
1425 + from
1426 + " for session with "
1427 + id.with);
1428 }
1429 }
1430
1431 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1432 if (transition(State.REJECTED)) {
1433 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1434 this.finish();
1435 if (serverMsgId != null) {
1436 this.message.setServerMsgId(serverMsgId);
1437 }
1438 this.message.setTime(timestamp);
1439 this.message.setCarbon(true); // indicate that call was rejected on other device
1440 writeLogMessageMissed();
1441 } else {
1442 Log.d(
1443 Config.LOGTAG,
1444 "not able to transition into REJECTED because already in " + this.state);
1445 }
1446 }
1447
1448 private void receiveRejectFromResponder() {
1449 if (isInState(State.PROCEED)) {
1450 Log.d(
1451 Config.LOGTAG,
1452 id.account.getJid()
1453 + ": received reject while still in proceed. callee reconsidered");
1454 closeTransitionLogFinish(State.REJECTED_RACED);
1455 return;
1456 }
1457 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1458 Log.d(
1459 Config.LOGTAG,
1460 id.account.getJid()
1461 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1462 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1463 return;
1464 }
1465 Log.d(
1466 Config.LOGTAG,
1467 id.account.getJid()
1468 + ": ignoring reject from responder because already in state "
1469 + this.state);
1470 }
1471
1472 private void receivePropose(
1473 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1474 final boolean originatedFromMyself =
1475 from.asBareJid().equals(id.account.getJid().asBareJid());
1476 if (originatedFromMyself) {
1477 Log.d(
1478 Config.LOGTAG,
1479 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1480 } else if (transition(
1481 State.PROPOSED,
1482 () -> {
1483 final Collection<RtpDescription> descriptions =
1484 Collections2.transform(
1485 Collections2.filter(
1486 propose.getDescriptions(),
1487 d -> d instanceof RtpDescription),
1488 input -> (RtpDescription) input);
1489 final Collection<Media> media =
1490 Collections2.transform(descriptions, RtpDescription::getMedia);
1491 Preconditions.checkState(
1492 !media.contains(Media.UNKNOWN),
1493 "RTP descriptions contain unknown media");
1494 Log.d(
1495 Config.LOGTAG,
1496 id.account.getJid().asBareJid()
1497 + ": received session proposal from "
1498 + from
1499 + " for "
1500 + media);
1501 this.proposedMedia = Sets.newHashSet(media);
1502 })) {
1503 if (serverMsgId != null) {
1504 this.message.setServerMsgId(serverMsgId);
1505 }
1506 this.message.setTime(timestamp);
1507 startRinging();
1508 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1509 sendJingleMessage("ringing");
1510 }
1511 } else {
1512 Log.d(
1513 Config.LOGTAG,
1514 id.account.getJid()
1515 + ": ignoring session proposal because already in "
1516 + state);
1517 }
1518 }
1519
1520 private void startRinging() {
1521 Log.d(
1522 Config.LOGTAG,
1523 id.account.getJid().asBareJid()
1524 + ": received call from "
1525 + id.with
1526 + ". start ringing");
1527 ringingTimeoutFuture =
1528 jingleConnectionManager.schedule(
1529 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1530 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1531 }
1532
1533 private synchronized void ringingTimeout() {
1534 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1535 switch (this.state) {
1536 case PROPOSED:
1537 message.markUnread();
1538 rejectCallFromProposed();
1539 break;
1540 case SESSION_INITIALIZED:
1541 message.markUnread();
1542 rejectCallFromSessionInitiate();
1543 break;
1544 }
1545 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1546 }
1547
1548 private void cancelRingingTimeout() {
1549 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1550 if (future != null && !future.isCancelled()) {
1551 future.cancel(false);
1552 }
1553 }
1554
1555 private void receiveProceed(
1556 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1557 final Set<Media> media =
1558 Preconditions.checkNotNull(
1559 this.proposedMedia, "Proposed media has to be set before handling proceed");
1560 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1561 if (from.equals(id.with)) {
1562 if (isInitiator()) {
1563 if (transition(State.PROCEED)) {
1564 if (serverMsgId != null) {
1565 this.message.setServerMsgId(serverMsgId);
1566 }
1567 this.message.setTime(timestamp);
1568 final Integer remoteDeviceId = proceed.getDeviceId();
1569 if (isOmemoEnabled()) {
1570 this.omemoVerification.setDeviceId(remoteDeviceId);
1571 } else {
1572 if (remoteDeviceId != null) {
1573 Log.d(
1574 Config.LOGTAG,
1575 id.account.getJid().asBareJid()
1576 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1577 }
1578 this.omemoVerification.setDeviceId(null);
1579 }
1580 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1581 } else {
1582 Log.d(
1583 Config.LOGTAG,
1584 String.format(
1585 "%s: ignoring proceed because already in %s",
1586 id.account.getJid().asBareJid(), this.state));
1587 }
1588 } else {
1589 Log.d(
1590 Config.LOGTAG,
1591 String.format(
1592 "%s: ignoring proceed because we were not initializing",
1593 id.account.getJid().asBareJid()));
1594 }
1595 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1596 if (transition(State.ACCEPTED)) {
1597 Log.d(
1598 Config.LOGTAG,
1599 id.account.getJid().asBareJid()
1600 + ": moved session with "
1601 + id.with
1602 + " into state accepted after received carbon copied proceed");
1603 acceptedOnOtherDevice(serverMsgId, timestamp);
1604 }
1605 } else {
1606 Log.d(
1607 Config.LOGTAG,
1608 String.format(
1609 "%s: ignoring proceed from %s. was expected from %s",
1610 id.account.getJid().asBareJid(), from, id.with));
1611 }
1612 }
1613
1614 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1615 if (from.equals(id.with)) {
1616 final State target =
1617 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1618 if (transition(target)) {
1619 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1620 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1621 Log.d(
1622 Config.LOGTAG,
1623 id.account.getJid().asBareJid()
1624 + ": session with "
1625 + id.with
1626 + " has been retracted (serverMsgId="
1627 + serverMsgId
1628 + ")");
1629 if (serverMsgId != null) {
1630 this.message.setServerMsgId(serverMsgId);
1631 }
1632 this.message.setTime(timestamp);
1633 if (target == State.RETRACTED) {
1634 this.message.markUnread();
1635 }
1636 writeLogMessageMissed();
1637 finish();
1638 } else {
1639 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1640 }
1641 } else {
1642 // TODO parse retract from self
1643 Log.d(
1644 Config.LOGTAG,
1645 id.account.getJid().asBareJid()
1646 + ": received retract from "
1647 + from
1648 + ". expected retract from"
1649 + id.with
1650 + ". ignoring");
1651 }
1652 }
1653
1654 public void sendSessionInitiate() {
1655 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1656 }
1657
1658 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1659 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1660 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1661 }
1662
1663 private synchronized void sendSessionInitiate(
1664 final Set<Media> media,
1665 final State targetState,
1666 final List<PeerConnection.IceServer> iceServers) {
1667 if (isTerminated()) {
1668 Log.w(
1669 Config.LOGTAG,
1670 id.account.getJid().asBareJid()
1671 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1672 return;
1673 }
1674 try {
1675 setupWebRTC(media, iceServers);
1676 } catch (final WebRTCWrapper.InitializationException e) {
1677 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1678 webRTCWrapper.close();
1679 sendRetract(Reason.ofThrowable(e));
1680 return;
1681 }
1682 try {
1683 org.webrtc.SessionDescription webRTCSessionDescription =
1684 this.webRTCWrapper.setLocalDescription().get();
1685 prepareSessionInitiate(webRTCSessionDescription, targetState);
1686 } catch (final Exception e) {
1687 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1688 // exceptions
1689 failureToInitiateSession(e, targetState);
1690 }
1691 }
1692
1693 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1694 if (isTerminated()) {
1695 return;
1696 }
1697 Log.d(
1698 Config.LOGTAG,
1699 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1700 Throwables.getRootCause(throwable));
1701 webRTCWrapper.close();
1702 final Reason reason = Reason.ofThrowable(throwable);
1703 if (isInState(targetState)) {
1704 sendSessionTerminate(reason, throwable.getMessage());
1705 } else {
1706 sendRetract(reason);
1707 }
1708 }
1709
1710 private void sendRetract(final Reason reason) {
1711 // TODO embed reason into retract
1712 sendJingleMessage("retract", id.with.asBareJid());
1713 transitionOrThrow(reasonToState(reason));
1714 this.finish();
1715 }
1716
1717 private void prepareSessionInitiate(
1718 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1719 final SessionDescription sessionDescription =
1720 SessionDescription.parse(webRTCSessionDescription.description);
1721 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1722 this.initiatorRtpContentMap = rtpContentMap;
1723 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1724 encryptSessionInitiate(rtpContentMap);
1725 Futures.addCallback(
1726 outgoingContentMapFuture,
1727 new FutureCallback<RtpContentMap>() {
1728 @Override
1729 public void onSuccess(final RtpContentMap outgoingContentMap) {
1730 sendSessionInitiate(outgoingContentMap, targetState);
1731 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1732 }
1733
1734 @Override
1735 public void onFailure(@NonNull final Throwable throwable) {
1736 failureToInitiateSession(throwable, targetState);
1737 }
1738 },
1739 MoreExecutors.directExecutor());
1740 }
1741
1742 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1743 if (isTerminated()) {
1744 Log.w(
1745 Config.LOGTAG,
1746 id.account.getJid().asBareJid()
1747 + ": preparing session was too slow. already terminated. nothing to do.");
1748 return;
1749 }
1750 this.transitionOrThrow(targetState);
1751 final JinglePacket sessionInitiate =
1752 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1753 send(sessionInitiate);
1754 }
1755
1756 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1757 final RtpContentMap rtpContentMap) {
1758 if (this.omemoVerification.hasDeviceId()) {
1759 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1760 verifiedPayloadFuture =
1761 id.account
1762 .getAxolotlService()
1763 .encrypt(
1764 rtpContentMap,
1765 id.with,
1766 omemoVerification.getDeviceId());
1767 final ListenableFuture<RtpContentMap> future =
1768 Futures.transform(
1769 verifiedPayloadFuture,
1770 verifiedPayload -> {
1771 omemoVerification.setSessionFingerprint(
1772 verifiedPayload.getFingerprint());
1773 return verifiedPayload.getPayload();
1774 },
1775 MoreExecutors.directExecutor());
1776 if (Config.REQUIRE_RTP_VERIFICATION) {
1777 return future;
1778 }
1779 return Futures.catching(
1780 future,
1781 CryptoFailedException.class,
1782 e -> {
1783 Log.w(
1784 Config.LOGTAG,
1785 id.account.getJid().asBareJid()
1786 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1787 e);
1788 return rtpContentMap;
1789 },
1790 MoreExecutors.directExecutor());
1791 } else {
1792 return Futures.immediateFuture(rtpContentMap);
1793 }
1794 }
1795
1796 private void sendSessionTerminate(final Reason reason) {
1797 sendSessionTerminate(reason, null);
1798 }
1799
1800 private void sendSessionTerminate(final Reason reason, final String text) {
1801 final State previous = this.state;
1802 final State target = reasonToState(reason);
1803 transitionOrThrow(target);
1804 if (previous != State.NULL) {
1805 writeLogMessage(target);
1806 }
1807 final JinglePacket jinglePacket =
1808 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1809 jinglePacket.setReason(reason, text);
1810 Log.d(Config.LOGTAG, jinglePacket.toString());
1811 send(jinglePacket);
1812 finish();
1813 }
1814
1815 private void sendTransportInfo(
1816 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1817 final RtpContentMap transportInfo;
1818 try {
1819 final RtpContentMap rtpContentMap =
1820 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1821 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1822 } catch (final Exception e) {
1823 Log.d(
1824 Config.LOGTAG,
1825 id.account.getJid().asBareJid()
1826 + ": unable to prepare transport-info from candidate for content="
1827 + contentName);
1828 return;
1829 }
1830 final JinglePacket jinglePacket =
1831 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1832 send(jinglePacket);
1833 }
1834
1835 private void send(final JinglePacket jinglePacket) {
1836 jinglePacket.setTo(id.with);
1837 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1838 }
1839
1840 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1841 if (response.getType() == IqPacket.TYPE.ERROR) {
1842 handleIqErrorResponse(response);
1843 return;
1844 }
1845 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1846 handleIqTimeoutResponse(response);
1847 }
1848 }
1849
1850 private void handleIqErrorResponse(final IqPacket response) {
1851 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1852 final String errorCondition = response.getErrorCondition();
1853 Log.d(
1854 Config.LOGTAG,
1855 id.account.getJid().asBareJid()
1856 + ": received IQ-error from "
1857 + response.getFrom()
1858 + " in RTP session. "
1859 + errorCondition);
1860 if (isTerminated()) {
1861 Log.i(
1862 Config.LOGTAG,
1863 id.account.getJid().asBareJid()
1864 + ": ignoring error because session was already terminated");
1865 return;
1866 }
1867 this.webRTCWrapper.close();
1868 final State target;
1869 if (Arrays.asList(
1870 "service-unavailable",
1871 "recipient-unavailable",
1872 "remote-server-not-found",
1873 "remote-server-timeout")
1874 .contains(errorCondition)) {
1875 target = State.TERMINATED_CONNECTIVITY_ERROR;
1876 } else {
1877 target = State.TERMINATED_APPLICATION_FAILURE;
1878 }
1879 transitionOrThrow(target);
1880 this.finish();
1881 }
1882
1883 private void handleIqTimeoutResponse(final IqPacket response) {
1884 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1885 Log.d(
1886 Config.LOGTAG,
1887 id.account.getJid().asBareJid()
1888 + ": received IQ timeout in RTP session with "
1889 + id.with
1890 + ". terminating with connectivity error");
1891 if (isTerminated()) {
1892 Log.i(
1893 Config.LOGTAG,
1894 id.account.getJid().asBareJid()
1895 + ": ignoring error because session was already terminated");
1896 return;
1897 }
1898 this.webRTCWrapper.close();
1899 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
1900 this.finish();
1901 }
1902
1903 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1904 Log.d(
1905 Config.LOGTAG,
1906 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1907 this.webRTCWrapper.close();
1908 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1909 respondWithOutOfOrder(jinglePacket);
1910 this.finish();
1911 }
1912
1913 private void respondWithTieBreak(final JinglePacket jinglePacket) {
1914 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
1915 }
1916
1917 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1918 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
1919 }
1920
1921 void respondWithJingleError(
1922 final IqPacket original,
1923 String jingleCondition,
1924 String condition,
1925 String conditionType) {
1926 jingleConnectionManager.respondWithJingleError(
1927 id.account, original, jingleCondition, condition, conditionType);
1928 }
1929
1930 private void respondOk(final JinglePacket jinglePacket) {
1931 xmppConnectionService.sendIqPacket(
1932 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1933 }
1934
1935 public RtpEndUserState getEndUserState() {
1936 switch (this.state) {
1937 case NULL:
1938 case PROPOSED:
1939 case SESSION_INITIALIZED:
1940 if (isInitiator()) {
1941 return RtpEndUserState.RINGING;
1942 } else {
1943 return RtpEndUserState.INCOMING_CALL;
1944 }
1945 case PROCEED:
1946 if (isInitiator()) {
1947 return RtpEndUserState.RINGING;
1948 } else {
1949 return RtpEndUserState.ACCEPTING_CALL;
1950 }
1951 case SESSION_INITIALIZED_PRE_APPROVED:
1952 if (isInitiator()) {
1953 return RtpEndUserState.RINGING;
1954 } else {
1955 return RtpEndUserState.CONNECTING;
1956 }
1957 case SESSION_ACCEPTED:
1958 final ContentAddition ca = getPendingContentAddition();
1959 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
1960 return RtpEndUserState.INCOMING_CONTENT_ADD;
1961 }
1962 return getPeerConnectionStateAsEndUserState();
1963 case REJECTED:
1964 case REJECTED_RACED:
1965 case TERMINATED_DECLINED_OR_BUSY:
1966 if (isInitiator()) {
1967 return RtpEndUserState.DECLINED_OR_BUSY;
1968 } else {
1969 return RtpEndUserState.ENDED;
1970 }
1971 case TERMINATED_SUCCESS:
1972 case ACCEPTED:
1973 case RETRACTED:
1974 case TERMINATED_CANCEL_OR_TIMEOUT:
1975 return RtpEndUserState.ENDED;
1976 case RETRACTED_RACED:
1977 if (isInitiator()) {
1978 return RtpEndUserState.ENDED;
1979 } else {
1980 return RtpEndUserState.RETRACTED;
1981 }
1982 case TERMINATED_CONNECTIVITY_ERROR:
1983 return zeroDuration()
1984 ? RtpEndUserState.CONNECTIVITY_ERROR
1985 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1986 case TERMINATED_APPLICATION_FAILURE:
1987 return RtpEndUserState.APPLICATION_ERROR;
1988 case TERMINATED_SECURITY_ERROR:
1989 return RtpEndUserState.SECURITY_ERROR;
1990 }
1991 throw new IllegalStateException(
1992 String.format("%s has no equivalent EndUserState", this.state));
1993 }
1994
1995 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
1996 final PeerConnection.PeerConnectionState state;
1997 try {
1998 state = webRTCWrapper.getState();
1999 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2000 // We usually close the WebRTCWrapper *before* transitioning so we might still
2001 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2002 return RtpEndUserState.ENDING_CALL;
2003 }
2004 switch (state) {
2005 case CONNECTED:
2006 return RtpEndUserState.CONNECTED;
2007 case NEW:
2008 case CONNECTING:
2009 return RtpEndUserState.CONNECTING;
2010 case CLOSED:
2011 return RtpEndUserState.ENDING_CALL;
2012 default:
2013 return zeroDuration()
2014 ? RtpEndUserState.CONNECTIVITY_ERROR
2015 : RtpEndUserState.RECONNECTING;
2016 }
2017 }
2018
2019 public ContentAddition getPendingContentAddition() {
2020 final RtpContentMap in = this.incomingContentAdd;
2021 final RtpContentMap out = this.outgoingContentAdd;
2022 if (out != null) {
2023 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2024 } else if (in != null) {
2025 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2026 } else {
2027 return null;
2028 }
2029 }
2030
2031 public Set<Media> getMedia() {
2032 final State current = getState();
2033 if (current == State.NULL) {
2034 if (isInitiator()) {
2035 return Preconditions.checkNotNull(
2036 this.proposedMedia, "RTP connection has not been initialized properly");
2037 }
2038 throw new IllegalStateException("RTP connection has not been initialized yet");
2039 }
2040 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2041 return Preconditions.checkNotNull(
2042 this.proposedMedia, "RTP connection has not been initialized properly");
2043 }
2044 final RtpContentMap localContentMap = getLocalContentMap();
2045 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2046 if (localContentMap != null) {
2047 return localContentMap.getMedia();
2048 } else if (initiatorContentMap != null) {
2049 return initiatorContentMap.getMedia();
2050 } else if (isTerminated()) {
2051 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2052 } else {
2053 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2054 }
2055 }
2056
2057 public boolean isVerified() {
2058 final String fingerprint = this.omemoVerification.getFingerprint();
2059 if (fingerprint == null) {
2060 return false;
2061 }
2062 final FingerprintStatus status =
2063 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2064 return status != null && status.isVerified();
2065 }
2066
2067 public boolean addMedia(final Media media) {
2068 final Set<Media> currentMedia = getMedia();
2069 if (currentMedia.contains(media)) {
2070 throw new IllegalStateException(String.format("%s has already been proposed", media));
2071 }
2072 // TODO add state protection - can only add while ACCEPTED or so
2073 Log.d(Config.LOGTAG,"adding media: "+media);
2074 return webRTCWrapper.addTrack(media);
2075 }
2076
2077 public synchronized void acceptCall() {
2078 switch (this.state) {
2079 case PROPOSED:
2080 cancelRingingTimeout();
2081 acceptCallFromProposed();
2082 break;
2083 case SESSION_INITIALIZED:
2084 cancelRingingTimeout();
2085 acceptCallFromSessionInitialized();
2086 break;
2087 case ACCEPTED:
2088 Log.w(
2089 Config.LOGTAG,
2090 id.account.getJid().asBareJid()
2091 + ": the call has already been accepted with another client. UI was just lagging behind");
2092 break;
2093 case PROCEED:
2094 case SESSION_ACCEPTED:
2095 Log.w(
2096 Config.LOGTAG,
2097 id.account.getJid().asBareJid()
2098 + ": the call has already been accepted. user probably double tapped the UI");
2099 break;
2100 default:
2101 throw new IllegalStateException("Can not accept call from " + this.state);
2102 }
2103 }
2104
2105 public void notifyPhoneCall() {
2106 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2107 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2108 rejectCall();
2109 } else {
2110 endCall();
2111 }
2112 }
2113
2114 public synchronized void rejectCall() {
2115 if (isTerminated()) {
2116 Log.w(
2117 Config.LOGTAG,
2118 id.account.getJid().asBareJid()
2119 + ": received rejectCall() when session has already been terminated. nothing to do");
2120 return;
2121 }
2122 switch (this.state) {
2123 case PROPOSED:
2124 rejectCallFromProposed();
2125 break;
2126 case SESSION_INITIALIZED:
2127 rejectCallFromSessionInitiate();
2128 break;
2129 default:
2130 throw new IllegalStateException("Can not reject call from " + this.state);
2131 }
2132 }
2133
2134 public synchronized void endCall() {
2135 if (isTerminated()) {
2136 Log.w(
2137 Config.LOGTAG,
2138 id.account.getJid().asBareJid()
2139 + ": received endCall() when session has already been terminated. nothing to do");
2140 return;
2141 }
2142 if (isInState(State.PROPOSED) && !isInitiator()) {
2143 rejectCallFromProposed();
2144 return;
2145 }
2146 if (isInState(State.PROCEED)) {
2147 if (isInitiator()) {
2148 retractFromProceed();
2149 } else {
2150 rejectCallFromProceed();
2151 }
2152 return;
2153 }
2154 if (isInitiator()
2155 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2156 this.webRTCWrapper.close();
2157 sendSessionTerminate(Reason.CANCEL);
2158 return;
2159 }
2160 if (isInState(State.SESSION_INITIALIZED)) {
2161 rejectCallFromSessionInitiate();
2162 return;
2163 }
2164 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2165 this.webRTCWrapper.close();
2166 sendSessionTerminate(Reason.SUCCESS);
2167 return;
2168 }
2169 if (isInState(
2170 State.TERMINATED_APPLICATION_FAILURE,
2171 State.TERMINATED_CONNECTIVITY_ERROR,
2172 State.TERMINATED_DECLINED_OR_BUSY)) {
2173 Log.d(
2174 Config.LOGTAG,
2175 "ignoring request to end call because already in state " + this.state);
2176 return;
2177 }
2178 throw new IllegalStateException(
2179 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2180 }
2181
2182 private void retractFromProceed() {
2183 Log.d(Config.LOGTAG, "retract from proceed");
2184 this.sendJingleMessage("retract");
2185 closeTransitionLogFinish(State.RETRACTED_RACED);
2186 }
2187
2188 private void closeTransitionLogFinish(final State state) {
2189 this.webRTCWrapper.close();
2190 transitionOrThrow(state);
2191 writeLogMessage(state);
2192 finish();
2193 }
2194
2195 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2196 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2197 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2198 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2199 }
2200
2201 private void acceptCallFromProposed() {
2202 transitionOrThrow(State.PROCEED);
2203 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2204 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2205 this.sendJingleMessage("proceed");
2206 }
2207
2208 private void rejectCallFromProposed() {
2209 transitionOrThrow(State.REJECTED);
2210 writeLogMessageMissed();
2211 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2212 this.sendJingleMessage("reject");
2213 finish();
2214 }
2215
2216 private void rejectCallFromProceed() {
2217 this.sendJingleMessage("reject");
2218 closeTransitionLogFinish(State.REJECTED_RACED);
2219 }
2220
2221 private void rejectCallFromSessionInitiate() {
2222 webRTCWrapper.close();
2223 sendSessionTerminate(Reason.DECLINE);
2224 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2225 }
2226
2227 private void sendJingleMessage(final String action) {
2228 sendJingleMessage(action, id.with);
2229 }
2230
2231 private void sendJingleMessage(final String action, final Jid to) {
2232 final MessagePacket messagePacket = new MessagePacket();
2233 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2234 messagePacket.setTo(to);
2235 final Element intent =
2236 messagePacket
2237 .addChild(action, Namespace.JINGLE_MESSAGE)
2238 .setAttribute("id", id.sessionId);
2239 if ("proceed".equals(action)) {
2240 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2241 if (isOmemoEnabled()) {
2242 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2243 final Element device =
2244 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2245 device.setAttribute("id", deviceId);
2246 }
2247 }
2248 messagePacket.addChild("store", "urn:xmpp:hints");
2249 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2250 }
2251
2252 private boolean isOmemoEnabled() {
2253 final Conversational conversational = message.getConversation();
2254 if (conversational instanceof Conversation) {
2255 return ((Conversation) conversational).getNextEncryption()
2256 == Message.ENCRYPTION_AXOLOTL;
2257 }
2258 return false;
2259 }
2260
2261 private void acceptCallFromSessionInitialized() {
2262 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2263 sendSessionAccept();
2264 }
2265
2266 private synchronized boolean isInState(State... state) {
2267 return Arrays.asList(state).contains(this.state);
2268 }
2269
2270 private boolean transition(final State target) {
2271 return transition(target, null);
2272 }
2273
2274 private synchronized boolean transition(final State target, final Runnable runnable) {
2275 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2276 if (validTransitions != null && validTransitions.contains(target)) {
2277 this.state = target;
2278 if (runnable != null) {
2279 runnable.run();
2280 }
2281 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2282 updateEndUserState();
2283 updateOngoingCallNotification();
2284 return true;
2285 } else {
2286 return false;
2287 }
2288 }
2289
2290 void transitionOrThrow(final State target) {
2291 if (!transition(target)) {
2292 throw new IllegalStateException(
2293 String.format("Unable to transition from %s to %s", this.state, target));
2294 }
2295 }
2296
2297 @Override
2298 public void onIceCandidate(final IceCandidate iceCandidate) {
2299 final RtpContentMap rtpContentMap =
2300 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2301 final IceUdpTransportInfo.Credentials credentials;
2302 try {
2303 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2304 } catch (final IllegalArgumentException e) {
2305 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2306 return;
2307 }
2308 final String uFrag = credentials.ufrag;
2309 final IceUdpTransportInfo.Candidate candidate =
2310 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2311 if (candidate == null) {
2312 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2313 return;
2314 }
2315 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2316 sendTransportInfo(iceCandidate.sdpMid, candidate);
2317 }
2318
2319 @Override
2320 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2321 Log.d(
2322 Config.LOGTAG,
2323 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2324 this.stateHistory.add(newState);
2325 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2326 this.sessionDuration.start();
2327 updateOngoingCallNotification();
2328 } else if (this.sessionDuration.isRunning()) {
2329 this.sessionDuration.stop();
2330 updateOngoingCallNotification();
2331 }
2332
2333 final boolean neverConnected =
2334 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2335
2336 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2337 if (neverConnected) {
2338 if (isTerminated()) {
2339 Log.d(
2340 Config.LOGTAG,
2341 id.account.getJid().asBareJid()
2342 + ": not sending session-terminate after connectivity error because session is already in state "
2343 + this.state);
2344 return;
2345 }
2346 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2347 return;
2348 } else {
2349 this.restartIce();
2350 }
2351 }
2352 updateEndUserState();
2353 }
2354
2355 private void restartIce() {
2356 this.stateHistory.clear();
2357 this.webRTCWrapper.restartIceAsync();
2358 }
2359
2360 @Override
2361 public void onRenegotiationNeeded() {
2362 this.webRTCWrapper.execute(this::renegotiate);
2363 }
2364
2365 private void renegotiate() {
2366 final SessionDescription sessionDescription;
2367 try {
2368 sessionDescription = setLocalSessionDescription();
2369 } catch (final Exception e) {
2370 final Throwable cause = Throwables.getRootCause(e);
2371 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2372 webRTCWrapper.close();
2373 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2374 return;
2375 }
2376 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2377 final RtpContentMap currentContentMap = getLocalContentMap();
2378 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2379 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2380
2381 Log.d(
2382 Config.LOGTAG,
2383 id.getAccount().getJid().asBareJid()
2384 + ": renegotiate. iceRestart="
2385 + iceRestart
2386 + " content id diff="
2387 + diff);
2388
2389 if (diff.hasModifications() && iceRestart) {
2390 webRTCWrapper.close();
2391 sendSessionTerminate(
2392 Reason.FAILED_APPLICATION,
2393 "WebRTC unexpectedly tried to modify content and transport at once");
2394 return;
2395 }
2396
2397 if (iceRestart) {
2398 initiateIceRestart(rtpContentMap);
2399 return;
2400 } else if (diff.isEmpty()) {
2401 Log.d(
2402 Config.LOGTAG,
2403 "renegotiation. nothing to do. SignalingState="
2404 + this.webRTCWrapper.getSignalingState());
2405 }
2406
2407 if (diff.added.size() > 0) {
2408 modifyLocalContentMap(rtpContentMap);
2409 sendContentAdd(rtpContentMap, diff.added);
2410 }
2411 }
2412
2413 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2414 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2415 final JinglePacket jinglePacket =
2416 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2417 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2418 jinglePacket.setTo(id.with);
2419 xmppConnectionService.sendIqPacket(
2420 id.account,
2421 jinglePacket,
2422 (account, response) -> {
2423 if (response.getType() == IqPacket.TYPE.RESULT) {
2424 Log.d(Config.LOGTAG, "received success to our ice restart");
2425 setLocalContentMap(rtpContentMap);
2426 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2427 return;
2428 }
2429 if (response.getType() == IqPacket.TYPE.ERROR) {
2430 if (isTieBreak(response)) {
2431 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2432 return;
2433 }
2434 handleIqErrorResponse(response);
2435 }
2436 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2437 handleIqTimeoutResponse(response);
2438 }
2439 });
2440 }
2441
2442 private boolean isTieBreak(final IqPacket response) {
2443 final Element error = response.findChild("error");
2444 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2445 }
2446
2447 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2448 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2449 this.outgoingContentAdd = contentAdd;
2450 final JinglePacket jinglePacket =
2451 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2452 jinglePacket.setTo(id.with);
2453 xmppConnectionService.sendIqPacket(
2454 id.account,
2455 jinglePacket,
2456 (connection, response) -> {
2457 if (response.getType() == IqPacket.TYPE.RESULT) {
2458 Log.d(
2459 Config.LOGTAG,
2460 id.getAccount().getJid().asBareJid()
2461 + ": received ACK to our content-add");
2462 return;
2463 }
2464 if (response.getType() == IqPacket.TYPE.ERROR) {
2465 if (isTieBreak(response)) {
2466 this.outgoingContentAdd = null;
2467 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2468 return;
2469 }
2470 handleIqErrorResponse(response);
2471 }
2472 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2473 handleIqTimeoutResponse(response);
2474 }
2475 });
2476 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2477 }
2478
2479 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2480 if (isInitiator()) {
2481 this.initiatorRtpContentMap = rtpContentMap;
2482 } else {
2483 this.responderRtpContentMap = rtpContentMap;
2484 }
2485 }
2486
2487 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2488 if (isInitiator()) {
2489 this.responderRtpContentMap = rtpContentMap;
2490 } else {
2491 this.initiatorRtpContentMap = rtpContentMap;
2492 }
2493 }
2494
2495 // this method is to be used for content map modifications that modify media
2496 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2497 final RtpContentMap activeContents = rtpContentMap.activeContents();
2498 setLocalContentMap(activeContents);
2499 this.webRTCWrapper.switchSpeakerPhonePreference(
2500 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2501 updateEndUserState();
2502 }
2503
2504 private SessionDescription setLocalSessionDescription()
2505 throws ExecutionException, InterruptedException {
2506 final org.webrtc.SessionDescription sessionDescription =
2507 this.webRTCWrapper.setLocalDescription().get();
2508 return SessionDescription.parse(sessionDescription.description);
2509 }
2510
2511 private void closeWebRTCSessionAfterFailedConnection() {
2512 this.webRTCWrapper.close();
2513 synchronized (this) {
2514 if (isTerminated()) {
2515 Log.d(
2516 Config.LOGTAG,
2517 id.account.getJid().asBareJid()
2518 + ": no need to send session-terminate after failed connection. Other party already did");
2519 return;
2520 }
2521 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2522 }
2523 }
2524
2525 public boolean zeroDuration() {
2526 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2527 }
2528
2529 public long getCallDuration() {
2530 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2531 }
2532
2533 public AppRTCAudioManager getAudioManager() {
2534 return webRTCWrapper.getAudioManager();
2535 }
2536
2537 public boolean isMicrophoneEnabled() {
2538 return webRTCWrapper.isMicrophoneEnabled();
2539 }
2540
2541 public boolean setMicrophoneEnabled(final boolean enabled) {
2542 return webRTCWrapper.setMicrophoneEnabled(enabled);
2543 }
2544
2545 public boolean isVideoEnabled() {
2546 return webRTCWrapper.isVideoEnabled();
2547 }
2548
2549 public void setVideoEnabled(final boolean enabled) {
2550 webRTCWrapper.setVideoEnabled(enabled);
2551 }
2552
2553 public boolean isCameraSwitchable() {
2554 return webRTCWrapper.isCameraSwitchable();
2555 }
2556
2557 public boolean isFrontCamera() {
2558 return webRTCWrapper.isFrontCamera();
2559 }
2560
2561 public ListenableFuture<Boolean> switchCamera() {
2562 return webRTCWrapper.switchCamera();
2563 }
2564
2565 @Override
2566 public void onAudioDeviceChanged(
2567 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2568 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2569 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2570 selectedAudioDevice, availableAudioDevices);
2571 }
2572
2573 private void updateEndUserState() {
2574 final RtpEndUserState endUserState = getEndUserState();
2575 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2576 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2577 id.account, id.with, id.sessionId, endUserState);
2578 }
2579
2580 private void updateOngoingCallNotification() {
2581 final State state = this.state;
2582 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2583 final boolean reconnecting;
2584 if (state == State.SESSION_ACCEPTED) {
2585 reconnecting =
2586 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2587 } else {
2588 reconnecting = false;
2589 }
2590 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2591 } else {
2592 xmppConnectionService.removeOngoingCall();
2593 }
2594 }
2595
2596 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2597 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2598 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2599 request.setTo(id.account.getDomain());
2600 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2601 xmppConnectionService.sendIqPacket(
2602 id.account,
2603 request,
2604 (account, response) -> {
2605 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2606 new ImmutableList.Builder<>();
2607 if (response.getType() == IqPacket.TYPE.RESULT) {
2608 final Element services =
2609 response.findChild(
2610 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2611 final List<Element> children =
2612 services == null
2613 ? Collections.emptyList()
2614 : services.getChildren();
2615 for (final Element child : children) {
2616 if ("service".equals(child.getName())) {
2617 final String type = child.getAttribute("type");
2618 final String host = child.getAttribute("host");
2619 final String sport = child.getAttribute("port");
2620 final Integer port =
2621 sport == null ? null : Ints.tryParse(sport);
2622 final String transport = child.getAttribute("transport");
2623 final String username = child.getAttribute("username");
2624 final String password = child.getAttribute("password");
2625 if (Strings.isNullOrEmpty(host) || port == null) {
2626 continue;
2627 }
2628 if (port < 0 || port > 65535) {
2629 continue;
2630 }
2631
2632
2633
2634
2635 if (Arrays.asList("stun", "stuns", "turn", "turns")
2636 .contains(type)
2637 && Arrays.asList("udp", "tcp").contains(transport)) {
2638 if (Arrays.asList("stuns", "turns").contains(type)
2639 && "udp".equals(transport)) {
2640 Log.d(
2641 Config.LOGTAG,
2642 id.account.getJid().asBareJid()
2643 + ": skipping invalid combination of udp/tls in external services");
2644 continue;
2645 }
2646
2647 // STUN URLs do not support a query section since M110
2648 final String uri;
2649 if (Arrays.asList("stun","stuns").contains(type)) {
2650 uri = String.format("%s:%s:%s", type, IP.wrapIPv6(host),port);
2651 } else {
2652 uri = String.format(
2653 "%s:%s:%s?transport=%s",
2654 type,
2655 IP.wrapIPv6(host),
2656 port,
2657 transport);
2658 }
2659
2660 final PeerConnection.IceServer.Builder iceServerBuilder =
2661 PeerConnection.IceServer.builder(uri);
2662 iceServerBuilder.setTlsCertPolicy(
2663 PeerConnection.TlsCertPolicy
2664 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2665 if (username != null && password != null) {
2666 iceServerBuilder.setUsername(username);
2667 iceServerBuilder.setPassword(password);
2668 } else if (Arrays.asList("turn", "turns").contains(type)) {
2669 // The WebRTC spec requires throwing an
2670 // InvalidAccessError when username (from libwebrtc
2671 // source coder)
2672 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2673 Log.d(
2674 Config.LOGTAG,
2675 id.account.getJid().asBareJid()
2676 + ": skipping "
2677 + type
2678 + "/"
2679 + transport
2680 + " without username and password");
2681 continue;
2682 }
2683 final PeerConnection.IceServer iceServer =
2684 iceServerBuilder.createIceServer();
2685 Log.d(
2686 Config.LOGTAG,
2687 id.account.getJid().asBareJid()
2688 + ": discovered ICE Server: "
2689 + iceServer);
2690 listBuilder.add(iceServer);
2691 }
2692 }
2693 }
2694 }
2695 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2696 if (iceServers.size() == 0) {
2697 Log.w(
2698 Config.LOGTAG,
2699 id.account.getJid().asBareJid()
2700 + ": no ICE server found "
2701 + response);
2702 }
2703 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2704 });
2705 } else {
2706 Log.w(
2707 Config.LOGTAG,
2708 id.account.getJid().asBareJid() + ": has no external service discovery");
2709 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2710 }
2711 }
2712
2713 private void finish() {
2714 if (isTerminated()) {
2715 this.cancelRingingTimeout();
2716 this.webRTCWrapper.verifyClosed();
2717 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2718 this.jingleConnectionManager.finishConnectionOrThrow(this);
2719 } else {
2720 throw new IllegalStateException(
2721 String.format("Unable to call finish from %s", this.state));
2722 }
2723 }
2724
2725 private void writeLogMessage(final State state) {
2726 final long duration = getCallDuration();
2727 if (state == State.TERMINATED_SUCCESS
2728 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2729 writeLogMessageSuccess(duration);
2730 } else {
2731 writeLogMessageMissed();
2732 }
2733 }
2734
2735 private void writeLogMessageSuccess(final long duration) {
2736 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2737 this.writeMessage();
2738 }
2739
2740 private void writeLogMessageMissed() {
2741 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2742 this.writeMessage();
2743 }
2744
2745 private void writeMessage() {
2746 final Conversational conversational = message.getConversation();
2747 if (conversational instanceof Conversation) {
2748 ((Conversation) conversational).add(this.message);
2749 xmppConnectionService.createMessageAsync(message);
2750 xmppConnectionService.updateConversationUi();
2751 } else {
2752 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2753 }
2754 }
2755
2756 public State getState() {
2757 return this.state;
2758 }
2759
2760 boolean isTerminated() {
2761 return TERMINATED.contains(this.state);
2762 }
2763
2764 public Optional<VideoTrack> getLocalVideoTrack() {
2765 return webRTCWrapper.getLocalVideoTrack();
2766 }
2767
2768 public Optional<VideoTrack> getRemoteVideoTrack() {
2769 return webRTCWrapper.getRemoteVideoTrack();
2770 }
2771
2772 public EglBase.Context getEglBaseContext() {
2773 return webRTCWrapper.getEglBaseContext();
2774 }
2775
2776 void setProposedMedia(final Set<Media> media) {
2777 this.proposedMedia = media;
2778 }
2779
2780 public void fireStateUpdate() {
2781 final RtpEndUserState endUserState = getEndUserState();
2782 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2783 id.account, id.with, id.sessionId, endUserState);
2784 }
2785
2786 public boolean isSwitchToVideoAvailable() {
2787 final boolean prerequisite =
2788 Media.audioOnly(getMedia())
2789 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2790 .contains(getEndUserState());
2791 return prerequisite && remoteHasVideoFeature();
2792 }
2793
2794 private boolean remoteHasVideoFeature() {
2795 final Contact contact = id.getContact();
2796 final Presence presence =
2797 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2798 final ServiceDiscoveryResult serviceDiscoveryResult =
2799 presence == null ? null : presence.getServiceDiscoveryResult();
2800 final List<String> features =
2801 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
2802 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
2803 }
2804
2805 private interface OnIceServersDiscovered {
2806 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2807 }
2808}