1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Joiner;
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.base.Stopwatch;
12import com.google.common.base.Strings;
13import com.google.common.base.Throwables;
14import com.google.common.collect.Collections2;
15import com.google.common.collect.ImmutableList;
16import com.google.common.collect.ImmutableMap;
17import com.google.common.collect.ImmutableSet;
18import com.google.common.collect.Maps;
19import com.google.common.collect.Sets;
20import com.google.common.primitives.Ints;
21import com.google.common.util.concurrent.FutureCallback;
22import com.google.common.util.concurrent.Futures;
23import com.google.common.util.concurrent.ListenableFuture;
24import com.google.common.util.concurrent.MoreExecutors;
25
26import org.webrtc.EglBase;
27import org.webrtc.IceCandidate;
28import org.webrtc.PeerConnection;
29import org.webrtc.VideoTrack;
30
31import java.util.Arrays;
32import java.util.Collection;
33import java.util.Collections;
34import java.util.LinkedList;
35import java.util.List;
36import java.util.Map;
37import java.util.Queue;
38import java.util.Set;
39import java.util.concurrent.ExecutionException;
40import java.util.concurrent.ScheduledFuture;
41import java.util.concurrent.TimeUnit;
42
43import eu.siacs.conversations.BuildConfig;
44import eu.siacs.conversations.Config;
45import eu.siacs.conversations.crypto.axolotl.AxolotlService;
46import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
47import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
48import eu.siacs.conversations.entities.Account;
49import eu.siacs.conversations.entities.Contact;
50import eu.siacs.conversations.entities.Conversation;
51import eu.siacs.conversations.entities.Conversational;
52import eu.siacs.conversations.entities.Message;
53import eu.siacs.conversations.entities.Presence;
54import eu.siacs.conversations.entities.RtpSessionStatus;
55import eu.siacs.conversations.entities.ServiceDiscoveryResult;
56import eu.siacs.conversations.services.AppRTCAudioManager;
57import eu.siacs.conversations.utils.IP;
58import eu.siacs.conversations.xml.Element;
59import eu.siacs.conversations.xml.Namespace;
60import eu.siacs.conversations.xmpp.Jid;
61import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
62import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
63import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
64import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
65import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
66import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
67import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
68import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
69import eu.siacs.conversations.xmpp.stanzas.IqPacket;
70import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
71
72public class JingleRtpConnection extends AbstractJingleConnection
73 implements WebRTCWrapper.EventCallback {
74
75 public static final List<State> STATES_SHOWING_ONGOING_CALL =
76 Arrays.asList(
77 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
78 private static final long BUSY_TIME_OUT = 30;
79 private static final List<State> TERMINATED =
80 Arrays.asList(
81 State.ACCEPTED,
82 State.REJECTED,
83 State.REJECTED_RACED,
84 State.RETRACTED,
85 State.RETRACTED_RACED,
86 State.TERMINATED_SUCCESS,
87 State.TERMINATED_DECLINED_OR_BUSY,
88 State.TERMINATED_CONNECTIVITY_ERROR,
89 State.TERMINATED_CANCEL_OR_TIMEOUT,
90 State.TERMINATED_APPLICATION_FAILURE,
91 State.TERMINATED_SECURITY_ERROR);
92
93 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
94
95 static {
96 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
97 new ImmutableMap.Builder<>();
98 transitionBuilder.put(
99 State.NULL,
100 ImmutableList.of(
101 State.PROPOSED,
102 State.SESSION_INITIALIZED,
103 State.TERMINATED_APPLICATION_FAILURE,
104 State.TERMINATED_SECURITY_ERROR));
105 transitionBuilder.put(
106 State.PROPOSED,
107 ImmutableList.of(
108 State.ACCEPTED,
109 State.PROCEED,
110 State.REJECTED,
111 State.RETRACTED,
112 State.TERMINATED_APPLICATION_FAILURE,
113 State.TERMINATED_SECURITY_ERROR,
114 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
115 // rebinds
116 ));
117 transitionBuilder.put(
118 State.PROCEED,
119 ImmutableList.of(
120 State.REJECTED_RACED,
121 State.RETRACTED_RACED,
122 State.SESSION_INITIALIZED_PRE_APPROVED,
123 State.TERMINATED_SUCCESS,
124 State.TERMINATED_APPLICATION_FAILURE,
125 State.TERMINATED_SECURITY_ERROR,
126 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
127 // bounces of the proceed message
128 ));
129 transitionBuilder.put(
130 State.SESSION_INITIALIZED,
131 ImmutableList.of(
132 State.SESSION_ACCEPTED,
133 State.TERMINATED_SUCCESS,
134 State.TERMINATED_DECLINED_OR_BUSY,
135 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
136 // and IQ timeouts
137 State.TERMINATED_CANCEL_OR_TIMEOUT,
138 State.TERMINATED_APPLICATION_FAILURE,
139 State.TERMINATED_SECURITY_ERROR));
140 transitionBuilder.put(
141 State.SESSION_INITIALIZED_PRE_APPROVED,
142 ImmutableList.of(
143 State.SESSION_ACCEPTED,
144 State.TERMINATED_SUCCESS,
145 State.TERMINATED_DECLINED_OR_BUSY,
146 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
147 // and IQ timeouts
148 State.TERMINATED_CANCEL_OR_TIMEOUT,
149 State.TERMINATED_APPLICATION_FAILURE,
150 State.TERMINATED_SECURITY_ERROR));
151 transitionBuilder.put(
152 State.SESSION_ACCEPTED,
153 ImmutableList.of(
154 State.TERMINATED_SUCCESS,
155 State.TERMINATED_DECLINED_OR_BUSY,
156 State.TERMINATED_CONNECTIVITY_ERROR,
157 State.TERMINATED_CANCEL_OR_TIMEOUT,
158 State.TERMINATED_APPLICATION_FAILURE,
159 State.TERMINATED_SECURITY_ERROR));
160 VALID_TRANSITIONS = transitionBuilder.build();
161 }
162
163 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
164 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
165 pendingIceCandidates = new LinkedList<>();
166 private final OmemoVerification omemoVerification = new OmemoVerification();
167 private final Message message;
168 private State state = State.NULL;
169 private Set<Media> proposedMedia;
170 private RtpContentMap initiatorRtpContentMap;
171 private RtpContentMap responderRtpContentMap;
172 private RtpContentMap incomingContentAdd;
173 private RtpContentMap outgoingContentAdd;
174 private IceUdpTransportInfo.Setup peerDtlsSetup;
175 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
176 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
177 private ScheduledFuture<?> ringingTimeoutFuture;
178
179 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
180 super(jingleConnectionManager, id, initiator);
181 final Conversation conversation =
182 jingleConnectionManager
183 .getXmppConnectionService()
184 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
185 this.message =
186 new Message(
187 conversation,
188 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
189 Message.TYPE_RTP_SESSION,
190 id.sessionId);
191 }
192
193 private static State reasonToState(Reason reason) {
194 switch (reason) {
195 case SUCCESS:
196 return State.TERMINATED_SUCCESS;
197 case DECLINE:
198 case BUSY:
199 return State.TERMINATED_DECLINED_OR_BUSY;
200 case CANCEL:
201 case TIMEOUT:
202 return State.TERMINATED_CANCEL_OR_TIMEOUT;
203 case SECURITY_ERROR:
204 return State.TERMINATED_SECURITY_ERROR;
205 case FAILED_APPLICATION:
206 case UNSUPPORTED_TRANSPORTS:
207 case UNSUPPORTED_APPLICATIONS:
208 return State.TERMINATED_APPLICATION_FAILURE;
209 default:
210 return State.TERMINATED_CONNECTIVITY_ERROR;
211 }
212 }
213
214 @Override
215 synchronized void deliverPacket(final JinglePacket jinglePacket) {
216 switch (jinglePacket.getAction()) {
217 case SESSION_INITIATE:
218 receiveSessionInitiate(jinglePacket);
219 break;
220 case TRANSPORT_INFO:
221 receiveTransportInfo(jinglePacket);
222 break;
223 case SESSION_ACCEPT:
224 receiveSessionAccept(jinglePacket);
225 break;
226 case SESSION_TERMINATE:
227 receiveSessionTerminate(jinglePacket);
228 break;
229 case CONTENT_ADD:
230 receiveContentAdd(jinglePacket);
231 break;
232 case CONTENT_ACCEPT:
233 receiveContentAccept(jinglePacket);
234 break;
235 case CONTENT_REJECT:
236 receiveContentReject(jinglePacket);
237 break;
238 case CONTENT_REMOVE:
239 receiveContentRemove(jinglePacket);
240 break;
241 case CONTENT_MODIFY:
242 receiveContentModify(jinglePacket);
243 break;
244 default:
245 respondOk(jinglePacket);
246 Log.d(
247 Config.LOGTAG,
248 String.format(
249 "%s: received unhandled jingle action %s",
250 id.account.getJid().asBareJid(), jinglePacket.getAction()));
251 break;
252 }
253 }
254
255 @Override
256 synchronized void notifyRebound() {
257 if (isTerminated()) {
258 return;
259 }
260 webRTCWrapper.close();
261 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
262 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
263 }
264 if (isInState(
265 State.SESSION_INITIALIZED,
266 State.SESSION_INITIALIZED_PRE_APPROVED,
267 State.SESSION_ACCEPTED)) {
268 // we might have already changed resources (full jid) at this point; so this might not
269 // even reach the other party
270 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
271 } else {
272 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
273 finish();
274 }
275 }
276
277 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
278 respondOk(jinglePacket);
279 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
280 final State previous = this.state;
281 Log.d(
282 Config.LOGTAG,
283 id.account.getJid().asBareJid()
284 + ": received session terminate reason="
285 + wrapper.reason
286 + "("
287 + Strings.nullToEmpty(wrapper.text)
288 + ") while in state "
289 + previous);
290 if (TERMINATED.contains(previous)) {
291 Log.d(
292 Config.LOGTAG,
293 id.account.getJid().asBareJid()
294 + ": ignoring session terminate because already in "
295 + previous);
296 return;
297 }
298 webRTCWrapper.close();
299 final State target = reasonToState(wrapper.reason);
300 transitionOrThrow(target);
301 writeLogMessage(target);
302 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
303 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
304 }
305 finish();
306 }
307
308 private void receiveTransportInfo(final JinglePacket jinglePacket) {
309 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
310 // INITIALIZED only after transport-info has been received
311 if (isInState(
312 State.NULL,
313 State.PROCEED,
314 State.SESSION_INITIALIZED,
315 State.SESSION_INITIALIZED_PRE_APPROVED,
316 State.SESSION_ACCEPTED)) {
317 final RtpContentMap contentMap;
318 try {
319 contentMap = RtpContentMap.of(jinglePacket);
320 } catch (final IllegalArgumentException | NullPointerException e) {
321 Log.d(
322 Config.LOGTAG,
323 id.account.getJid().asBareJid()
324 + ": improperly formatted contents; ignoring",
325 e);
326 respondOk(jinglePacket);
327 return;
328 }
329 receiveTransportInfo(jinglePacket, contentMap);
330 } else {
331 if (isTerminated()) {
332 respondOk(jinglePacket);
333 Log.d(
334 Config.LOGTAG,
335 id.account.getJid().asBareJid()
336 + ": ignoring out-of-order transport info; we where already terminated");
337 } else {
338 Log.d(
339 Config.LOGTAG,
340 id.account.getJid().asBareJid()
341 + ": received transport info while in state="
342 + this.state);
343 terminateWithOutOfOrder(jinglePacket);
344 }
345 }
346 }
347
348 private void receiveTransportInfo(
349 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
350 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
351 contentMap.contents.entrySet();
352 if (this.state == State.SESSION_ACCEPTED) {
353 // zero candidates + modified credentials are an ICE restart offer
354 if (checkForIceRestart(jinglePacket, contentMap)) {
355 return;
356 }
357 respondOk(jinglePacket);
358 try {
359 processCandidates(candidates);
360 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
361 Log.w(
362 Config.LOGTAG,
363 id.account.getJid().asBareJid()
364 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
365 }
366 } else {
367 respondOk(jinglePacket);
368 pendingIceCandidates.addAll(candidates);
369 }
370 }
371
372 private void receiveContentAdd(final JinglePacket jinglePacket) {
373 final RtpContentMap modification;
374 try {
375 modification = RtpContentMap.of(jinglePacket);
376 modification.requireContentDescriptions();
377 } catch (final RuntimeException e) {
378 Log.d(
379 Config.LOGTAG,
380 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
381 Throwables.getRootCause(e));
382 respondOk(jinglePacket);
383 webRTCWrapper.close();
384 sendSessionTerminate(Reason.of(e), e.getMessage());
385 return;
386 }
387 if (isInState(State.SESSION_ACCEPTED)) {
388 receiveContentAdd(jinglePacket, modification);
389 } else {
390 terminateWithOutOfOrder(jinglePacket);
391 }
392 }
393
394 private void receiveContentAdd(
395 final JinglePacket jinglePacket, final RtpContentMap modification) {
396 final RtpContentMap remote = getRemoteContentMap();
397 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
398 respondOk(jinglePacket);
399 this.webRTCWrapper.close();
400 sendSessionTerminate(
401 Reason.FAILED_APPLICATION,
402 String.format(
403 "contents with names %s already exists",
404 Joiner.on(", ").join(modification.getNames())));
405 return;
406 }
407 final ContentAddition contentAddition =
408 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
409
410 final RtpContentMap outgoing = this.outgoingContentAdd;
411 final Set<ContentAddition.Summary> outgoingContentAddSummary =
412 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
413
414 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
415 if (isInitiator()) {
416 Log.d(
417 Config.LOGTAG,
418 id.getAccount().getJid().asBareJid()
419 + ": respond with tie break to matching content-add offer");
420 respondWithTieBreak(jinglePacket);
421 } else {
422 Log.d(
423 Config.LOGTAG,
424 id.getAccount().getJid().asBareJid()
425 + ": automatically accept matching content-add offer");
426 acceptContentAdd(contentAddition.summary, modification);
427 }
428 return;
429 }
430
431 // once we can display multiple video tracks we can be more loose with this condition
432 // theoretically it should also be fine to automatically accept audio only contents
433 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
434 Log.d(
435 Config.LOGTAG,
436 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
437 this.incomingContentAdd = modification;
438 respondOk(jinglePacket);
439 updateEndUserState();
440 } else {
441 respondOk(jinglePacket);
442 // TODO do we want to add a reason?
443 rejectContentAdd(modification);
444 }
445 }
446
447 private void receiveContentAccept(final JinglePacket jinglePacket) {
448 final RtpContentMap receivedContentAccept;
449 try {
450 receivedContentAccept = RtpContentMap.of(jinglePacket);
451 receivedContentAccept.requireContentDescriptions();
452 } catch (final RuntimeException e) {
453 Log.d(
454 Config.LOGTAG,
455 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
456 Throwables.getRootCause(e));
457 respondOk(jinglePacket);
458 webRTCWrapper.close();
459 sendSessionTerminate(Reason.of(e), e.getMessage());
460 return;
461 }
462
463 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
464 if (outgoingContentAdd == null) {
465 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
466 terminateWithOutOfOrder(jinglePacket);
467 return;
468 }
469 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
470 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
471 this.outgoingContentAdd = null;
472 respondOk(jinglePacket);
473 receiveContentAccept(receivedContentAccept);
474 } else {
475 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
476 terminateWithOutOfOrder(jinglePacket);
477 }
478 }
479
480 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
481 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
482 final RtpContentMap modifiedContentMap =
483 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
484
485 setRemoteContentMap(modifiedContentMap);
486
487 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
488
489 final org.webrtc.SessionDescription sdp =
490 new org.webrtc.SessionDescription(
491 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
492
493 try {
494 this.webRTCWrapper.setRemoteDescription(sdp).get();
495 } catch (final Exception e) {
496 final Throwable cause = Throwables.getRootCause(e);
497 Log.d(
498 Config.LOGTAG,
499 id.getAccount().getJid().asBareJid()
500 + ": unable to set remote description after receiving content-accept",
501 cause);
502 webRTCWrapper.close();
503 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
504 return;
505 }
506 updateEndUserState();
507 Log.d(
508 Config.LOGTAG,
509 id.getAccount().getJid().asBareJid()
510 + ": remote has accepted content-add "
511 + ContentAddition.summary(receivedContentAccept));
512 }
513
514 private void receiveContentModify(final JinglePacket jinglePacket) {
515 final Map<String, Content.Senders> modification =
516 Maps.transformEntries(
517 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
518 respondOk(jinglePacket);
519 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
520 }
521
522 private void receiveContentReject(final JinglePacket jinglePacket) {
523 final RtpContentMap receivedContentReject;
524 try {
525 receivedContentReject = RtpContentMap.of(jinglePacket);
526 } catch (final RuntimeException e) {
527 Log.d(
528 Config.LOGTAG,
529 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
530 Throwables.getRootCause(e));
531 respondOk(jinglePacket);
532 this.webRTCWrapper.close();
533 sendSessionTerminate(Reason.of(e), e.getMessage());
534 return;
535 }
536
537 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
538 if (outgoingContentAdd == null) {
539 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
540 terminateWithOutOfOrder(jinglePacket);
541 return;
542 }
543 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
544 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
545 this.outgoingContentAdd = null;
546 respondOk(jinglePacket);
547 Log.d(Config.LOGTAG,jinglePacket.toString());
548 receiveContentReject(ourSummary);
549 } else {
550 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
551 terminateWithOutOfOrder(jinglePacket);
552 }
553 }
554
555 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
556 try {
557 this.webRTCWrapper.removeTrack(Media.VIDEO);
558 final RtpContentMap localContentMap = customRollback();
559 modifyLocalContentMap(localContentMap);
560 } catch (final Exception e) {
561 final Throwable cause = Throwables.getRootCause(e);
562 Log.d(
563 Config.LOGTAG,
564 id.getAccount().getJid().asBareJid()
565 + ": unable to rollback local description after receiving content-reject",
566 cause);
567 webRTCWrapper.close();
568 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
569 return;
570 }
571 Log.d(
572 Config.LOGTAG,
573 id.getAccount().getJid().asBareJid()
574 + ": remote has rejected our content-add "
575 + summary);
576 }
577
578 private void receiveContentRemove(final JinglePacket jinglePacket) {
579 final RtpContentMap receivedContentRemove;
580 try {
581 receivedContentRemove = RtpContentMap.of(jinglePacket);
582 receivedContentRemove.requireContentDescriptions();
583 } catch (final RuntimeException e) {
584 Log.d(
585 Config.LOGTAG,
586 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
587 Throwables.getRootCause(e));
588 respondOk(jinglePacket);
589 this.webRTCWrapper.close();
590 sendSessionTerminate(Reason.of(e), e.getMessage());
591 return;
592 }
593 respondOk(jinglePacket);
594 receiveContentRemove(receivedContentRemove);
595 }
596
597 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
598 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
599 final Set<ContentAddition.Summary> contentAddSummary =
600 incomingContentAdd == null
601 ? Collections.emptySet()
602 : ContentAddition.summary(incomingContentAdd);
603 final Set<ContentAddition.Summary> removeSummary =
604 ContentAddition.summary(receivedContentRemove);
605 if (contentAddSummary.equals(removeSummary)) {
606 this.incomingContentAdd = null;
607 updateEndUserState();
608 } else {
609 webRTCWrapper.close();
610 sendSessionTerminate(
611 Reason.FAILED_APPLICATION,
612 String.format(
613 "%s only supports %s as a means to retract a not yet accepted %s",
614 BuildConfig.APP_NAME,
615 JinglePacket.Action.CONTENT_REMOVE,
616 JinglePacket.Action.CONTENT_ACCEPT));
617 }
618 }
619
620 public synchronized void retractContentAdd() {
621 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
622 if (outgoingContentAdd == null) {
623 throw new IllegalStateException("Not outgoing content add");
624 }
625 try {
626 webRTCWrapper.removeTrack(Media.VIDEO);
627 final RtpContentMap localContentMap = customRollback();
628 modifyLocalContentMap(localContentMap);
629 } catch (final Exception e) {
630 final Throwable cause = Throwables.getRootCause(e);
631 Log.d(
632 Config.LOGTAG,
633 id.getAccount().getJid().asBareJid()
634 + ": unable to rollback local description after trying to retract content-add",
635 cause);
636 webRTCWrapper.close();
637 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
638 return;
639 }
640 this.outgoingContentAdd = null;
641 final JinglePacket retract =
642 outgoingContentAdd
643 .toStub()
644 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
645 this.send(retract);
646 Log.d(
647 Config.LOGTAG,
648 id.getAccount().getJid()
649 + ": retract content-add "
650 + ContentAddition.summary(outgoingContentAdd));
651 }
652
653 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
654 final SessionDescription sdp = setLocalSessionDescription();
655 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
656 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
657 this.webRTCWrapper
658 .setRemoteDescription(
659 new org.webrtc.SessionDescription(
660 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
661 .get();
662 return localRtpContentMap;
663 }
664
665 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
666 final RtpContentMap currentRemote = getRemoteContentMap();
667 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
668 if (diff.isEmpty()) {
669 throw new IllegalStateException(
670 "Unexpected rollback condition. No difference between local and remote");
671 }
672 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
673 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
674 final RtpContentMap nextRemote =
675 currentRemote.addContent(
676 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
677 return SessionDescription.of(nextRemote, !isInitiator());
678 }
679 throw new IllegalStateException(
680 "Unexpected rollback condition. Senders were not uniformly none");
681 }
682
683 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
684 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
685 if (incomingContentAdd == null) {
686 throw new IllegalStateException("No incoming content add");
687 }
688
689 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
690 this.incomingContentAdd = null;
691 acceptContentAdd(contentAddition, incomingContentAdd);
692 } else {
693 throw new IllegalStateException("Accepted content add does not match pending content-add");
694 }
695 }
696
697 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
698 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
699 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
700 this.setRemoteContentMap(modifiedContentMap);
701
702 final SessionDescription offer;
703 try {
704 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
705 } catch (final IllegalArgumentException | NullPointerException e) {
706 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
707 webRTCWrapper.close();
708 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
709 return;
710 }
711 this.incomingContentAdd = null;
712 acceptContentAdd(contentAddition, offer);
713 }
714
715 private void acceptContentAdd(
716 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
717 final org.webrtc.SessionDescription sdp =
718 new org.webrtc.SessionDescription(
719 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
720 try {
721 this.webRTCWrapper.setRemoteDescription(sdp).get();
722
723 // TODO add tracks for 'media' where contentAddition.senders matches
724
725 // TODO if senders.sending(isInitiator())
726
727 this.webRTCWrapper.addTrack(Media.VIDEO);
728
729 // TODO add additional transceivers for recv only cases
730
731 final SessionDescription answer = setLocalSessionDescription();
732 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
733
734 final RtpContentMap contentAcceptMap =
735 rtpContentMap.toContentModification(
736 Collections2.transform(contentAddition, ca -> ca.name));
737 Log.d(
738 Config.LOGTAG,
739 id.getAccount().getJid().asBareJid()
740 + ": sending content-accept "
741 + ContentAddition.summary(contentAcceptMap));
742 modifyLocalContentMap(rtpContentMap);
743 sendContentAccept(contentAcceptMap);
744 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
745 } catch (final Exception e) {
746 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
747 webRTCWrapper.close();
748 sendSessionTerminate(Reason.FAILED_APPLICATION);
749 }
750 }
751
752 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
753 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
754 send(jinglePacket);
755 }
756
757 public synchronized void rejectContentAdd() {
758 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
759 if (incomingContentAdd == null) {
760 throw new IllegalStateException("No incoming content add");
761 }
762 this.incomingContentAdd = null;
763 updateEndUserState();
764 rejectContentAdd(incomingContentAdd);
765 }
766
767 private void rejectContentAdd(final RtpContentMap contentMap) {
768 final JinglePacket jinglePacket =
769 contentMap
770 .toStub()
771 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
772 Log.d(
773 Config.LOGTAG,
774 id.getAccount().getJid().asBareJid()
775 + ": rejecting content "
776 + ContentAddition.summary(contentMap));
777 send(jinglePacket);
778 }
779
780 private boolean checkForIceRestart(
781 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
782 final RtpContentMap existing = getRemoteContentMap();
783 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
784 final IceUdpTransportInfo.Credentials newCredentials;
785 try {
786 existingCredentials = existing.getCredentials();
787 newCredentials = rtpContentMap.getDistinctCredentials();
788 } catch (final IllegalStateException e) {
789 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
790 return false;
791 }
792 if (existingCredentials.contains(newCredentials)) {
793 return false;
794 }
795 // TODO an alternative approach is to check if we already got an iq result to our
796 // ICE-restart
797 // and if that's the case we are seeing an answer.
798 // This might be more spec compliant but also more error prone potentially
799 final boolean isOffer = rtpContentMap.emptyCandidates();
800 final RtpContentMap restartContentMap;
801 try {
802 if (isOffer) {
803 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
804 restartContentMap =
805 existing.modifiedCredentials(
806 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
807 } else {
808 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
809 Log.d(
810 Config.LOGTAG,
811 "received confirmation of ICE restart"
812 + newCredentials
813 + " peer_setup="
814 + setup);
815 // DTLS setup attribute needs to be rewritten to reflect current peer state
816 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
817 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
818 }
819 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
820 return isOffer;
821 } else {
822 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
823 respondWithTieBreak(jinglePacket);
824 return true;
825 }
826 } catch (final Exception exception) {
827 respondOk(jinglePacket);
828 final Throwable rootCause = Throwables.getRootCause(exception);
829 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
830 // If this happens a termination is already in progress
831 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
832 return true;
833 }
834 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
835 webRTCWrapper.close();
836 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
837 return true;
838 }
839 }
840
841 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
842 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
843 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
844 throw new IllegalStateException("Invalid peer setup");
845 }
846 return peerSetup;
847 }
848
849 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
850 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
851 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
852 }
853 this.peerDtlsSetup = setup;
854 }
855
856 private boolean applyIceRestart(
857 final JinglePacket jinglePacket,
858 final RtpContentMap restartContentMap,
859 final boolean isOffer)
860 throws ExecutionException, InterruptedException {
861 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
862 final org.webrtc.SessionDescription.Type type =
863 isOffer
864 ? org.webrtc.SessionDescription.Type.OFFER
865 : org.webrtc.SessionDescription.Type.ANSWER;
866 org.webrtc.SessionDescription sdp =
867 new org.webrtc.SessionDescription(type, sessionDescription.toString());
868 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
869 if (isInitiator()) {
870 // We ignore the offer and respond with tie-break. This will clause the responder
871 // not to apply the content map
872 return false;
873 }
874 }
875 webRTCWrapper.setRemoteDescription(sdp).get();
876 setRemoteContentMap(restartContentMap);
877 if (isOffer) {
878 final SessionDescription localSessionDescription = setLocalSessionDescription();
879 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
880 // We need to respond OK before sending any candidates
881 respondOk(jinglePacket);
882 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
883 } else {
884 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
885 }
886 return true;
887 }
888
889 private void processCandidates(
890 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
891 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
892 processCandidate(content);
893 }
894 }
895
896 private void processCandidate(
897 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
898 final RtpContentMap rtpContentMap = getRemoteContentMap();
899 final List<String> indices = toIdentificationTags(rtpContentMap);
900 final String sdpMid = content.getKey(); // aka content name
901 final IceUdpTransportInfo transport = content.getValue().transport;
902 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
903
904 // TODO check that credentials remained the same
905
906 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
907 final String sdp;
908 try {
909 sdp = candidate.toSdpAttribute(credentials.ufrag);
910 } catch (final IllegalArgumentException e) {
911 Log.d(
912 Config.LOGTAG,
913 id.account.getJid().asBareJid()
914 + ": ignoring invalid ICE candidate "
915 + e.getMessage());
916 continue;
917 }
918 final int mLineIndex = indices.indexOf(sdpMid);
919 if (mLineIndex < 0) {
920 Log.w(
921 Config.LOGTAG,
922 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
923 }
924 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
925 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
926 this.webRTCWrapper.addIceCandidate(iceCandidate);
927 }
928 }
929
930 private RtpContentMap getRemoteContentMap() {
931 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
932 }
933
934 private RtpContentMap getLocalContentMap() {
935 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
936 }
937
938 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
939 final Group originalGroup = rtpContentMap.group;
940 final List<String> identificationTags =
941 originalGroup == null
942 ? rtpContentMap.getNames()
943 : originalGroup.getIdentificationTags();
944 if (identificationTags.size() == 0) {
945 Log.w(
946 Config.LOGTAG,
947 id.account.getJid().asBareJid()
948 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
949 }
950 return identificationTags;
951 }
952
953 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
954 final JinglePacket jinglePacket, final boolean expectVerification) {
955 final RtpContentMap receivedContentMap;
956 try {
957 receivedContentMap = RtpContentMap.of(jinglePacket);
958 } catch (final Exception e) {
959 return Futures.immediateFailedFuture(e);
960 }
961 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
962 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
963 id.account
964 .getAxolotlService()
965 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
966 return Futures.transform(
967 future,
968 omemoVerifiedPayload -> {
969 // TODO test if an exception here triggers a correct abort
970 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
971 Log.d(
972 Config.LOGTAG,
973 id.account.getJid().asBareJid()
974 + ": received verifiable DTLS fingerprint via "
975 + omemoVerification);
976 return omemoVerifiedPayload.getPayload();
977 },
978 MoreExecutors.directExecutor());
979 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
980 return Futures.immediateFailedFuture(
981 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
982 } else {
983 return Futures.immediateFuture(receivedContentMap);
984 }
985 }
986
987 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
988 if (isInitiator()) {
989 Log.d(
990 Config.LOGTAG,
991 String.format(
992 "%s: received session-initiate even though we were initiating",
993 id.account.getJid().asBareJid()));
994 if (isTerminated()) {
995 Log.d(
996 Config.LOGTAG,
997 String.format(
998 "%s: got a reason to terminate with out-of-order. but already in state %s",
999 id.account.getJid().asBareJid(), getState()));
1000 respondWithOutOfOrder(jinglePacket);
1001 } else {
1002 terminateWithOutOfOrder(jinglePacket);
1003 }
1004 return;
1005 }
1006 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1007 Futures.addCallback(
1008 future,
1009 new FutureCallback<RtpContentMap>() {
1010 @Override
1011 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1012 receiveSessionInitiate(jinglePacket, rtpContentMap);
1013 }
1014
1015 @Override
1016 public void onFailure(@NonNull final Throwable throwable) {
1017 respondOk(jinglePacket);
1018 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1019 }
1020 },
1021 MoreExecutors.directExecutor());
1022 }
1023
1024 private void receiveSessionInitiate(
1025 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1026 try {
1027 contentMap.requireContentDescriptions();
1028 contentMap.requireDTLSFingerprint(true);
1029 } catch (final RuntimeException e) {
1030 Log.d(
1031 Config.LOGTAG,
1032 id.account.getJid().asBareJid() + ": improperly formatted contents",
1033 Throwables.getRootCause(e));
1034 respondOk(jinglePacket);
1035 sendSessionTerminate(Reason.of(e), e.getMessage());
1036 return;
1037 }
1038 Log.d(
1039 Config.LOGTAG,
1040 "processing session-init with " + contentMap.contents.size() + " contents");
1041 final State target;
1042 if (this.state == State.PROCEED) {
1043 Preconditions.checkState(
1044 proposedMedia != null && proposedMedia.size() > 0,
1045 "proposed media must be set when processing pre-approved session-initiate");
1046 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1047 sendSessionTerminate(
1048 Reason.SECURITY_ERROR,
1049 String.format(
1050 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1051 this.proposedMedia, contentMap.getMedia()));
1052 return;
1053 }
1054 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1055 } else {
1056 target = State.SESSION_INITIALIZED;
1057 }
1058 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1059 respondOk(jinglePacket);
1060 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1061 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1062 Log.d(
1063 Config.LOGTAG,
1064 id.account.getJid().asBareJid()
1065 + ": automatically accepting session-initiate");
1066 sendSessionAccept();
1067 } else {
1068 Log.d(
1069 Config.LOGTAG,
1070 id.account.getJid().asBareJid()
1071 + ": received not pre-approved session-initiate. start ringing");
1072 startRinging();
1073 }
1074 } else {
1075 Log.d(
1076 Config.LOGTAG,
1077 String.format(
1078 "%s: received session-initiate while in state %s",
1079 id.account.getJid().asBareJid(), state));
1080 terminateWithOutOfOrder(jinglePacket);
1081 }
1082 }
1083
1084 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1085 if (!isInitiator()) {
1086 Log.d(
1087 Config.LOGTAG,
1088 String.format(
1089 "%s: received session-accept even though we were responding",
1090 id.account.getJid().asBareJid()));
1091 terminateWithOutOfOrder(jinglePacket);
1092 return;
1093 }
1094 final ListenableFuture<RtpContentMap> future =
1095 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1096 Futures.addCallback(
1097 future,
1098 new FutureCallback<RtpContentMap>() {
1099 @Override
1100 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1101 receiveSessionAccept(jinglePacket, rtpContentMap);
1102 }
1103
1104 @Override
1105 public void onFailure(@NonNull final Throwable throwable) {
1106 respondOk(jinglePacket);
1107 Log.d(
1108 Config.LOGTAG,
1109 id.account.getJid().asBareJid()
1110 + ": improperly formatted contents in session-accept",
1111 throwable);
1112 webRTCWrapper.close();
1113 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1114 }
1115 },
1116 MoreExecutors.directExecutor());
1117 }
1118
1119 private void receiveSessionAccept(
1120 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1121 try {
1122 contentMap.requireContentDescriptions();
1123 contentMap.requireDTLSFingerprint();
1124 } catch (final RuntimeException e) {
1125 respondOk(jinglePacket);
1126 Log.d(
1127 Config.LOGTAG,
1128 id.account.getJid().asBareJid()
1129 + ": improperly formatted contents in session-accept",
1130 e);
1131 webRTCWrapper.close();
1132 sendSessionTerminate(Reason.of(e), e.getMessage());
1133 return;
1134 }
1135 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1136 if (!initiatorMedia.equals(contentMap.getMedia())) {
1137 sendSessionTerminate(
1138 Reason.SECURITY_ERROR,
1139 String.format(
1140 "Your session-included included media %s but our session-initiate was %s",
1141 this.proposedMedia, contentMap.getMedia()));
1142 return;
1143 }
1144 Log.d(
1145 Config.LOGTAG,
1146 "processing session-accept with " + contentMap.contents.size() + " contents");
1147 if (transition(State.SESSION_ACCEPTED)) {
1148 respondOk(jinglePacket);
1149 receiveSessionAccept(contentMap);
1150 } else {
1151 Log.d(
1152 Config.LOGTAG,
1153 String.format(
1154 "%s: received session-accept while in state %s",
1155 id.account.getJid().asBareJid(), state));
1156 respondOk(jinglePacket);
1157 }
1158 }
1159
1160 private void receiveSessionAccept(final RtpContentMap contentMap) {
1161 this.responderRtpContentMap = contentMap;
1162 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1163 final SessionDescription sessionDescription;
1164 try {
1165 sessionDescription = SessionDescription.of(contentMap, false);
1166 } catch (final IllegalArgumentException | NullPointerException e) {
1167 Log.d(
1168 Config.LOGTAG,
1169 id.account.getJid().asBareJid()
1170 + ": unable convert offer from session-accept to SDP",
1171 e);
1172 webRTCWrapper.close();
1173 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1174 return;
1175 }
1176 final org.webrtc.SessionDescription answer =
1177 new org.webrtc.SessionDescription(
1178 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1179 try {
1180 this.webRTCWrapper.setRemoteDescription(answer).get();
1181 } catch (final Exception e) {
1182 Log.d(
1183 Config.LOGTAG,
1184 id.account.getJid().asBareJid()
1185 + ": unable to set remote description after receiving session-accept",
1186 Throwables.getRootCause(e));
1187 webRTCWrapper.close();
1188 sendSessionTerminate(
1189 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1190 return;
1191 }
1192 processCandidates(contentMap.contents.entrySet());
1193 }
1194
1195 private void sendSessionAccept() {
1196 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1197 if (rtpContentMap == null) {
1198 throw new IllegalStateException("initiator RTP Content Map has not been set");
1199 }
1200 final SessionDescription offer;
1201 try {
1202 offer = SessionDescription.of(rtpContentMap, true);
1203 } catch (final IllegalArgumentException | NullPointerException e) {
1204 Log.d(
1205 Config.LOGTAG,
1206 id.account.getJid().asBareJid()
1207 + ": unable convert offer from session-initiate to SDP",
1208 e);
1209 webRTCWrapper.close();
1210 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1211 return;
1212 }
1213 sendSessionAccept(rtpContentMap.getMedia(), offer);
1214 }
1215
1216 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1217 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1218 }
1219
1220 private synchronized void sendSessionAccept(
1221 final Set<Media> media,
1222 final SessionDescription offer,
1223 final List<PeerConnection.IceServer> iceServers) {
1224 if (isTerminated()) {
1225 Log.w(
1226 Config.LOGTAG,
1227 id.account.getJid().asBareJid()
1228 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1229 return;
1230 }
1231 try {
1232 setupWebRTC(media, iceServers);
1233 } catch (final WebRTCWrapper.InitializationException e) {
1234 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1235 webRTCWrapper.close();
1236 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1237 return;
1238 }
1239 final org.webrtc.SessionDescription sdp =
1240 new org.webrtc.SessionDescription(
1241 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1242 try {
1243 this.webRTCWrapper.setRemoteDescription(sdp).get();
1244 addIceCandidatesFromBlackLog();
1245 org.webrtc.SessionDescription webRTCSessionDescription =
1246 this.webRTCWrapper.setLocalDescription().get();
1247 prepareSessionAccept(webRTCSessionDescription);
1248 } catch (final Exception e) {
1249 failureToAcceptSession(e);
1250 }
1251 }
1252
1253 private void failureToAcceptSession(final Throwable throwable) {
1254 if (isTerminated()) {
1255 return;
1256 }
1257 final Throwable rootCause = Throwables.getRootCause(throwable);
1258 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1259 webRTCWrapper.close();
1260 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1261 }
1262
1263 private void addIceCandidatesFromBlackLog() {
1264 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1265 while ((foo = this.pendingIceCandidates.poll()) != null) {
1266 processCandidate(foo);
1267 Log.d(
1268 Config.LOGTAG,
1269 id.account.getJid().asBareJid() + ": added candidate from back log");
1270 }
1271 }
1272
1273 private void prepareSessionAccept(
1274 final org.webrtc.SessionDescription webRTCSessionDescription) {
1275 final SessionDescription sessionDescription =
1276 SessionDescription.parse(webRTCSessionDescription.description);
1277 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1278 this.responderRtpContentMap = respondingRtpContentMap;
1279 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1280 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1281 prepareOutgoingContentMap(respondingRtpContentMap);
1282 Futures.addCallback(
1283 outgoingContentMapFuture,
1284 new FutureCallback<RtpContentMap>() {
1285 @Override
1286 public void onSuccess(final RtpContentMap outgoingContentMap) {
1287 sendSessionAccept(outgoingContentMap);
1288 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1289 }
1290
1291 @Override
1292 public void onFailure(@NonNull Throwable throwable) {
1293 failureToAcceptSession(throwable);
1294 }
1295 },
1296 MoreExecutors.directExecutor());
1297 }
1298
1299 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1300 if (isTerminated()) {
1301 Log.w(
1302 Config.LOGTAG,
1303 id.account.getJid().asBareJid()
1304 + ": preparing session accept was too slow. already terminated. nothing to do.");
1305 return;
1306 }
1307 transitionOrThrow(State.SESSION_ACCEPTED);
1308 final JinglePacket sessionAccept =
1309 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1310 send(sessionAccept);
1311 }
1312
1313 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1314 final RtpContentMap rtpContentMap) {
1315 if (this.omemoVerification.hasDeviceId()) {
1316 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1317 verifiedPayloadFuture =
1318 id.account
1319 .getAxolotlService()
1320 .encrypt(
1321 rtpContentMap,
1322 id.with,
1323 omemoVerification.getDeviceId());
1324 return Futures.transform(
1325 verifiedPayloadFuture,
1326 verifiedPayload -> {
1327 omemoVerification.setOrEnsureEqual(verifiedPayload);
1328 return verifiedPayload.getPayload();
1329 },
1330 MoreExecutors.directExecutor());
1331 } else {
1332 return Futures.immediateFuture(rtpContentMap);
1333 }
1334 }
1335
1336 synchronized void deliveryMessage(
1337 final Jid from,
1338 final Element message,
1339 final String serverMessageId,
1340 final long timestamp) {
1341 Log.d(
1342 Config.LOGTAG,
1343 id.account.getJid().asBareJid()
1344 + ": delivered message to JingleRtpConnection "
1345 + message);
1346 switch (message.getName()) {
1347 case "propose":
1348 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1349 break;
1350 case "proceed":
1351 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1352 break;
1353 case "retract":
1354 receiveRetract(from, serverMessageId, timestamp);
1355 break;
1356 case "reject":
1357 receiveReject(from, serverMessageId, timestamp);
1358 break;
1359 case "accept":
1360 receiveAccept(from, serverMessageId, timestamp);
1361 break;
1362 default:
1363 break;
1364 }
1365 }
1366
1367 void deliverFailedProceed(final String message) {
1368 Log.d(
1369 Config.LOGTAG,
1370 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1371 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1372 webRTCWrapper.close();
1373 Log.d(
1374 Config.LOGTAG,
1375 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1376 this.finish();
1377 }
1378 }
1379
1380 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1381 final boolean originatedFromMyself =
1382 from.asBareJid().equals(id.account.getJid().asBareJid());
1383 if (originatedFromMyself) {
1384 if (transition(State.ACCEPTED)) {
1385 acceptedOnOtherDevice(serverMsgId, timestamp);
1386 } else {
1387 Log.d(
1388 Config.LOGTAG,
1389 id.account.getJid().asBareJid()
1390 + ": unable to transition to accept because already in state="
1391 + this.state);
1392 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1393 }
1394 } else {
1395 Log.d(
1396 Config.LOGTAG,
1397 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1398 }
1399 }
1400
1401 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1402 if (serverMsgId != null) {
1403 this.message.setServerMsgId(serverMsgId);
1404 }
1405 this.message.setTime(timestamp);
1406 this.message.setCarbon(true); // indicate that call was accepted on other device
1407 this.writeLogMessageSuccess(0);
1408 this.xmppConnectionService
1409 .getNotificationService()
1410 .cancelIncomingCallNotification();
1411 this.finish();
1412 }
1413
1414 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1415 final boolean originatedFromMyself =
1416 from.asBareJid().equals(id.account.getJid().asBareJid());
1417 // reject from another one of my clients
1418 if (originatedFromMyself) {
1419 receiveRejectFromMyself(serverMsgId, timestamp);
1420 } else if (isInitiator()) {
1421 if (from.equals(id.with)) {
1422 receiveRejectFromResponder();
1423 } else {
1424 Log.d(
1425 Config.LOGTAG,
1426 id.account.getJid()
1427 + ": ignoring reject from "
1428 + from
1429 + " for session with "
1430 + id.with);
1431 }
1432 } else {
1433 Log.d(
1434 Config.LOGTAG,
1435 id.account.getJid()
1436 + ": ignoring reject from "
1437 + from
1438 + " for session with "
1439 + id.with);
1440 }
1441 }
1442
1443 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1444 if (transition(State.REJECTED)) {
1445 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1446 this.finish();
1447 if (serverMsgId != null) {
1448 this.message.setServerMsgId(serverMsgId);
1449 }
1450 this.message.setTime(timestamp);
1451 this.message.setCarbon(true); // indicate that call was rejected on other device
1452 writeLogMessageMissed();
1453 } else {
1454 Log.d(
1455 Config.LOGTAG,
1456 "not able to transition into REJECTED because already in " + this.state);
1457 }
1458 }
1459
1460 private void receiveRejectFromResponder() {
1461 if (isInState(State.PROCEED)) {
1462 Log.d(
1463 Config.LOGTAG,
1464 id.account.getJid()
1465 + ": received reject while still in proceed. callee reconsidered");
1466 closeTransitionLogFinish(State.REJECTED_RACED);
1467 return;
1468 }
1469 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1470 Log.d(
1471 Config.LOGTAG,
1472 id.account.getJid()
1473 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1474 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1475 return;
1476 }
1477 Log.d(
1478 Config.LOGTAG,
1479 id.account.getJid()
1480 + ": ignoring reject from responder because already in state "
1481 + this.state);
1482 }
1483
1484 private void receivePropose(
1485 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1486 final boolean originatedFromMyself =
1487 from.asBareJid().equals(id.account.getJid().asBareJid());
1488 if (originatedFromMyself) {
1489 Log.d(
1490 Config.LOGTAG,
1491 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1492 } else if (transition(
1493 State.PROPOSED,
1494 () -> {
1495 final Collection<RtpDescription> descriptions =
1496 Collections2.transform(
1497 Collections2.filter(
1498 propose.getDescriptions(),
1499 d -> d instanceof RtpDescription),
1500 input -> (RtpDescription) input);
1501 final Collection<Media> media =
1502 Collections2.transform(descriptions, RtpDescription::getMedia);
1503 Preconditions.checkState(
1504 !media.contains(Media.UNKNOWN),
1505 "RTP descriptions contain unknown media");
1506 Log.d(
1507 Config.LOGTAG,
1508 id.account.getJid().asBareJid()
1509 + ": received session proposal from "
1510 + from
1511 + " for "
1512 + media);
1513 this.proposedMedia = Sets.newHashSet(media);
1514 })) {
1515 if (serverMsgId != null) {
1516 this.message.setServerMsgId(serverMsgId);
1517 }
1518 this.message.setTime(timestamp);
1519 startRinging();
1520 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1521 sendJingleMessage("ringing");
1522 }
1523 } else {
1524 Log.d(
1525 Config.LOGTAG,
1526 id.account.getJid()
1527 + ": ignoring session proposal because already in "
1528 + state);
1529 }
1530 }
1531
1532 private void startRinging() {
1533 Log.d(
1534 Config.LOGTAG,
1535 id.account.getJid().asBareJid()
1536 + ": received call from "
1537 + id.with
1538 + ". start ringing");
1539 ringingTimeoutFuture =
1540 jingleConnectionManager.schedule(
1541 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1542 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1543 }
1544
1545 private synchronized void ringingTimeout() {
1546 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1547 switch (this.state) {
1548 case PROPOSED:
1549 message.markUnread();
1550 rejectCallFromProposed();
1551 break;
1552 case SESSION_INITIALIZED:
1553 message.markUnread();
1554 rejectCallFromSessionInitiate();
1555 break;
1556 }
1557 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1558 }
1559
1560 private void cancelRingingTimeout() {
1561 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1562 if (future != null && !future.isCancelled()) {
1563 future.cancel(false);
1564 }
1565 }
1566
1567 private void receiveProceed(
1568 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1569 final Set<Media> media =
1570 Preconditions.checkNotNull(
1571 this.proposedMedia, "Proposed media has to be set before handling proceed");
1572 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1573 if (from.equals(id.with)) {
1574 if (isInitiator()) {
1575 if (transition(State.PROCEED)) {
1576 if (serverMsgId != null) {
1577 this.message.setServerMsgId(serverMsgId);
1578 }
1579 this.message.setTime(timestamp);
1580 final Integer remoteDeviceId = proceed.getDeviceId();
1581 if (isOmemoEnabled()) {
1582 this.omemoVerification.setDeviceId(remoteDeviceId);
1583 } else {
1584 if (remoteDeviceId != null) {
1585 Log.d(
1586 Config.LOGTAG,
1587 id.account.getJid().asBareJid()
1588 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1589 }
1590 this.omemoVerification.setDeviceId(null);
1591 }
1592 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1593 } else {
1594 Log.d(
1595 Config.LOGTAG,
1596 String.format(
1597 "%s: ignoring proceed because already in %s",
1598 id.account.getJid().asBareJid(), this.state));
1599 }
1600 } else {
1601 Log.d(
1602 Config.LOGTAG,
1603 String.format(
1604 "%s: ignoring proceed because we were not initializing",
1605 id.account.getJid().asBareJid()));
1606 }
1607 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1608 if (transition(State.ACCEPTED)) {
1609 Log.d(
1610 Config.LOGTAG,
1611 id.account.getJid().asBareJid()
1612 + ": moved session with "
1613 + id.with
1614 + " into state accepted after received carbon copied proceed");
1615 acceptedOnOtherDevice(serverMsgId, timestamp);
1616 }
1617 } else {
1618 Log.d(
1619 Config.LOGTAG,
1620 String.format(
1621 "%s: ignoring proceed from %s. was expected from %s",
1622 id.account.getJid().asBareJid(), from, id.with));
1623 }
1624 }
1625
1626 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1627 if (from.equals(id.with)) {
1628 final State target =
1629 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1630 if (transition(target)) {
1631 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1632 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1633 Log.d(
1634 Config.LOGTAG,
1635 id.account.getJid().asBareJid()
1636 + ": session with "
1637 + id.with
1638 + " has been retracted (serverMsgId="
1639 + serverMsgId
1640 + ")");
1641 if (serverMsgId != null) {
1642 this.message.setServerMsgId(serverMsgId);
1643 }
1644 this.message.setTime(timestamp);
1645 if (target == State.RETRACTED) {
1646 this.message.markUnread();
1647 }
1648 writeLogMessageMissed();
1649 finish();
1650 } else {
1651 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1652 }
1653 } else {
1654 // TODO parse retract from self
1655 Log.d(
1656 Config.LOGTAG,
1657 id.account.getJid().asBareJid()
1658 + ": received retract from "
1659 + from
1660 + ". expected retract from"
1661 + id.with
1662 + ". ignoring");
1663 }
1664 }
1665
1666 public void sendSessionInitiate() {
1667 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1668 }
1669
1670 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1671 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1672 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1673 }
1674
1675 private synchronized void sendSessionInitiate(
1676 final Set<Media> media,
1677 final State targetState,
1678 final List<PeerConnection.IceServer> iceServers) {
1679 if (isTerminated()) {
1680 Log.w(
1681 Config.LOGTAG,
1682 id.account.getJid().asBareJid()
1683 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1684 return;
1685 }
1686 try {
1687 setupWebRTC(media, iceServers);
1688 } catch (final WebRTCWrapper.InitializationException e) {
1689 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1690 webRTCWrapper.close();
1691 sendRetract(Reason.ofThrowable(e));
1692 return;
1693 }
1694 try {
1695 org.webrtc.SessionDescription webRTCSessionDescription =
1696 this.webRTCWrapper.setLocalDescription().get();
1697 prepareSessionInitiate(webRTCSessionDescription, targetState);
1698 } catch (final Exception e) {
1699 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1700 // exceptions
1701 failureToInitiateSession(e, targetState);
1702 }
1703 }
1704
1705 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1706 if (isTerminated()) {
1707 return;
1708 }
1709 Log.d(
1710 Config.LOGTAG,
1711 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1712 Throwables.getRootCause(throwable));
1713 webRTCWrapper.close();
1714 final Reason reason = Reason.ofThrowable(throwable);
1715 if (isInState(targetState)) {
1716 sendSessionTerminate(reason, throwable.getMessage());
1717 } else {
1718 sendRetract(reason);
1719 }
1720 }
1721
1722 private void sendRetract(final Reason reason) {
1723 // TODO embed reason into retract
1724 sendJingleMessage("retract", id.with.asBareJid());
1725 transitionOrThrow(reasonToState(reason));
1726 this.finish();
1727 }
1728
1729 private void prepareSessionInitiate(
1730 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1731 final SessionDescription sessionDescription =
1732 SessionDescription.parse(webRTCSessionDescription.description);
1733 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1734 this.initiatorRtpContentMap = rtpContentMap;
1735 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1736 encryptSessionInitiate(rtpContentMap);
1737 Futures.addCallback(
1738 outgoingContentMapFuture,
1739 new FutureCallback<RtpContentMap>() {
1740 @Override
1741 public void onSuccess(final RtpContentMap outgoingContentMap) {
1742 sendSessionInitiate(outgoingContentMap, targetState);
1743 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1744 }
1745
1746 @Override
1747 public void onFailure(@NonNull final Throwable throwable) {
1748 failureToInitiateSession(throwable, targetState);
1749 }
1750 },
1751 MoreExecutors.directExecutor());
1752 }
1753
1754 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1755 if (isTerminated()) {
1756 Log.w(
1757 Config.LOGTAG,
1758 id.account.getJid().asBareJid()
1759 + ": preparing session was too slow. already terminated. nothing to do.");
1760 return;
1761 }
1762 this.transitionOrThrow(targetState);
1763 final JinglePacket sessionInitiate =
1764 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1765 send(sessionInitiate);
1766 }
1767
1768 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1769 final RtpContentMap rtpContentMap) {
1770 if (this.omemoVerification.hasDeviceId()) {
1771 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1772 verifiedPayloadFuture =
1773 id.account
1774 .getAxolotlService()
1775 .encrypt(
1776 rtpContentMap,
1777 id.with,
1778 omemoVerification.getDeviceId());
1779 final ListenableFuture<RtpContentMap> future =
1780 Futures.transform(
1781 verifiedPayloadFuture,
1782 verifiedPayload -> {
1783 omemoVerification.setSessionFingerprint(
1784 verifiedPayload.getFingerprint());
1785 return verifiedPayload.getPayload();
1786 },
1787 MoreExecutors.directExecutor());
1788 if (Config.REQUIRE_RTP_VERIFICATION) {
1789 return future;
1790 }
1791 return Futures.catching(
1792 future,
1793 CryptoFailedException.class,
1794 e -> {
1795 Log.w(
1796 Config.LOGTAG,
1797 id.account.getJid().asBareJid()
1798 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1799 e);
1800 return rtpContentMap;
1801 },
1802 MoreExecutors.directExecutor());
1803 } else {
1804 return Futures.immediateFuture(rtpContentMap);
1805 }
1806 }
1807
1808 private void sendSessionTerminate(final Reason reason) {
1809 sendSessionTerminate(reason, null);
1810 }
1811
1812 private void sendSessionTerminate(final Reason reason, final String text) {
1813 final State previous = this.state;
1814 final State target = reasonToState(reason);
1815 transitionOrThrow(target);
1816 if (previous != State.NULL) {
1817 writeLogMessage(target);
1818 }
1819 final JinglePacket jinglePacket =
1820 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1821 jinglePacket.setReason(reason, text);
1822 Log.d(Config.LOGTAG, jinglePacket.toString());
1823 send(jinglePacket);
1824 finish();
1825 }
1826
1827 private void sendTransportInfo(
1828 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1829 final RtpContentMap transportInfo;
1830 try {
1831 final RtpContentMap rtpContentMap =
1832 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1833 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1834 } catch (final Exception e) {
1835 Log.d(
1836 Config.LOGTAG,
1837 id.account.getJid().asBareJid()
1838 + ": unable to prepare transport-info from candidate for content="
1839 + contentName);
1840 return;
1841 }
1842 final JinglePacket jinglePacket =
1843 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1844 send(jinglePacket);
1845 }
1846
1847 private void send(final JinglePacket jinglePacket) {
1848 jinglePacket.setTo(id.with);
1849 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1850 }
1851
1852 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1853 if (response.getType() == IqPacket.TYPE.ERROR) {
1854 handleIqErrorResponse(response);
1855 return;
1856 }
1857 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1858 handleIqTimeoutResponse(response);
1859 }
1860 }
1861
1862 private void handleIqErrorResponse(final IqPacket response) {
1863 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1864 final String errorCondition = response.getErrorCondition();
1865 Log.d(
1866 Config.LOGTAG,
1867 id.account.getJid().asBareJid()
1868 + ": received IQ-error from "
1869 + response.getFrom()
1870 + " in RTP session. "
1871 + errorCondition);
1872 if (isTerminated()) {
1873 Log.i(
1874 Config.LOGTAG,
1875 id.account.getJid().asBareJid()
1876 + ": ignoring error because session was already terminated");
1877 return;
1878 }
1879 this.webRTCWrapper.close();
1880 final State target;
1881 if (Arrays.asList(
1882 "service-unavailable",
1883 "recipient-unavailable",
1884 "remote-server-not-found",
1885 "remote-server-timeout")
1886 .contains(errorCondition)) {
1887 target = State.TERMINATED_CONNECTIVITY_ERROR;
1888 } else {
1889 target = State.TERMINATED_APPLICATION_FAILURE;
1890 }
1891 transitionOrThrow(target);
1892 this.finish();
1893 }
1894
1895 private void handleIqTimeoutResponse(final IqPacket response) {
1896 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1897 Log.d(
1898 Config.LOGTAG,
1899 id.account.getJid().asBareJid()
1900 + ": received IQ timeout in RTP session with "
1901 + id.with
1902 + ". terminating with connectivity error");
1903 if (isTerminated()) {
1904 Log.i(
1905 Config.LOGTAG,
1906 id.account.getJid().asBareJid()
1907 + ": ignoring error because session was already terminated");
1908 return;
1909 }
1910 this.webRTCWrapper.close();
1911 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
1912 this.finish();
1913 }
1914
1915 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1916 Log.d(
1917 Config.LOGTAG,
1918 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1919 this.webRTCWrapper.close();
1920 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1921 respondWithOutOfOrder(jinglePacket);
1922 this.finish();
1923 }
1924
1925 private void respondWithTieBreak(final JinglePacket jinglePacket) {
1926 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
1927 }
1928
1929 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1930 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
1931 }
1932
1933 void respondWithJingleError(
1934 final IqPacket original,
1935 String jingleCondition,
1936 String condition,
1937 String conditionType) {
1938 jingleConnectionManager.respondWithJingleError(
1939 id.account, original, jingleCondition, condition, conditionType);
1940 }
1941
1942 private void respondOk(final JinglePacket jinglePacket) {
1943 xmppConnectionService.sendIqPacket(
1944 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1945 }
1946
1947 public RtpEndUserState getEndUserState() {
1948 switch (this.state) {
1949 case NULL:
1950 case PROPOSED:
1951 case SESSION_INITIALIZED:
1952 if (isInitiator()) {
1953 return RtpEndUserState.RINGING;
1954 } else {
1955 return RtpEndUserState.INCOMING_CALL;
1956 }
1957 case PROCEED:
1958 if (isInitiator()) {
1959 return RtpEndUserState.RINGING;
1960 } else {
1961 return RtpEndUserState.ACCEPTING_CALL;
1962 }
1963 case SESSION_INITIALIZED_PRE_APPROVED:
1964 if (isInitiator()) {
1965 return RtpEndUserState.RINGING;
1966 } else {
1967 return RtpEndUserState.CONNECTING;
1968 }
1969 case SESSION_ACCEPTED:
1970 final ContentAddition ca = getPendingContentAddition();
1971 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
1972 return RtpEndUserState.INCOMING_CONTENT_ADD;
1973 }
1974 return getPeerConnectionStateAsEndUserState();
1975 case REJECTED:
1976 case REJECTED_RACED:
1977 case TERMINATED_DECLINED_OR_BUSY:
1978 if (isInitiator()) {
1979 return RtpEndUserState.DECLINED_OR_BUSY;
1980 } else {
1981 return RtpEndUserState.ENDED;
1982 }
1983 case TERMINATED_SUCCESS:
1984 case ACCEPTED:
1985 case RETRACTED:
1986 case TERMINATED_CANCEL_OR_TIMEOUT:
1987 return RtpEndUserState.ENDED;
1988 case RETRACTED_RACED:
1989 if (isInitiator()) {
1990 return RtpEndUserState.ENDED;
1991 } else {
1992 return RtpEndUserState.RETRACTED;
1993 }
1994 case TERMINATED_CONNECTIVITY_ERROR:
1995 return zeroDuration()
1996 ? RtpEndUserState.CONNECTIVITY_ERROR
1997 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1998 case TERMINATED_APPLICATION_FAILURE:
1999 return RtpEndUserState.APPLICATION_ERROR;
2000 case TERMINATED_SECURITY_ERROR:
2001 return RtpEndUserState.SECURITY_ERROR;
2002 }
2003 throw new IllegalStateException(
2004 String.format("%s has no equivalent EndUserState", this.state));
2005 }
2006
2007 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2008 final PeerConnection.PeerConnectionState state;
2009 try {
2010 state = webRTCWrapper.getState();
2011 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2012 // We usually close the WebRTCWrapper *before* transitioning so we might still
2013 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2014 return RtpEndUserState.ENDING_CALL;
2015 }
2016 switch (state) {
2017 case CONNECTED:
2018 return RtpEndUserState.CONNECTED;
2019 case NEW:
2020 case CONNECTING:
2021 return RtpEndUserState.CONNECTING;
2022 case CLOSED:
2023 return RtpEndUserState.ENDING_CALL;
2024 default:
2025 return zeroDuration()
2026 ? RtpEndUserState.CONNECTIVITY_ERROR
2027 : RtpEndUserState.RECONNECTING;
2028 }
2029 }
2030
2031 public ContentAddition getPendingContentAddition() {
2032 final RtpContentMap in = this.incomingContentAdd;
2033 final RtpContentMap out = this.outgoingContentAdd;
2034 if (out != null) {
2035 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2036 } else if (in != null) {
2037 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2038 } else {
2039 return null;
2040 }
2041 }
2042
2043 public Set<Media> getMedia() {
2044 final State current = getState();
2045 if (current == State.NULL) {
2046 if (isInitiator()) {
2047 return Preconditions.checkNotNull(
2048 this.proposedMedia, "RTP connection has not been initialized properly");
2049 }
2050 throw new IllegalStateException("RTP connection has not been initialized yet");
2051 }
2052 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2053 return Preconditions.checkNotNull(
2054 this.proposedMedia, "RTP connection has not been initialized properly");
2055 }
2056 final RtpContentMap localContentMap = getLocalContentMap();
2057 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2058 if (localContentMap != null) {
2059 return localContentMap.getMedia();
2060 } else if (initiatorContentMap != null) {
2061 return initiatorContentMap.getMedia();
2062 } else if (isTerminated()) {
2063 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2064 } else {
2065 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2066 }
2067 }
2068
2069 public boolean isVerified() {
2070 final String fingerprint = this.omemoVerification.getFingerprint();
2071 if (fingerprint == null) {
2072 return false;
2073 }
2074 final FingerprintStatus status =
2075 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2076 return status != null && status.isVerified();
2077 }
2078
2079 public boolean addMedia(final Media media) {
2080 final Set<Media> currentMedia = getMedia();
2081 if (currentMedia.contains(media)) {
2082 throw new IllegalStateException(String.format("%s has already been proposed", media));
2083 }
2084 // TODO add state protection - can only add while ACCEPTED or so
2085 Log.d(Config.LOGTAG,"adding media: "+media);
2086 return webRTCWrapper.addTrack(media);
2087 }
2088
2089 public synchronized void acceptCall() {
2090 switch (this.state) {
2091 case PROPOSED:
2092 cancelRingingTimeout();
2093 acceptCallFromProposed();
2094 break;
2095 case SESSION_INITIALIZED:
2096 cancelRingingTimeout();
2097 acceptCallFromSessionInitialized();
2098 break;
2099 case ACCEPTED:
2100 Log.w(
2101 Config.LOGTAG,
2102 id.account.getJid().asBareJid()
2103 + ": the call has already been accepted with another client. UI was just lagging behind");
2104 break;
2105 case PROCEED:
2106 case SESSION_ACCEPTED:
2107 Log.w(
2108 Config.LOGTAG,
2109 id.account.getJid().asBareJid()
2110 + ": the call has already been accepted. user probably double tapped the UI");
2111 break;
2112 default:
2113 throw new IllegalStateException("Can not accept call from " + this.state);
2114 }
2115 }
2116
2117 public void notifyPhoneCall() {
2118 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2119 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2120 rejectCall();
2121 } else {
2122 endCall();
2123 }
2124 }
2125
2126 public synchronized void rejectCall() {
2127 if (isTerminated()) {
2128 Log.w(
2129 Config.LOGTAG,
2130 id.account.getJid().asBareJid()
2131 + ": received rejectCall() when session has already been terminated. nothing to do");
2132 return;
2133 }
2134 switch (this.state) {
2135 case PROPOSED:
2136 rejectCallFromProposed();
2137 break;
2138 case SESSION_INITIALIZED:
2139 rejectCallFromSessionInitiate();
2140 break;
2141 default:
2142 throw new IllegalStateException("Can not reject call from " + this.state);
2143 }
2144 }
2145
2146 public synchronized void endCall() {
2147 if (isTerminated()) {
2148 Log.w(
2149 Config.LOGTAG,
2150 id.account.getJid().asBareJid()
2151 + ": received endCall() when session has already been terminated. nothing to do");
2152 return;
2153 }
2154 if (isInState(State.PROPOSED) && !isInitiator()) {
2155 rejectCallFromProposed();
2156 return;
2157 }
2158 if (isInState(State.PROCEED)) {
2159 if (isInitiator()) {
2160 retractFromProceed();
2161 } else {
2162 rejectCallFromProceed();
2163 }
2164 return;
2165 }
2166 if (isInitiator()
2167 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2168 this.webRTCWrapper.close();
2169 sendSessionTerminate(Reason.CANCEL);
2170 return;
2171 }
2172 if (isInState(State.SESSION_INITIALIZED)) {
2173 rejectCallFromSessionInitiate();
2174 return;
2175 }
2176 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2177 this.webRTCWrapper.close();
2178 sendSessionTerminate(Reason.SUCCESS);
2179 return;
2180 }
2181 if (isInState(
2182 State.TERMINATED_APPLICATION_FAILURE,
2183 State.TERMINATED_CONNECTIVITY_ERROR,
2184 State.TERMINATED_DECLINED_OR_BUSY)) {
2185 Log.d(
2186 Config.LOGTAG,
2187 "ignoring request to end call because already in state " + this.state);
2188 return;
2189 }
2190 throw new IllegalStateException(
2191 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2192 }
2193
2194 private void retractFromProceed() {
2195 Log.d(Config.LOGTAG, "retract from proceed");
2196 this.sendJingleMessage("retract");
2197 closeTransitionLogFinish(State.RETRACTED_RACED);
2198 }
2199
2200 private void closeTransitionLogFinish(final State state) {
2201 this.webRTCWrapper.close();
2202 transitionOrThrow(state);
2203 writeLogMessage(state);
2204 finish();
2205 }
2206
2207 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2208 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2209 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2210 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2211 }
2212
2213 private void acceptCallFromProposed() {
2214 transitionOrThrow(State.PROCEED);
2215 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2216 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2217 this.sendJingleMessage("proceed");
2218 }
2219
2220 private void rejectCallFromProposed() {
2221 transitionOrThrow(State.REJECTED);
2222 writeLogMessageMissed();
2223 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2224 this.sendJingleMessage("reject");
2225 finish();
2226 }
2227
2228 private void rejectCallFromProceed() {
2229 this.sendJingleMessage("reject");
2230 closeTransitionLogFinish(State.REJECTED_RACED);
2231 }
2232
2233 private void rejectCallFromSessionInitiate() {
2234 webRTCWrapper.close();
2235 sendSessionTerminate(Reason.DECLINE);
2236 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2237 }
2238
2239 private void sendJingleMessage(final String action) {
2240 sendJingleMessage(action, id.with);
2241 }
2242
2243 private void sendJingleMessage(final String action, final Jid to) {
2244 final MessagePacket messagePacket = new MessagePacket();
2245 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2246 messagePacket.setTo(to);
2247 final Element intent =
2248 messagePacket
2249 .addChild(action, Namespace.JINGLE_MESSAGE)
2250 .setAttribute("id", id.sessionId);
2251 if ("proceed".equals(action)) {
2252 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2253 if (isOmemoEnabled()) {
2254 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2255 final Element device =
2256 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2257 device.setAttribute("id", deviceId);
2258 }
2259 }
2260 messagePacket.addChild("store", "urn:xmpp:hints");
2261 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2262 }
2263
2264 private boolean isOmemoEnabled() {
2265 final Conversational conversational = message.getConversation();
2266 if (conversational instanceof Conversation) {
2267 return ((Conversation) conversational).getNextEncryption()
2268 == Message.ENCRYPTION_AXOLOTL;
2269 }
2270 return false;
2271 }
2272
2273 private void acceptCallFromSessionInitialized() {
2274 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2275 sendSessionAccept();
2276 }
2277
2278 private synchronized boolean isInState(State... state) {
2279 return Arrays.asList(state).contains(this.state);
2280 }
2281
2282 private boolean transition(final State target) {
2283 return transition(target, null);
2284 }
2285
2286 private synchronized boolean transition(final State target, final Runnable runnable) {
2287 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2288 if (validTransitions != null && validTransitions.contains(target)) {
2289 this.state = target;
2290 if (runnable != null) {
2291 runnable.run();
2292 }
2293 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2294 updateEndUserState();
2295 updateOngoingCallNotification();
2296 return true;
2297 } else {
2298 return false;
2299 }
2300 }
2301
2302 void transitionOrThrow(final State target) {
2303 if (!transition(target)) {
2304 throw new IllegalStateException(
2305 String.format("Unable to transition from %s to %s", this.state, target));
2306 }
2307 }
2308
2309 @Override
2310 public void onIceCandidate(final IceCandidate iceCandidate) {
2311 final RtpContentMap rtpContentMap =
2312 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2313 final IceUdpTransportInfo.Credentials credentials;
2314 try {
2315 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2316 } catch (final IllegalArgumentException e) {
2317 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2318 return;
2319 }
2320 final String uFrag = credentials.ufrag;
2321 final IceUdpTransportInfo.Candidate candidate =
2322 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2323 if (candidate == null) {
2324 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2325 return;
2326 }
2327 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2328 sendTransportInfo(iceCandidate.sdpMid, candidate);
2329 }
2330
2331 @Override
2332 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2333 Log.d(
2334 Config.LOGTAG,
2335 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2336 this.stateHistory.add(newState);
2337 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2338 this.sessionDuration.start();
2339 updateOngoingCallNotification();
2340 } else if (this.sessionDuration.isRunning()) {
2341 this.sessionDuration.stop();
2342 updateOngoingCallNotification();
2343 }
2344
2345 final boolean neverConnected =
2346 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2347
2348 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2349 if (neverConnected) {
2350 if (isTerminated()) {
2351 Log.d(
2352 Config.LOGTAG,
2353 id.account.getJid().asBareJid()
2354 + ": not sending session-terminate after connectivity error because session is already in state "
2355 + this.state);
2356 return;
2357 }
2358 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2359 return;
2360 } else {
2361 this.restartIce();
2362 }
2363 }
2364 updateEndUserState();
2365 }
2366
2367 private void restartIce() {
2368 this.stateHistory.clear();
2369 this.webRTCWrapper.restartIceAsync();
2370 }
2371
2372 @Override
2373 public void onRenegotiationNeeded() {
2374 this.webRTCWrapper.execute(this::renegotiate);
2375 }
2376
2377 private void renegotiate() {
2378 final SessionDescription sessionDescription;
2379 try {
2380 sessionDescription = setLocalSessionDescription();
2381 } catch (final Exception e) {
2382 final Throwable cause = Throwables.getRootCause(e);
2383 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2384 webRTCWrapper.close();
2385 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2386 return;
2387 }
2388 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2389 final RtpContentMap currentContentMap = getLocalContentMap();
2390 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2391 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2392
2393 Log.d(
2394 Config.LOGTAG,
2395 id.getAccount().getJid().asBareJid()
2396 + ": renegotiate. iceRestart="
2397 + iceRestart
2398 + " content id diff="
2399 + diff);
2400
2401 if (diff.hasModifications() && iceRestart) {
2402 webRTCWrapper.close();
2403 sendSessionTerminate(
2404 Reason.FAILED_APPLICATION,
2405 "WebRTC unexpectedly tried to modify content and transport at once");
2406 return;
2407 }
2408
2409 if (iceRestart) {
2410 initiateIceRestart(rtpContentMap);
2411 return;
2412 } else if (diff.isEmpty()) {
2413 Log.d(
2414 Config.LOGTAG,
2415 "renegotiation. nothing to do. SignalingState="
2416 + this.webRTCWrapper.getSignalingState());
2417 }
2418
2419 if (diff.added.size() > 0) {
2420 modifyLocalContentMap(rtpContentMap);
2421 sendContentAdd(rtpContentMap, diff.added);
2422 }
2423 }
2424
2425 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2426 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2427 final JinglePacket jinglePacket =
2428 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2429 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2430 jinglePacket.setTo(id.with);
2431 xmppConnectionService.sendIqPacket(
2432 id.account,
2433 jinglePacket,
2434 (account, response) -> {
2435 if (response.getType() == IqPacket.TYPE.RESULT) {
2436 Log.d(Config.LOGTAG, "received success to our ice restart");
2437 setLocalContentMap(rtpContentMap);
2438 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2439 return;
2440 }
2441 if (response.getType() == IqPacket.TYPE.ERROR) {
2442 if (isTieBreak(response)) {
2443 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2444 return;
2445 }
2446 handleIqErrorResponse(response);
2447 }
2448 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2449 handleIqTimeoutResponse(response);
2450 }
2451 });
2452 }
2453
2454 private boolean isTieBreak(final IqPacket response) {
2455 final Element error = response.findChild("error");
2456 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2457 }
2458
2459 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2460 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2461 this.outgoingContentAdd = contentAdd;
2462 final JinglePacket jinglePacket =
2463 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2464 jinglePacket.setTo(id.with);
2465 xmppConnectionService.sendIqPacket(
2466 id.account,
2467 jinglePacket,
2468 (connection, response) -> {
2469 if (response.getType() == IqPacket.TYPE.RESULT) {
2470 Log.d(
2471 Config.LOGTAG,
2472 id.getAccount().getJid().asBareJid()
2473 + ": received ACK to our content-add");
2474 return;
2475 }
2476 if (response.getType() == IqPacket.TYPE.ERROR) {
2477 if (isTieBreak(response)) {
2478 this.outgoingContentAdd = null;
2479 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2480 return;
2481 }
2482 handleIqErrorResponse(response);
2483 }
2484 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2485 handleIqTimeoutResponse(response);
2486 }
2487 });
2488 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2489 }
2490
2491 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2492 if (isInitiator()) {
2493 this.initiatorRtpContentMap = rtpContentMap;
2494 } else {
2495 this.responderRtpContentMap = rtpContentMap;
2496 }
2497 }
2498
2499 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2500 if (isInitiator()) {
2501 this.responderRtpContentMap = rtpContentMap;
2502 } else {
2503 this.initiatorRtpContentMap = rtpContentMap;
2504 }
2505 }
2506
2507 // this method is to be used for content map modifications that modify media
2508 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2509 final RtpContentMap activeContents = rtpContentMap.activeContents();
2510 setLocalContentMap(activeContents);
2511 this.webRTCWrapper.switchSpeakerPhonePreference(
2512 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2513 updateEndUserState();
2514 }
2515
2516 private SessionDescription setLocalSessionDescription()
2517 throws ExecutionException, InterruptedException {
2518 final org.webrtc.SessionDescription sessionDescription =
2519 this.webRTCWrapper.setLocalDescription().get();
2520 return SessionDescription.parse(sessionDescription.description);
2521 }
2522
2523 private void closeWebRTCSessionAfterFailedConnection() {
2524 this.webRTCWrapper.close();
2525 synchronized (this) {
2526 if (isTerminated()) {
2527 Log.d(
2528 Config.LOGTAG,
2529 id.account.getJid().asBareJid()
2530 + ": no need to send session-terminate after failed connection. Other party already did");
2531 return;
2532 }
2533 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2534 }
2535 }
2536
2537 public boolean zeroDuration() {
2538 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2539 }
2540
2541 public long getCallDuration() {
2542 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2543 }
2544
2545 public AppRTCAudioManager getAudioManager() {
2546 return webRTCWrapper.getAudioManager();
2547 }
2548
2549 public boolean isMicrophoneEnabled() {
2550 return webRTCWrapper.isMicrophoneEnabled();
2551 }
2552
2553 public boolean setMicrophoneEnabled(final boolean enabled) {
2554 return webRTCWrapper.setMicrophoneEnabled(enabled);
2555 }
2556
2557 public boolean isVideoEnabled() {
2558 return webRTCWrapper.isVideoEnabled();
2559 }
2560
2561 public void setVideoEnabled(final boolean enabled) {
2562 webRTCWrapper.setVideoEnabled(enabled);
2563 }
2564
2565 public boolean isCameraSwitchable() {
2566 return webRTCWrapper.isCameraSwitchable();
2567 }
2568
2569 public boolean isFrontCamera() {
2570 return webRTCWrapper.isFrontCamera();
2571 }
2572
2573 public ListenableFuture<Boolean> switchCamera() {
2574 return webRTCWrapper.switchCamera();
2575 }
2576
2577 @Override
2578 public void onAudioDeviceChanged(
2579 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2580 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2581 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2582 selectedAudioDevice, availableAudioDevices);
2583 }
2584
2585 private void updateEndUserState() {
2586 final RtpEndUserState endUserState = getEndUserState();
2587 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2588 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2589 id.account, id.with, id.sessionId, endUserState);
2590 }
2591
2592 private void updateOngoingCallNotification() {
2593 final State state = this.state;
2594 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2595 final boolean reconnecting;
2596 if (state == State.SESSION_ACCEPTED) {
2597 reconnecting =
2598 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2599 } else {
2600 reconnecting = false;
2601 }
2602 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2603 } else {
2604 xmppConnectionService.removeOngoingCall();
2605 }
2606 }
2607
2608 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2609 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2610 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2611 request.setTo(id.account.getDomain());
2612 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2613 xmppConnectionService.sendIqPacket(
2614 id.account,
2615 request,
2616 (account, response) -> {
2617 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2618 new ImmutableList.Builder<>();
2619 if (response.getType() == IqPacket.TYPE.RESULT) {
2620 final Element services =
2621 response.findChild(
2622 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2623 final List<Element> children =
2624 services == null
2625 ? Collections.emptyList()
2626 : services.getChildren();
2627 for (final Element child : children) {
2628 if ("service".equals(child.getName())) {
2629 final String type = child.getAttribute("type");
2630 final String host = child.getAttribute("host");
2631 final String sport = child.getAttribute("port");
2632 final Integer port =
2633 sport == null ? null : Ints.tryParse(sport);
2634 final String transport = child.getAttribute("transport");
2635 final String username = child.getAttribute("username");
2636 final String password = child.getAttribute("password");
2637 if (Strings.isNullOrEmpty(host) || port == null) {
2638 continue;
2639 }
2640 if (port < 0 || port > 65535) {
2641 continue;
2642 }
2643
2644
2645
2646
2647 if (Arrays.asList("stun", "stuns", "turn", "turns")
2648 .contains(type)
2649 && Arrays.asList("udp", "tcp").contains(transport)) {
2650 if (Arrays.asList("stuns", "turns").contains(type)
2651 && "udp".equals(transport)) {
2652 Log.d(
2653 Config.LOGTAG,
2654 id.account.getJid().asBareJid()
2655 + ": skipping invalid combination of udp/tls in external services");
2656 continue;
2657 }
2658
2659 // STUN URLs do not support a query section since M110
2660 final String uri;
2661 if (Arrays.asList("stun","stuns").contains(type)) {
2662 uri = String.format("%s:%s:%s", type, IP.wrapIPv6(host),port);
2663 } else {
2664 uri = String.format(
2665 "%s:%s:%s?transport=%s",
2666 type,
2667 IP.wrapIPv6(host),
2668 port,
2669 transport);
2670 }
2671
2672 final PeerConnection.IceServer.Builder iceServerBuilder =
2673 PeerConnection.IceServer.builder(uri);
2674 iceServerBuilder.setTlsCertPolicy(
2675 PeerConnection.TlsCertPolicy
2676 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2677 if (username != null && password != null) {
2678 iceServerBuilder.setUsername(username);
2679 iceServerBuilder.setPassword(password);
2680 } else if (Arrays.asList("turn", "turns").contains(type)) {
2681 // The WebRTC spec requires throwing an
2682 // InvalidAccessError when username (from libwebrtc
2683 // source coder)
2684 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2685 Log.d(
2686 Config.LOGTAG,
2687 id.account.getJid().asBareJid()
2688 + ": skipping "
2689 + type
2690 + "/"
2691 + transport
2692 + " without username and password");
2693 continue;
2694 }
2695 final PeerConnection.IceServer iceServer =
2696 iceServerBuilder.createIceServer();
2697 Log.d(
2698 Config.LOGTAG,
2699 id.account.getJid().asBareJid()
2700 + ": discovered ICE Server: "
2701 + iceServer);
2702 listBuilder.add(iceServer);
2703 }
2704 }
2705 }
2706 }
2707 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2708 if (iceServers.size() == 0) {
2709 Log.w(
2710 Config.LOGTAG,
2711 id.account.getJid().asBareJid()
2712 + ": no ICE server found "
2713 + response);
2714 }
2715 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2716 });
2717 } else {
2718 Log.w(
2719 Config.LOGTAG,
2720 id.account.getJid().asBareJid() + ": has no external service discovery");
2721 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2722 }
2723 }
2724
2725 private void finish() {
2726 if (isTerminated()) {
2727 this.cancelRingingTimeout();
2728 this.webRTCWrapper.verifyClosed();
2729 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2730 this.jingleConnectionManager.finishConnectionOrThrow(this);
2731 } else {
2732 throw new IllegalStateException(
2733 String.format("Unable to call finish from %s", this.state));
2734 }
2735 }
2736
2737 private void writeLogMessage(final State state) {
2738 final long duration = getCallDuration();
2739 if (state == State.TERMINATED_SUCCESS
2740 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2741 writeLogMessageSuccess(duration);
2742 } else {
2743 writeLogMessageMissed();
2744 }
2745 }
2746
2747 private void writeLogMessageSuccess(final long duration) {
2748 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2749 this.writeMessage();
2750 }
2751
2752 private void writeLogMessageMissed() {
2753 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2754 this.writeMessage();
2755 }
2756
2757 private void writeMessage() {
2758 final Conversational conversational = message.getConversation();
2759 if (conversational instanceof Conversation) {
2760 ((Conversation) conversational).add(this.message);
2761 xmppConnectionService.createMessageAsync(message);
2762 xmppConnectionService.updateConversationUi();
2763 } else {
2764 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2765 }
2766 }
2767
2768 public State getState() {
2769 return this.state;
2770 }
2771
2772 boolean isTerminated() {
2773 return TERMINATED.contains(this.state);
2774 }
2775
2776 public Optional<VideoTrack> getLocalVideoTrack() {
2777 return webRTCWrapper.getLocalVideoTrack();
2778 }
2779
2780 public Optional<VideoTrack> getRemoteVideoTrack() {
2781 return webRTCWrapper.getRemoteVideoTrack();
2782 }
2783
2784 public EglBase.Context getEglBaseContext() {
2785 return webRTCWrapper.getEglBaseContext();
2786 }
2787
2788 void setProposedMedia(final Set<Media> media) {
2789 this.proposedMedia = media;
2790 }
2791
2792 public void fireStateUpdate() {
2793 final RtpEndUserState endUserState = getEndUserState();
2794 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2795 id.account, id.with, id.sessionId, endUserState);
2796 }
2797
2798 public boolean isSwitchToVideoAvailable() {
2799 final boolean prerequisite =
2800 Media.audioOnly(getMedia())
2801 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2802 .contains(getEndUserState());
2803 return prerequisite && remoteHasVideoFeature();
2804 }
2805
2806 private boolean remoteHasVideoFeature() {
2807 final Contact contact = id.getContact();
2808 final Presence presence =
2809 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2810 final ServiceDiscoveryResult serviceDiscoveryResult =
2811 presence == null ? null : presence.getServiceDiscoveryResult();
2812 final List<String> features =
2813 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
2814 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
2815 }
2816
2817 private interface OnIceServersDiscovered {
2818 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2819 }
2820}