1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Joiner;
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.base.Stopwatch;
12import com.google.common.base.Strings;
13import com.google.common.base.Throwables;
14import com.google.common.collect.Collections2;
15import com.google.common.collect.ImmutableList;
16import com.google.common.collect.ImmutableMap;
17import com.google.common.collect.ImmutableSet;
18import com.google.common.collect.Maps;
19import com.google.common.collect.Sets;
20import com.google.common.primitives.Ints;
21import com.google.common.util.concurrent.FutureCallback;
22import com.google.common.util.concurrent.Futures;
23import com.google.common.util.concurrent.ListenableFuture;
24import com.google.common.util.concurrent.MoreExecutors;
25
26import org.webrtc.EglBase;
27import org.webrtc.IceCandidate;
28import org.webrtc.PeerConnection;
29import org.webrtc.VideoTrack;
30
31import java.util.Arrays;
32import java.util.Collection;
33import java.util.Collections;
34import java.util.LinkedList;
35import java.util.List;
36import java.util.Map;
37import java.util.Queue;
38import java.util.Set;
39import java.util.concurrent.ExecutionException;
40import java.util.concurrent.ScheduledFuture;
41import java.util.concurrent.TimeUnit;
42
43import eu.siacs.conversations.BuildConfig;
44import eu.siacs.conversations.Config;
45import eu.siacs.conversations.crypto.axolotl.AxolotlService;
46import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
47import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
48import eu.siacs.conversations.entities.Account;
49import eu.siacs.conversations.entities.Contact;
50import eu.siacs.conversations.entities.Conversation;
51import eu.siacs.conversations.entities.Conversational;
52import eu.siacs.conversations.entities.Message;
53import eu.siacs.conversations.entities.Presence;
54import eu.siacs.conversations.entities.RtpSessionStatus;
55import eu.siacs.conversations.entities.ServiceDiscoveryResult;
56import eu.siacs.conversations.services.AppRTCAudioManager;
57import eu.siacs.conversations.utils.IP;
58import eu.siacs.conversations.xml.Element;
59import eu.siacs.conversations.xml.Namespace;
60import eu.siacs.conversations.xmpp.Jid;
61import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
62import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
63import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
64import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
65import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
66import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
67import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
68import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
69import eu.siacs.conversations.xmpp.stanzas.IqPacket;
70import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
71
72public class JingleRtpConnection extends AbstractJingleConnection
73 implements WebRTCWrapper.EventCallback {
74
75 public static final List<State> STATES_SHOWING_ONGOING_CALL =
76 Arrays.asList(
77 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
78 private static final long BUSY_TIME_OUT = 30;
79 private static final List<State> TERMINATED =
80 Arrays.asList(
81 State.ACCEPTED,
82 State.REJECTED,
83 State.REJECTED_RACED,
84 State.RETRACTED,
85 State.RETRACTED_RACED,
86 State.TERMINATED_SUCCESS,
87 State.TERMINATED_DECLINED_OR_BUSY,
88 State.TERMINATED_CONNECTIVITY_ERROR,
89 State.TERMINATED_CANCEL_OR_TIMEOUT,
90 State.TERMINATED_APPLICATION_FAILURE,
91 State.TERMINATED_SECURITY_ERROR);
92
93 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
94
95 static {
96 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
97 new ImmutableMap.Builder<>();
98 transitionBuilder.put(
99 State.NULL,
100 ImmutableList.of(
101 State.PROPOSED,
102 State.SESSION_INITIALIZED,
103 State.TERMINATED_APPLICATION_FAILURE,
104 State.TERMINATED_SECURITY_ERROR));
105 transitionBuilder.put(
106 State.PROPOSED,
107 ImmutableList.of(
108 State.ACCEPTED,
109 State.PROCEED,
110 State.REJECTED,
111 State.RETRACTED,
112 State.TERMINATED_APPLICATION_FAILURE,
113 State.TERMINATED_SECURITY_ERROR,
114 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
115 // rebinds
116 ));
117 transitionBuilder.put(
118 State.PROCEED,
119 ImmutableList.of(
120 State.REJECTED_RACED,
121 State.RETRACTED_RACED,
122 State.SESSION_INITIALIZED_PRE_APPROVED,
123 State.TERMINATED_SUCCESS,
124 State.TERMINATED_APPLICATION_FAILURE,
125 State.TERMINATED_SECURITY_ERROR,
126 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
127 // bounces of the proceed message
128 ));
129 transitionBuilder.put(
130 State.SESSION_INITIALIZED,
131 ImmutableList.of(
132 State.SESSION_ACCEPTED,
133 State.TERMINATED_SUCCESS,
134 State.TERMINATED_DECLINED_OR_BUSY,
135 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
136 // and IQ timeouts
137 State.TERMINATED_CANCEL_OR_TIMEOUT,
138 State.TERMINATED_APPLICATION_FAILURE,
139 State.TERMINATED_SECURITY_ERROR));
140 transitionBuilder.put(
141 State.SESSION_INITIALIZED_PRE_APPROVED,
142 ImmutableList.of(
143 State.SESSION_ACCEPTED,
144 State.TERMINATED_SUCCESS,
145 State.TERMINATED_DECLINED_OR_BUSY,
146 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
147 // and IQ timeouts
148 State.TERMINATED_CANCEL_OR_TIMEOUT,
149 State.TERMINATED_APPLICATION_FAILURE,
150 State.TERMINATED_SECURITY_ERROR));
151 transitionBuilder.put(
152 State.SESSION_ACCEPTED,
153 ImmutableList.of(
154 State.TERMINATED_SUCCESS,
155 State.TERMINATED_DECLINED_OR_BUSY,
156 State.TERMINATED_CONNECTIVITY_ERROR,
157 State.TERMINATED_CANCEL_OR_TIMEOUT,
158 State.TERMINATED_APPLICATION_FAILURE,
159 State.TERMINATED_SECURITY_ERROR));
160 VALID_TRANSITIONS = transitionBuilder.build();
161 }
162
163 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
164 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
165 pendingIceCandidates = new LinkedList<>();
166 private final OmemoVerification omemoVerification = new OmemoVerification();
167 private final Message message;
168 private State state = State.NULL;
169 private Set<Media> proposedMedia;
170 private RtpContentMap initiatorRtpContentMap;
171 private RtpContentMap responderRtpContentMap;
172 private RtpContentMap incomingContentAdd;
173 private RtpContentMap outgoingContentAdd;
174 private IceUdpTransportInfo.Setup peerDtlsSetup;
175 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
176 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
177 private ScheduledFuture<?> ringingTimeoutFuture;
178
179 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
180 super(jingleConnectionManager, id, initiator);
181 final Conversation conversation =
182 jingleConnectionManager
183 .getXmppConnectionService()
184 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
185 this.message =
186 new Message(
187 conversation,
188 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
189 Message.TYPE_RTP_SESSION,
190 id.sessionId);
191 }
192
193 private static State reasonToState(Reason reason) {
194 switch (reason) {
195 case SUCCESS:
196 return State.TERMINATED_SUCCESS;
197 case DECLINE:
198 case BUSY:
199 return State.TERMINATED_DECLINED_OR_BUSY;
200 case CANCEL:
201 case TIMEOUT:
202 return State.TERMINATED_CANCEL_OR_TIMEOUT;
203 case SECURITY_ERROR:
204 return State.TERMINATED_SECURITY_ERROR;
205 case FAILED_APPLICATION:
206 case UNSUPPORTED_TRANSPORTS:
207 case UNSUPPORTED_APPLICATIONS:
208 return State.TERMINATED_APPLICATION_FAILURE;
209 default:
210 return State.TERMINATED_CONNECTIVITY_ERROR;
211 }
212 }
213
214 @Override
215 synchronized void deliverPacket(final JinglePacket jinglePacket) {
216 switch (jinglePacket.getAction()) {
217 case SESSION_INITIATE:
218 receiveSessionInitiate(jinglePacket);
219 break;
220 case TRANSPORT_INFO:
221 receiveTransportInfo(jinglePacket);
222 break;
223 case SESSION_ACCEPT:
224 receiveSessionAccept(jinglePacket);
225 break;
226 case SESSION_TERMINATE:
227 receiveSessionTerminate(jinglePacket);
228 break;
229 case CONTENT_ADD:
230 receiveContentAdd(jinglePacket);
231 break;
232 case CONTENT_ACCEPT:
233 receiveContentAccept(jinglePacket);
234 break;
235 case CONTENT_REJECT:
236 receiveContentReject(jinglePacket);
237 break;
238 case CONTENT_REMOVE:
239 receiveContentRemove(jinglePacket);
240 break;
241 case CONTENT_MODIFY:
242 receiveContentModify(jinglePacket);
243 break;
244 default:
245 respondOk(jinglePacket);
246 Log.d(
247 Config.LOGTAG,
248 String.format(
249 "%s: received unhandled jingle action %s",
250 id.account.getJid().asBareJid(), jinglePacket.getAction()));
251 break;
252 }
253 }
254
255 @Override
256 synchronized void notifyRebound() {
257 if (isTerminated()) {
258 return;
259 }
260 webRTCWrapper.close();
261 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
262 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
263 }
264 if (isInState(
265 State.SESSION_INITIALIZED,
266 State.SESSION_INITIALIZED_PRE_APPROVED,
267 State.SESSION_ACCEPTED)) {
268 // we might have already changed resources (full jid) at this point; so this might not
269 // even reach the other party
270 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
271 } else {
272 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
273 finish();
274 }
275 }
276
277 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
278 respondOk(jinglePacket);
279 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
280 final State previous = this.state;
281 Log.d(
282 Config.LOGTAG,
283 id.account.getJid().asBareJid()
284 + ": received session terminate reason="
285 + wrapper.reason
286 + "("
287 + Strings.nullToEmpty(wrapper.text)
288 + ") while in state "
289 + previous);
290 if (TERMINATED.contains(previous)) {
291 Log.d(
292 Config.LOGTAG,
293 id.account.getJid().asBareJid()
294 + ": ignoring session terminate because already in "
295 + previous);
296 return;
297 }
298 webRTCWrapper.close();
299 final State target = reasonToState(wrapper.reason);
300 transitionOrThrow(target);
301 writeLogMessage(target);
302 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
303 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
304 }
305 finish();
306 }
307
308 private void receiveTransportInfo(final JinglePacket jinglePacket) {
309 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
310 // INITIALIZED only after transport-info has been received
311 if (isInState(
312 State.NULL,
313 State.PROCEED,
314 State.SESSION_INITIALIZED,
315 State.SESSION_INITIALIZED_PRE_APPROVED,
316 State.SESSION_ACCEPTED)) {
317 final RtpContentMap contentMap;
318 try {
319 contentMap = RtpContentMap.of(jinglePacket);
320 } catch (final IllegalArgumentException | NullPointerException e) {
321 Log.d(
322 Config.LOGTAG,
323 id.account.getJid().asBareJid()
324 + ": improperly formatted contents; ignoring",
325 e);
326 respondOk(jinglePacket);
327 return;
328 }
329 receiveTransportInfo(jinglePacket, contentMap);
330 } else {
331 if (isTerminated()) {
332 respondOk(jinglePacket);
333 Log.d(
334 Config.LOGTAG,
335 id.account.getJid().asBareJid()
336 + ": ignoring out-of-order transport info; we where already terminated");
337 } else {
338 Log.d(
339 Config.LOGTAG,
340 id.account.getJid().asBareJid()
341 + ": received transport info while in state="
342 + this.state);
343 terminateWithOutOfOrder(jinglePacket);
344 }
345 }
346 }
347
348 private void receiveTransportInfo(
349 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
350 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
351 contentMap.contents.entrySet();
352 if (this.state == State.SESSION_ACCEPTED) {
353 // zero candidates + modified credentials are an ICE restart offer
354 if (checkForIceRestart(jinglePacket, contentMap)) {
355 return;
356 }
357 respondOk(jinglePacket);
358 try {
359 processCandidates(candidates);
360 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
361 Log.w(
362 Config.LOGTAG,
363 id.account.getJid().asBareJid()
364 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
365 }
366 } else {
367 respondOk(jinglePacket);
368 pendingIceCandidates.addAll(candidates);
369 }
370 }
371
372 private void receiveContentAdd(final JinglePacket jinglePacket) {
373 final RtpContentMap modification;
374 try {
375 modification = RtpContentMap.of(jinglePacket);
376 modification.requireContentDescriptions();
377 } catch (final RuntimeException e) {
378 Log.d(
379 Config.LOGTAG,
380 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
381 Throwables.getRootCause(e));
382 respondOk(jinglePacket);
383 webRTCWrapper.close();
384 sendSessionTerminate(Reason.of(e), e.getMessage());
385 return;
386 }
387 if (isInState(State.SESSION_ACCEPTED)) {
388 receiveContentAdd(jinglePacket, modification);
389 } else {
390 terminateWithOutOfOrder(jinglePacket);
391 }
392 }
393
394 private void receiveContentAdd(
395 final JinglePacket jinglePacket, final RtpContentMap modification) {
396 final RtpContentMap remote = getRemoteContentMap();
397 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
398 respondOk(jinglePacket);
399 this.webRTCWrapper.close();
400 sendSessionTerminate(
401 Reason.FAILED_APPLICATION,
402 String.format(
403 "contents with names %s already exists",
404 Joiner.on(", ").join(modification.getNames())));
405 return;
406 }
407 final ContentAddition contentAddition =
408 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
409
410 final RtpContentMap outgoing = this.outgoingContentAdd;
411 final Set<ContentAddition.Summary> outgoingContentAddSummary =
412 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
413
414 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
415 if (isInitiator()) {
416 Log.d(
417 Config.LOGTAG,
418 id.getAccount().getJid().asBareJid()
419 + ": respond with tie break to matching content-add offer");
420 respondWithTieBreak(jinglePacket);
421 } else {
422 Log.d(
423 Config.LOGTAG,
424 id.getAccount().getJid().asBareJid()
425 + ": automatically accept matching content-add offer");
426 acceptContentAdd(contentAddition.summary, modification);
427 }
428 return;
429 }
430
431 // once we can display multiple video tracks we can be more loose with this condition
432 // theoretically it should also be fine to automatically accept audio only contents
433 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
434 Log.d(
435 Config.LOGTAG,
436 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
437 this.incomingContentAdd = modification;
438 respondOk(jinglePacket);
439 updateEndUserState();
440 } else {
441 respondOk(jinglePacket);
442 // TODO do we want to add a reason?
443 rejectContentAdd(modification);
444 }
445 }
446
447 private void receiveContentAccept(final JinglePacket jinglePacket) {
448 final RtpContentMap receivedContentAccept;
449 try {
450 receivedContentAccept = RtpContentMap.of(jinglePacket);
451 receivedContentAccept.requireContentDescriptions();
452 } catch (final RuntimeException e) {
453 Log.d(
454 Config.LOGTAG,
455 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
456 Throwables.getRootCause(e));
457 respondOk(jinglePacket);
458 webRTCWrapper.close();
459 sendSessionTerminate(Reason.of(e), e.getMessage());
460 return;
461 }
462
463 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
464 if (outgoingContentAdd == null) {
465 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
466 terminateWithOutOfOrder(jinglePacket);
467 return;
468 }
469 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
470 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
471 this.outgoingContentAdd = null;
472 respondOk(jinglePacket);
473 receiveContentAccept(receivedContentAccept);
474 } else {
475 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
476 terminateWithOutOfOrder(jinglePacket);
477 }
478 }
479
480 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
481 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
482 final RtpContentMap modifiedContentMap =
483 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
484
485 setRemoteContentMap(modifiedContentMap);
486
487 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
488
489 final org.webrtc.SessionDescription sdp =
490 new org.webrtc.SessionDescription(
491 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
492
493 try {
494 this.webRTCWrapper.setRemoteDescription(sdp).get();
495 } catch (final Exception e) {
496 final Throwable cause = Throwables.getRootCause(e);
497 Log.d(
498 Config.LOGTAG,
499 id.getAccount().getJid().asBareJid()
500 + ": unable to set remote description after receiving content-accept",
501 cause);
502 webRTCWrapper.close();
503 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
504 return;
505 }
506 processCandidates(receivedContentAccept.contents.entrySet());
507 updateEndUserState();
508 Log.d(
509 Config.LOGTAG,
510 id.getAccount().getJid().asBareJid()
511 + ": remote has accepted content-add "
512 + ContentAddition.summary(receivedContentAccept));
513 }
514
515 private void receiveContentModify(final JinglePacket jinglePacket) {
516 final Map<String, Content.Senders> modification =
517 Maps.transformEntries(
518 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
519 respondOk(jinglePacket);
520 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
521 final Set<String> currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet();
522 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
523 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
524 final boolean isInitiator = isInitiator();
525 final RtpContentMap modifiedContentMap;
526 try {
527 modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification);
528 } catch (final IllegalArgumentException e) {
529 webRTCWrapper.close();
530 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
531 return;
532 }
533 this.outgoingContentAdd = modifiedContentMap;
534 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add");
535 } else {
536 webRTCWrapper.close();
537 sendSessionTerminate(
538 Reason.FAILED_APPLICATION,
539 String.format(
540 "%s only supports %s as a means to modify a not yet accepted %s",
541 BuildConfig.APP_NAME,
542 JinglePacket.Action.CONTENT_MODIFY,
543 JinglePacket.Action.CONTENT_ADD));
544 }
545 }
546
547 private void receiveContentReject(final JinglePacket jinglePacket) {
548 final RtpContentMap receivedContentReject;
549 try {
550 receivedContentReject = RtpContentMap.of(jinglePacket);
551 } catch (final RuntimeException e) {
552 Log.d(
553 Config.LOGTAG,
554 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
555 Throwables.getRootCause(e));
556 respondOk(jinglePacket);
557 this.webRTCWrapper.close();
558 sendSessionTerminate(Reason.of(e), e.getMessage());
559 return;
560 }
561
562 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
563 if (outgoingContentAdd == null) {
564 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
565 terminateWithOutOfOrder(jinglePacket);
566 return;
567 }
568 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
569 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
570 this.outgoingContentAdd = null;
571 respondOk(jinglePacket);
572 Log.d(Config.LOGTAG,jinglePacket.toString());
573 receiveContentReject(ourSummary);
574 } else {
575 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
576 terminateWithOutOfOrder(jinglePacket);
577 }
578 }
579
580 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
581 try {
582 this.webRTCWrapper.removeTrack(Media.VIDEO);
583 final RtpContentMap localContentMap = customRollback();
584 modifyLocalContentMap(localContentMap);
585 } catch (final Exception e) {
586 final Throwable cause = Throwables.getRootCause(e);
587 Log.d(
588 Config.LOGTAG,
589 id.getAccount().getJid().asBareJid()
590 + ": unable to rollback local description after receiving content-reject",
591 cause);
592 webRTCWrapper.close();
593 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
594 return;
595 }
596 Log.d(
597 Config.LOGTAG,
598 id.getAccount().getJid().asBareJid()
599 + ": remote has rejected our content-add "
600 + summary);
601 }
602
603 private void receiveContentRemove(final JinglePacket jinglePacket) {
604 final RtpContentMap receivedContentRemove;
605 try {
606 receivedContentRemove = RtpContentMap.of(jinglePacket);
607 receivedContentRemove.requireContentDescriptions();
608 } catch (final RuntimeException e) {
609 Log.d(
610 Config.LOGTAG,
611 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
612 Throwables.getRootCause(e));
613 respondOk(jinglePacket);
614 this.webRTCWrapper.close();
615 sendSessionTerminate(Reason.of(e), e.getMessage());
616 return;
617 }
618 respondOk(jinglePacket);
619 receiveContentRemove(receivedContentRemove);
620 }
621
622 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
623 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
624 final Set<ContentAddition.Summary> contentAddSummary =
625 incomingContentAdd == null
626 ? Collections.emptySet()
627 : ContentAddition.summary(incomingContentAdd);
628 final Set<ContentAddition.Summary> removeSummary =
629 ContentAddition.summary(receivedContentRemove);
630 if (contentAddSummary.equals(removeSummary)) {
631 this.incomingContentAdd = null;
632 updateEndUserState();
633 } else {
634 webRTCWrapper.close();
635 sendSessionTerminate(
636 Reason.FAILED_APPLICATION,
637 String.format(
638 "%s only supports %s as a means to retract a not yet accepted %s",
639 BuildConfig.APP_NAME,
640 JinglePacket.Action.CONTENT_REMOVE,
641 JinglePacket.Action.CONTENT_ADD));
642 }
643 }
644
645 public synchronized void retractContentAdd() {
646 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
647 if (outgoingContentAdd == null) {
648 throw new IllegalStateException("Not outgoing content add");
649 }
650 try {
651 webRTCWrapper.removeTrack(Media.VIDEO);
652 final RtpContentMap localContentMap = customRollback();
653 modifyLocalContentMap(localContentMap);
654 } catch (final Exception e) {
655 final Throwable cause = Throwables.getRootCause(e);
656 Log.d(
657 Config.LOGTAG,
658 id.getAccount().getJid().asBareJid()
659 + ": unable to rollback local description after trying to retract content-add",
660 cause);
661 webRTCWrapper.close();
662 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
663 return;
664 }
665 this.outgoingContentAdd = null;
666 final JinglePacket retract =
667 outgoingContentAdd
668 .toStub()
669 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
670 this.send(retract);
671 Log.d(
672 Config.LOGTAG,
673 id.getAccount().getJid()
674 + ": retract content-add "
675 + ContentAddition.summary(outgoingContentAdd));
676 }
677
678 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
679 final SessionDescription sdp = setLocalSessionDescription();
680 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
681 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
682 this.webRTCWrapper
683 .setRemoteDescription(
684 new org.webrtc.SessionDescription(
685 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
686 .get();
687 return localRtpContentMap;
688 }
689
690 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
691 final RtpContentMap currentRemote = getRemoteContentMap();
692 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
693 if (diff.isEmpty()) {
694 throw new IllegalStateException(
695 "Unexpected rollback condition. No difference between local and remote");
696 }
697 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
698 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
699 final RtpContentMap nextRemote =
700 currentRemote.addContent(
701 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
702 return SessionDescription.of(nextRemote, !isInitiator());
703 }
704 throw new IllegalStateException(
705 "Unexpected rollback condition. Senders were not uniformly none");
706 }
707
708 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
709 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
710 if (incomingContentAdd == null) {
711 throw new IllegalStateException("No incoming content add");
712 }
713
714 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
715 this.incomingContentAdd = null;
716 acceptContentAdd(contentAddition, incomingContentAdd);
717 } else {
718 throw new IllegalStateException("Accepted content add does not match pending content-add");
719 }
720 }
721
722 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
723 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
724 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
725 this.setRemoteContentMap(modifiedContentMap);
726
727 final SessionDescription offer;
728 try {
729 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
730 } catch (final IllegalArgumentException | NullPointerException e) {
731 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
732 webRTCWrapper.close();
733 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
734 return;
735 }
736 this.incomingContentAdd = null;
737 acceptContentAdd(contentAddition, offer);
738 }
739
740 private void acceptContentAdd(
741 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
742 final org.webrtc.SessionDescription sdp =
743 new org.webrtc.SessionDescription(
744 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
745 try {
746 this.webRTCWrapper.setRemoteDescription(sdp).get();
747
748 // TODO add tracks for 'media' where contentAddition.senders matches
749
750 // TODO if senders.sending(isInitiator())
751
752 this.webRTCWrapper.addTrack(Media.VIDEO);
753
754 // TODO add additional transceivers for recv only cases
755
756 final SessionDescription answer = setLocalSessionDescription();
757 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
758
759 final RtpContentMap contentAcceptMap =
760 rtpContentMap.toContentModification(
761 Collections2.transform(contentAddition, ca -> ca.name));
762 Log.d(
763 Config.LOGTAG,
764 id.getAccount().getJid().asBareJid()
765 + ": sending content-accept "
766 + ContentAddition.summary(contentAcceptMap));
767 modifyLocalContentMap(rtpContentMap);
768 sendContentAccept(contentAcceptMap);
769 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
770 } catch (final Exception e) {
771 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
772 webRTCWrapper.close();
773 sendSessionTerminate(Reason.FAILED_APPLICATION);
774 }
775 }
776
777 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
778 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
779 send(jinglePacket);
780 }
781
782 public synchronized void rejectContentAdd() {
783 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
784 if (incomingContentAdd == null) {
785 throw new IllegalStateException("No incoming content add");
786 }
787 this.incomingContentAdd = null;
788 updateEndUserState();
789 rejectContentAdd(incomingContentAdd);
790 }
791
792 private void rejectContentAdd(final RtpContentMap contentMap) {
793 final JinglePacket jinglePacket =
794 contentMap
795 .toStub()
796 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
797 Log.d(
798 Config.LOGTAG,
799 id.getAccount().getJid().asBareJid()
800 + ": rejecting content "
801 + ContentAddition.summary(contentMap));
802 send(jinglePacket);
803 }
804
805 private boolean checkForIceRestart(
806 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
807 final RtpContentMap existing = getRemoteContentMap();
808 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
809 final IceUdpTransportInfo.Credentials newCredentials;
810 try {
811 existingCredentials = existing.getCredentials();
812 newCredentials = rtpContentMap.getDistinctCredentials();
813 } catch (final IllegalStateException e) {
814 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
815 return false;
816 }
817 if (existingCredentials.contains(newCredentials)) {
818 return false;
819 }
820 // TODO an alternative approach is to check if we already got an iq result to our
821 // ICE-restart
822 // and if that's the case we are seeing an answer.
823 // This might be more spec compliant but also more error prone potentially
824 final boolean isSignalStateStable = this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
825 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
826 final boolean isOffer = rtpContentMap.emptyCandidates();
827 final RtpContentMap restartContentMap;
828 try {
829 if (isOffer) {
830 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
831 restartContentMap =
832 existing.modifiedCredentials(
833 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
834 } else {
835 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
836 Log.d(
837 Config.LOGTAG,
838 "received confirmation of ICE restart"
839 + newCredentials
840 + " peer_setup="
841 + setup);
842 // DTLS setup attribute needs to be rewritten to reflect current peer state
843 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
844 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
845 }
846 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
847 return isOffer;
848 } else {
849 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
850 respondWithTieBreak(jinglePacket);
851 return true;
852 }
853 } catch (final Exception exception) {
854 respondOk(jinglePacket);
855 final Throwable rootCause = Throwables.getRootCause(exception);
856 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
857 // If this happens a termination is already in progress
858 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
859 return true;
860 }
861 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
862 webRTCWrapper.close();
863 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
864 return true;
865 }
866 }
867
868 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
869 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
870 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
871 throw new IllegalStateException("Invalid peer setup");
872 }
873 return peerSetup;
874 }
875
876 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
877 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
878 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
879 }
880 this.peerDtlsSetup = setup;
881 }
882
883 private boolean applyIceRestart(
884 final JinglePacket jinglePacket,
885 final RtpContentMap restartContentMap,
886 final boolean isOffer)
887 throws ExecutionException, InterruptedException {
888 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
889 final org.webrtc.SessionDescription.Type type =
890 isOffer
891 ? org.webrtc.SessionDescription.Type.OFFER
892 : org.webrtc.SessionDescription.Type.ANSWER;
893 org.webrtc.SessionDescription sdp =
894 new org.webrtc.SessionDescription(type, sessionDescription.toString());
895 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
896 if (isInitiator()) {
897 // We ignore the offer and respond with tie-break. This will clause the responder
898 // not to apply the content map
899 return false;
900 }
901 }
902 webRTCWrapper.setRemoteDescription(sdp).get();
903 setRemoteContentMap(restartContentMap);
904 if (isOffer) {
905 final SessionDescription localSessionDescription = setLocalSessionDescription();
906 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
907 // We need to respond OK before sending any candidates
908 respondOk(jinglePacket);
909 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
910 } else {
911 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
912 }
913 return true;
914 }
915
916 private void processCandidates(
917 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
918 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
919 processCandidate(content);
920 }
921 }
922
923 private void processCandidate(
924 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
925 final RtpContentMap rtpContentMap = getRemoteContentMap();
926 final List<String> indices = toIdentificationTags(rtpContentMap);
927 final String sdpMid = content.getKey(); // aka content name
928 final IceUdpTransportInfo transport = content.getValue().transport;
929 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
930
931 // TODO check that credentials remained the same
932
933 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
934 final String sdp;
935 try {
936 sdp = candidate.toSdpAttribute(credentials.ufrag);
937 } catch (final IllegalArgumentException e) {
938 Log.d(
939 Config.LOGTAG,
940 id.account.getJid().asBareJid()
941 + ": ignoring invalid ICE candidate "
942 + e.getMessage());
943 continue;
944 }
945 final int mLineIndex = indices.indexOf(sdpMid);
946 if (mLineIndex < 0) {
947 Log.w(
948 Config.LOGTAG,
949 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
950 }
951 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
952 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
953 this.webRTCWrapper.addIceCandidate(iceCandidate);
954 }
955 }
956
957 private RtpContentMap getRemoteContentMap() {
958 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
959 }
960
961 private RtpContentMap getLocalContentMap() {
962 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
963 }
964
965 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
966 final Group originalGroup = rtpContentMap.group;
967 final List<String> identificationTags =
968 originalGroup == null
969 ? rtpContentMap.getNames()
970 : originalGroup.getIdentificationTags();
971 if (identificationTags.size() == 0) {
972 Log.w(
973 Config.LOGTAG,
974 id.account.getJid().asBareJid()
975 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
976 }
977 return identificationTags;
978 }
979
980 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
981 final JinglePacket jinglePacket, final boolean expectVerification) {
982 final RtpContentMap receivedContentMap;
983 try {
984 receivedContentMap = RtpContentMap.of(jinglePacket);
985 } catch (final Exception e) {
986 return Futures.immediateFailedFuture(e);
987 }
988 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
989 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
990 id.account
991 .getAxolotlService()
992 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
993 return Futures.transform(
994 future,
995 omemoVerifiedPayload -> {
996 // TODO test if an exception here triggers a correct abort
997 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
998 Log.d(
999 Config.LOGTAG,
1000 id.account.getJid().asBareJid()
1001 + ": received verifiable DTLS fingerprint via "
1002 + omemoVerification);
1003 return omemoVerifiedPayload.getPayload();
1004 },
1005 MoreExecutors.directExecutor());
1006 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1007 return Futures.immediateFailedFuture(
1008 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1009 } else {
1010 return Futures.immediateFuture(receivedContentMap);
1011 }
1012 }
1013
1014 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1015 if (isInitiator()) {
1016 Log.d(
1017 Config.LOGTAG,
1018 String.format(
1019 "%s: received session-initiate even though we were initiating",
1020 id.account.getJid().asBareJid()));
1021 if (isTerminated()) {
1022 Log.d(
1023 Config.LOGTAG,
1024 String.format(
1025 "%s: got a reason to terminate with out-of-order. but already in state %s",
1026 id.account.getJid().asBareJid(), getState()));
1027 respondWithOutOfOrder(jinglePacket);
1028 } else {
1029 terminateWithOutOfOrder(jinglePacket);
1030 }
1031 return;
1032 }
1033 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1034 Futures.addCallback(
1035 future,
1036 new FutureCallback<RtpContentMap>() {
1037 @Override
1038 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1039 receiveSessionInitiate(jinglePacket, rtpContentMap);
1040 }
1041
1042 @Override
1043 public void onFailure(@NonNull final Throwable throwable) {
1044 respondOk(jinglePacket);
1045 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1046 }
1047 },
1048 MoreExecutors.directExecutor());
1049 }
1050
1051 private void receiveSessionInitiate(
1052 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1053 try {
1054 contentMap.requireContentDescriptions();
1055 contentMap.requireDTLSFingerprint(true);
1056 } catch (final RuntimeException e) {
1057 Log.d(
1058 Config.LOGTAG,
1059 id.account.getJid().asBareJid() + ": improperly formatted contents",
1060 Throwables.getRootCause(e));
1061 respondOk(jinglePacket);
1062 sendSessionTerminate(Reason.of(e), e.getMessage());
1063 return;
1064 }
1065 Log.d(
1066 Config.LOGTAG,
1067 "processing session-init with " + contentMap.contents.size() + " contents");
1068 final State target;
1069 if (this.state == State.PROCEED) {
1070 Preconditions.checkState(
1071 proposedMedia != null && proposedMedia.size() > 0,
1072 "proposed media must be set when processing pre-approved session-initiate");
1073 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1074 sendSessionTerminate(
1075 Reason.SECURITY_ERROR,
1076 String.format(
1077 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1078 this.proposedMedia, contentMap.getMedia()));
1079 return;
1080 }
1081 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1082 } else {
1083 target = State.SESSION_INITIALIZED;
1084 }
1085 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1086 respondOk(jinglePacket);
1087 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1088 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1089 Log.d(
1090 Config.LOGTAG,
1091 id.account.getJid().asBareJid()
1092 + ": automatically accepting session-initiate");
1093 sendSessionAccept();
1094 } else {
1095 Log.d(
1096 Config.LOGTAG,
1097 id.account.getJid().asBareJid()
1098 + ": received not pre-approved session-initiate. start ringing");
1099 startRinging();
1100 }
1101 } else {
1102 Log.d(
1103 Config.LOGTAG,
1104 String.format(
1105 "%s: received session-initiate while in state %s",
1106 id.account.getJid().asBareJid(), state));
1107 terminateWithOutOfOrder(jinglePacket);
1108 }
1109 }
1110
1111 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1112 if (!isInitiator()) {
1113 Log.d(
1114 Config.LOGTAG,
1115 String.format(
1116 "%s: received session-accept even though we were responding",
1117 id.account.getJid().asBareJid()));
1118 terminateWithOutOfOrder(jinglePacket);
1119 return;
1120 }
1121 final ListenableFuture<RtpContentMap> future =
1122 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1123 Futures.addCallback(
1124 future,
1125 new FutureCallback<RtpContentMap>() {
1126 @Override
1127 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1128 receiveSessionAccept(jinglePacket, rtpContentMap);
1129 }
1130
1131 @Override
1132 public void onFailure(@NonNull final Throwable throwable) {
1133 respondOk(jinglePacket);
1134 Log.d(
1135 Config.LOGTAG,
1136 id.account.getJid().asBareJid()
1137 + ": improperly formatted contents in session-accept",
1138 throwable);
1139 webRTCWrapper.close();
1140 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1141 }
1142 },
1143 MoreExecutors.directExecutor());
1144 }
1145
1146 private void receiveSessionAccept(
1147 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1148 try {
1149 contentMap.requireContentDescriptions();
1150 contentMap.requireDTLSFingerprint();
1151 } catch (final RuntimeException e) {
1152 respondOk(jinglePacket);
1153 Log.d(
1154 Config.LOGTAG,
1155 id.account.getJid().asBareJid()
1156 + ": improperly formatted contents in session-accept",
1157 e);
1158 webRTCWrapper.close();
1159 sendSessionTerminate(Reason.of(e), e.getMessage());
1160 return;
1161 }
1162 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1163 if (!initiatorMedia.equals(contentMap.getMedia())) {
1164 sendSessionTerminate(
1165 Reason.SECURITY_ERROR,
1166 String.format(
1167 "Your session-included included media %s but our session-initiate was %s",
1168 this.proposedMedia, contentMap.getMedia()));
1169 return;
1170 }
1171 Log.d(
1172 Config.LOGTAG,
1173 "processing session-accept with " + contentMap.contents.size() + " contents");
1174 if (transition(State.SESSION_ACCEPTED)) {
1175 respondOk(jinglePacket);
1176 receiveSessionAccept(contentMap);
1177 } else {
1178 Log.d(
1179 Config.LOGTAG,
1180 String.format(
1181 "%s: received session-accept while in state %s",
1182 id.account.getJid().asBareJid(), state));
1183 respondOk(jinglePacket);
1184 }
1185 }
1186
1187 private void receiveSessionAccept(final RtpContentMap contentMap) {
1188 this.responderRtpContentMap = contentMap;
1189 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1190 final SessionDescription sessionDescription;
1191 try {
1192 sessionDescription = SessionDescription.of(contentMap, false);
1193 } catch (final IllegalArgumentException | NullPointerException e) {
1194 Log.d(
1195 Config.LOGTAG,
1196 id.account.getJid().asBareJid()
1197 + ": unable convert offer from session-accept to SDP",
1198 e);
1199 webRTCWrapper.close();
1200 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1201 return;
1202 }
1203 final org.webrtc.SessionDescription answer =
1204 new org.webrtc.SessionDescription(
1205 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1206 try {
1207 this.webRTCWrapper.setRemoteDescription(answer).get();
1208 } catch (final Exception e) {
1209 Log.d(
1210 Config.LOGTAG,
1211 id.account.getJid().asBareJid()
1212 + ": unable to set remote description after receiving session-accept",
1213 Throwables.getRootCause(e));
1214 webRTCWrapper.close();
1215 sendSessionTerminate(
1216 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1217 return;
1218 }
1219 processCandidates(contentMap.contents.entrySet());
1220 }
1221
1222 private void sendSessionAccept() {
1223 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1224 if (rtpContentMap == null) {
1225 throw new IllegalStateException("initiator RTP Content Map has not been set");
1226 }
1227 final SessionDescription offer;
1228 try {
1229 offer = SessionDescription.of(rtpContentMap, true);
1230 } catch (final IllegalArgumentException | NullPointerException e) {
1231 Log.d(
1232 Config.LOGTAG,
1233 id.account.getJid().asBareJid()
1234 + ": unable convert offer from session-initiate to SDP",
1235 e);
1236 webRTCWrapper.close();
1237 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1238 return;
1239 }
1240 sendSessionAccept(rtpContentMap.getMedia(), offer);
1241 }
1242
1243 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1244 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1245 }
1246
1247 private synchronized void sendSessionAccept(
1248 final Set<Media> media,
1249 final SessionDescription offer,
1250 final List<PeerConnection.IceServer> iceServers) {
1251 if (isTerminated()) {
1252 Log.w(
1253 Config.LOGTAG,
1254 id.account.getJid().asBareJid()
1255 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1256 return;
1257 }
1258 try {
1259 setupWebRTC(media, iceServers);
1260 } catch (final WebRTCWrapper.InitializationException e) {
1261 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1262 webRTCWrapper.close();
1263 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1264 return;
1265 }
1266 final org.webrtc.SessionDescription sdp =
1267 new org.webrtc.SessionDescription(
1268 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1269 try {
1270 this.webRTCWrapper.setRemoteDescription(sdp).get();
1271 addIceCandidatesFromBlackLog();
1272 org.webrtc.SessionDescription webRTCSessionDescription =
1273 this.webRTCWrapper.setLocalDescription().get();
1274 prepareSessionAccept(webRTCSessionDescription);
1275 } catch (final Exception e) {
1276 failureToAcceptSession(e);
1277 }
1278 }
1279
1280 private void failureToAcceptSession(final Throwable throwable) {
1281 if (isTerminated()) {
1282 return;
1283 }
1284 final Throwable rootCause = Throwables.getRootCause(throwable);
1285 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1286 webRTCWrapper.close();
1287 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1288 }
1289
1290 private void addIceCandidatesFromBlackLog() {
1291 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1292 while ((foo = this.pendingIceCandidates.poll()) != null) {
1293 processCandidate(foo);
1294 Log.d(
1295 Config.LOGTAG,
1296 id.account.getJid().asBareJid() + ": added candidate from back log");
1297 }
1298 }
1299
1300 private void prepareSessionAccept(
1301 final org.webrtc.SessionDescription webRTCSessionDescription) {
1302 final SessionDescription sessionDescription =
1303 SessionDescription.parse(webRTCSessionDescription.description);
1304 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1305 this.responderRtpContentMap = respondingRtpContentMap;
1306 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1307 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1308 prepareOutgoingContentMap(respondingRtpContentMap);
1309 Futures.addCallback(
1310 outgoingContentMapFuture,
1311 new FutureCallback<RtpContentMap>() {
1312 @Override
1313 public void onSuccess(final RtpContentMap outgoingContentMap) {
1314 sendSessionAccept(outgoingContentMap);
1315 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1316 }
1317
1318 @Override
1319 public void onFailure(@NonNull Throwable throwable) {
1320 failureToAcceptSession(throwable);
1321 }
1322 },
1323 MoreExecutors.directExecutor());
1324 }
1325
1326 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1327 if (isTerminated()) {
1328 Log.w(
1329 Config.LOGTAG,
1330 id.account.getJid().asBareJid()
1331 + ": preparing session accept was too slow. already terminated. nothing to do.");
1332 return;
1333 }
1334 transitionOrThrow(State.SESSION_ACCEPTED);
1335 final JinglePacket sessionAccept =
1336 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1337 send(sessionAccept);
1338 }
1339
1340 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1341 final RtpContentMap rtpContentMap) {
1342 if (this.omemoVerification.hasDeviceId()) {
1343 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1344 verifiedPayloadFuture =
1345 id.account
1346 .getAxolotlService()
1347 .encrypt(
1348 rtpContentMap,
1349 id.with,
1350 omemoVerification.getDeviceId());
1351 return Futures.transform(
1352 verifiedPayloadFuture,
1353 verifiedPayload -> {
1354 omemoVerification.setOrEnsureEqual(verifiedPayload);
1355 return verifiedPayload.getPayload();
1356 },
1357 MoreExecutors.directExecutor());
1358 } else {
1359 return Futures.immediateFuture(rtpContentMap);
1360 }
1361 }
1362
1363 synchronized void deliveryMessage(
1364 final Jid from,
1365 final Element message,
1366 final String serverMessageId,
1367 final long timestamp) {
1368 Log.d(
1369 Config.LOGTAG,
1370 id.account.getJid().asBareJid()
1371 + ": delivered message to JingleRtpConnection "
1372 + message);
1373 switch (message.getName()) {
1374 case "propose":
1375 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1376 break;
1377 case "proceed":
1378 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1379 break;
1380 case "retract":
1381 receiveRetract(from, serverMessageId, timestamp);
1382 break;
1383 case "reject":
1384 receiveReject(from, serverMessageId, timestamp);
1385 break;
1386 case "accept":
1387 receiveAccept(from, serverMessageId, timestamp);
1388 break;
1389 default:
1390 break;
1391 }
1392 }
1393
1394 void deliverFailedProceed(final String message) {
1395 Log.d(
1396 Config.LOGTAG,
1397 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1398 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1399 webRTCWrapper.close();
1400 Log.d(
1401 Config.LOGTAG,
1402 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1403 this.finish();
1404 }
1405 }
1406
1407 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1408 final boolean originatedFromMyself =
1409 from.asBareJid().equals(id.account.getJid().asBareJid());
1410 if (originatedFromMyself) {
1411 if (transition(State.ACCEPTED)) {
1412 acceptedOnOtherDevice(serverMsgId, timestamp);
1413 } else {
1414 Log.d(
1415 Config.LOGTAG,
1416 id.account.getJid().asBareJid()
1417 + ": unable to transition to accept because already in state="
1418 + this.state);
1419 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1420 }
1421 } else {
1422 Log.d(
1423 Config.LOGTAG,
1424 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1425 }
1426 }
1427
1428 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1429 if (serverMsgId != null) {
1430 this.message.setServerMsgId(serverMsgId);
1431 }
1432 this.message.setTime(timestamp);
1433 this.message.setCarbon(true); // indicate that call was accepted on other device
1434 this.writeLogMessageSuccess(0);
1435 this.xmppConnectionService
1436 .getNotificationService()
1437 .cancelIncomingCallNotification();
1438 this.finish();
1439 }
1440
1441 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1442 final boolean originatedFromMyself =
1443 from.asBareJid().equals(id.account.getJid().asBareJid());
1444 // reject from another one of my clients
1445 if (originatedFromMyself) {
1446 receiveRejectFromMyself(serverMsgId, timestamp);
1447 } else if (isInitiator()) {
1448 if (from.equals(id.with)) {
1449 receiveRejectFromResponder();
1450 } else {
1451 Log.d(
1452 Config.LOGTAG,
1453 id.account.getJid()
1454 + ": ignoring reject from "
1455 + from
1456 + " for session with "
1457 + id.with);
1458 }
1459 } else {
1460 Log.d(
1461 Config.LOGTAG,
1462 id.account.getJid()
1463 + ": ignoring reject from "
1464 + from
1465 + " for session with "
1466 + id.with);
1467 }
1468 }
1469
1470 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1471 if (transition(State.REJECTED)) {
1472 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1473 this.finish();
1474 if (serverMsgId != null) {
1475 this.message.setServerMsgId(serverMsgId);
1476 }
1477 this.message.setTime(timestamp);
1478 this.message.setCarbon(true); // indicate that call was rejected on other device
1479 writeLogMessageMissed();
1480 } else {
1481 Log.d(
1482 Config.LOGTAG,
1483 "not able to transition into REJECTED because already in " + this.state);
1484 }
1485 }
1486
1487 private void receiveRejectFromResponder() {
1488 if (isInState(State.PROCEED)) {
1489 Log.d(
1490 Config.LOGTAG,
1491 id.account.getJid()
1492 + ": received reject while still in proceed. callee reconsidered");
1493 closeTransitionLogFinish(State.REJECTED_RACED);
1494 return;
1495 }
1496 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1497 Log.d(
1498 Config.LOGTAG,
1499 id.account.getJid()
1500 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1501 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1502 return;
1503 }
1504 Log.d(
1505 Config.LOGTAG,
1506 id.account.getJid()
1507 + ": ignoring reject from responder because already in state "
1508 + this.state);
1509 }
1510
1511 private void receivePropose(
1512 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1513 final boolean originatedFromMyself =
1514 from.asBareJid().equals(id.account.getJid().asBareJid());
1515 if (originatedFromMyself) {
1516 Log.d(
1517 Config.LOGTAG,
1518 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1519 } else if (transition(
1520 State.PROPOSED,
1521 () -> {
1522 final Collection<RtpDescription> descriptions =
1523 Collections2.transform(
1524 Collections2.filter(
1525 propose.getDescriptions(),
1526 d -> d instanceof RtpDescription),
1527 input -> (RtpDescription) input);
1528 final Collection<Media> media =
1529 Collections2.transform(descriptions, RtpDescription::getMedia);
1530 Preconditions.checkState(
1531 !media.contains(Media.UNKNOWN),
1532 "RTP descriptions contain unknown media");
1533 Log.d(
1534 Config.LOGTAG,
1535 id.account.getJid().asBareJid()
1536 + ": received session proposal from "
1537 + from
1538 + " for "
1539 + media);
1540 this.proposedMedia = Sets.newHashSet(media);
1541 })) {
1542 if (serverMsgId != null) {
1543 this.message.setServerMsgId(serverMsgId);
1544 }
1545 this.message.setTime(timestamp);
1546 startRinging();
1547 if (xmppConnectionService.confirmMessages() && id.getContact().showInContactList()) {
1548 sendJingleMessage("ringing");
1549 }
1550 } else {
1551 Log.d(
1552 Config.LOGTAG,
1553 id.account.getJid()
1554 + ": ignoring session proposal because already in "
1555 + state);
1556 }
1557 }
1558
1559 private void startRinging() {
1560 Log.d(
1561 Config.LOGTAG,
1562 id.account.getJid().asBareJid()
1563 + ": received call from "
1564 + id.with
1565 + ". start ringing");
1566 ringingTimeoutFuture =
1567 jingleConnectionManager.schedule(
1568 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1569 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1570 }
1571
1572 private synchronized void ringingTimeout() {
1573 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1574 switch (this.state) {
1575 case PROPOSED:
1576 message.markUnread();
1577 rejectCallFromProposed();
1578 break;
1579 case SESSION_INITIALIZED:
1580 message.markUnread();
1581 rejectCallFromSessionInitiate();
1582 break;
1583 }
1584 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1585 }
1586
1587 private void cancelRingingTimeout() {
1588 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1589 if (future != null && !future.isCancelled()) {
1590 future.cancel(false);
1591 }
1592 }
1593
1594 private void receiveProceed(
1595 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1596 final Set<Media> media =
1597 Preconditions.checkNotNull(
1598 this.proposedMedia, "Proposed media has to be set before handling proceed");
1599 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1600 if (from.equals(id.with)) {
1601 if (isInitiator()) {
1602 if (transition(State.PROCEED)) {
1603 if (serverMsgId != null) {
1604 this.message.setServerMsgId(serverMsgId);
1605 }
1606 this.message.setTime(timestamp);
1607 final Integer remoteDeviceId = proceed.getDeviceId();
1608 if (isOmemoEnabled()) {
1609 this.omemoVerification.setDeviceId(remoteDeviceId);
1610 } else {
1611 if (remoteDeviceId != null) {
1612 Log.d(
1613 Config.LOGTAG,
1614 id.account.getJid().asBareJid()
1615 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1616 }
1617 this.omemoVerification.setDeviceId(null);
1618 }
1619 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1620 } else {
1621 Log.d(
1622 Config.LOGTAG,
1623 String.format(
1624 "%s: ignoring proceed because already in %s",
1625 id.account.getJid().asBareJid(), this.state));
1626 }
1627 } else {
1628 Log.d(
1629 Config.LOGTAG,
1630 String.format(
1631 "%s: ignoring proceed because we were not initializing",
1632 id.account.getJid().asBareJid()));
1633 }
1634 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1635 if (transition(State.ACCEPTED)) {
1636 Log.d(
1637 Config.LOGTAG,
1638 id.account.getJid().asBareJid()
1639 + ": moved session with "
1640 + id.with
1641 + " into state accepted after received carbon copied proceed");
1642 acceptedOnOtherDevice(serverMsgId, timestamp);
1643 }
1644 } else {
1645 Log.d(
1646 Config.LOGTAG,
1647 String.format(
1648 "%s: ignoring proceed from %s. was expected from %s",
1649 id.account.getJid().asBareJid(), from, id.with));
1650 }
1651 }
1652
1653 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1654 if (from.equals(id.with)) {
1655 final State target =
1656 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1657 if (transition(target)) {
1658 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1659 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1660 Log.d(
1661 Config.LOGTAG,
1662 id.account.getJid().asBareJid()
1663 + ": session with "
1664 + id.with
1665 + " has been retracted (serverMsgId="
1666 + serverMsgId
1667 + ")");
1668 if (serverMsgId != null) {
1669 this.message.setServerMsgId(serverMsgId);
1670 }
1671 this.message.setTime(timestamp);
1672 if (target == State.RETRACTED) {
1673 this.message.markUnread();
1674 }
1675 writeLogMessageMissed();
1676 finish();
1677 } else {
1678 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1679 }
1680 } else {
1681 // TODO parse retract from self
1682 Log.d(
1683 Config.LOGTAG,
1684 id.account.getJid().asBareJid()
1685 + ": received retract from "
1686 + from
1687 + ". expected retract from"
1688 + id.with
1689 + ". ignoring");
1690 }
1691 }
1692
1693 public void sendSessionInitiate() {
1694 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1695 }
1696
1697 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1698 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1699 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1700 }
1701
1702 private synchronized void sendSessionInitiate(
1703 final Set<Media> media,
1704 final State targetState,
1705 final List<PeerConnection.IceServer> iceServers) {
1706 if (isTerminated()) {
1707 Log.w(
1708 Config.LOGTAG,
1709 id.account.getJid().asBareJid()
1710 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1711 return;
1712 }
1713 try {
1714 setupWebRTC(media, iceServers);
1715 } catch (final WebRTCWrapper.InitializationException e) {
1716 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1717 webRTCWrapper.close();
1718 sendRetract(Reason.ofThrowable(e));
1719 return;
1720 }
1721 try {
1722 org.webrtc.SessionDescription webRTCSessionDescription =
1723 this.webRTCWrapper.setLocalDescription().get();
1724 prepareSessionInitiate(webRTCSessionDescription, targetState);
1725 } catch (final Exception e) {
1726 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1727 // exceptions
1728 failureToInitiateSession(e, targetState);
1729 }
1730 }
1731
1732 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1733 if (isTerminated()) {
1734 return;
1735 }
1736 Log.d(
1737 Config.LOGTAG,
1738 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1739 Throwables.getRootCause(throwable));
1740 webRTCWrapper.close();
1741 final Reason reason = Reason.ofThrowable(throwable);
1742 if (isInState(targetState)) {
1743 sendSessionTerminate(reason, throwable.getMessage());
1744 } else {
1745 sendRetract(reason);
1746 }
1747 }
1748
1749 private void sendRetract(final Reason reason) {
1750 // TODO embed reason into retract
1751 sendJingleMessage("retract", id.with.asBareJid());
1752 transitionOrThrow(reasonToState(reason));
1753 this.finish();
1754 }
1755
1756 private void prepareSessionInitiate(
1757 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1758 final SessionDescription sessionDescription =
1759 SessionDescription.parse(webRTCSessionDescription.description);
1760 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1761 this.initiatorRtpContentMap = rtpContentMap;
1762 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1763 encryptSessionInitiate(rtpContentMap);
1764 Futures.addCallback(
1765 outgoingContentMapFuture,
1766 new FutureCallback<RtpContentMap>() {
1767 @Override
1768 public void onSuccess(final RtpContentMap outgoingContentMap) {
1769 sendSessionInitiate(outgoingContentMap, targetState);
1770 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1771 }
1772
1773 @Override
1774 public void onFailure(@NonNull final Throwable throwable) {
1775 failureToInitiateSession(throwable, targetState);
1776 }
1777 },
1778 MoreExecutors.directExecutor());
1779 }
1780
1781 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1782 if (isTerminated()) {
1783 Log.w(
1784 Config.LOGTAG,
1785 id.account.getJid().asBareJid()
1786 + ": preparing session was too slow. already terminated. nothing to do.");
1787 return;
1788 }
1789 this.transitionOrThrow(targetState);
1790 final JinglePacket sessionInitiate =
1791 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1792 send(sessionInitiate);
1793 }
1794
1795 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1796 final RtpContentMap rtpContentMap) {
1797 if (this.omemoVerification.hasDeviceId()) {
1798 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1799 verifiedPayloadFuture =
1800 id.account
1801 .getAxolotlService()
1802 .encrypt(
1803 rtpContentMap,
1804 id.with,
1805 omemoVerification.getDeviceId());
1806 final ListenableFuture<RtpContentMap> future =
1807 Futures.transform(
1808 verifiedPayloadFuture,
1809 verifiedPayload -> {
1810 omemoVerification.setSessionFingerprint(
1811 verifiedPayload.getFingerprint());
1812 return verifiedPayload.getPayload();
1813 },
1814 MoreExecutors.directExecutor());
1815 if (Config.REQUIRE_RTP_VERIFICATION) {
1816 return future;
1817 }
1818 return Futures.catching(
1819 future,
1820 CryptoFailedException.class,
1821 e -> {
1822 Log.w(
1823 Config.LOGTAG,
1824 id.account.getJid().asBareJid()
1825 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1826 e);
1827 return rtpContentMap;
1828 },
1829 MoreExecutors.directExecutor());
1830 } else {
1831 return Futures.immediateFuture(rtpContentMap);
1832 }
1833 }
1834
1835 private void sendSessionTerminate(final Reason reason) {
1836 sendSessionTerminate(reason, null);
1837 }
1838
1839 private void sendSessionTerminate(final Reason reason, final String text) {
1840 final State previous = this.state;
1841 final State target = reasonToState(reason);
1842 transitionOrThrow(target);
1843 if (previous != State.NULL) {
1844 writeLogMessage(target);
1845 }
1846 final JinglePacket jinglePacket =
1847 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1848 jinglePacket.setReason(reason, text);
1849 Log.d(Config.LOGTAG, jinglePacket.toString());
1850 send(jinglePacket);
1851 finish();
1852 }
1853
1854 private void sendTransportInfo(
1855 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1856 final RtpContentMap transportInfo;
1857 try {
1858 final RtpContentMap rtpContentMap =
1859 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1860 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1861 } catch (final Exception e) {
1862 Log.d(
1863 Config.LOGTAG,
1864 id.account.getJid().asBareJid()
1865 + ": unable to prepare transport-info from candidate for content="
1866 + contentName);
1867 return;
1868 }
1869 final JinglePacket jinglePacket =
1870 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1871 send(jinglePacket);
1872 }
1873
1874 private void send(final JinglePacket jinglePacket) {
1875 jinglePacket.setTo(id.with);
1876 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1877 }
1878
1879 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1880 if (response.getType() == IqPacket.TYPE.ERROR) {
1881 handleIqErrorResponse(response);
1882 return;
1883 }
1884 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1885 handleIqTimeoutResponse(response);
1886 }
1887 }
1888
1889 private void handleIqErrorResponse(final IqPacket response) {
1890 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1891 final String errorCondition = response.getErrorCondition();
1892 Log.d(
1893 Config.LOGTAG,
1894 id.account.getJid().asBareJid()
1895 + ": received IQ-error from "
1896 + response.getFrom()
1897 + " in RTP session. "
1898 + errorCondition);
1899 if (isTerminated()) {
1900 Log.i(
1901 Config.LOGTAG,
1902 id.account.getJid().asBareJid()
1903 + ": ignoring error because session was already terminated");
1904 return;
1905 }
1906 this.webRTCWrapper.close();
1907 final State target;
1908 if (Arrays.asList(
1909 "service-unavailable",
1910 "recipient-unavailable",
1911 "remote-server-not-found",
1912 "remote-server-timeout")
1913 .contains(errorCondition)) {
1914 target = State.TERMINATED_CONNECTIVITY_ERROR;
1915 } else {
1916 target = State.TERMINATED_APPLICATION_FAILURE;
1917 }
1918 transitionOrThrow(target);
1919 this.finish();
1920 }
1921
1922 private void handleIqTimeoutResponse(final IqPacket response) {
1923 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1924 Log.d(
1925 Config.LOGTAG,
1926 id.account.getJid().asBareJid()
1927 + ": received IQ timeout in RTP session with "
1928 + id.with
1929 + ". terminating with connectivity error");
1930 if (isTerminated()) {
1931 Log.i(
1932 Config.LOGTAG,
1933 id.account.getJid().asBareJid()
1934 + ": ignoring error because session was already terminated");
1935 return;
1936 }
1937 this.webRTCWrapper.close();
1938 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
1939 this.finish();
1940 }
1941
1942 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1943 Log.d(
1944 Config.LOGTAG,
1945 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1946 this.webRTCWrapper.close();
1947 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1948 respondWithOutOfOrder(jinglePacket);
1949 this.finish();
1950 }
1951
1952 private void respondWithTieBreak(final JinglePacket jinglePacket) {
1953 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
1954 }
1955
1956 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1957 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
1958 }
1959
1960 void respondWithJingleError(
1961 final IqPacket original,
1962 String jingleCondition,
1963 String condition,
1964 String conditionType) {
1965 jingleConnectionManager.respondWithJingleError(
1966 id.account, original, jingleCondition, condition, conditionType);
1967 }
1968
1969 private void respondOk(final JinglePacket jinglePacket) {
1970 xmppConnectionService.sendIqPacket(
1971 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1972 }
1973
1974 public RtpEndUserState getEndUserState() {
1975 switch (this.state) {
1976 case NULL:
1977 case PROPOSED:
1978 case SESSION_INITIALIZED:
1979 if (isInitiator()) {
1980 return RtpEndUserState.RINGING;
1981 } else {
1982 return RtpEndUserState.INCOMING_CALL;
1983 }
1984 case PROCEED:
1985 if (isInitiator()) {
1986 return RtpEndUserState.RINGING;
1987 } else {
1988 return RtpEndUserState.ACCEPTING_CALL;
1989 }
1990 case SESSION_INITIALIZED_PRE_APPROVED:
1991 if (isInitiator()) {
1992 return RtpEndUserState.RINGING;
1993 } else {
1994 return RtpEndUserState.CONNECTING;
1995 }
1996 case SESSION_ACCEPTED:
1997 final ContentAddition ca = getPendingContentAddition();
1998 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
1999 return RtpEndUserState.INCOMING_CONTENT_ADD;
2000 }
2001 return getPeerConnectionStateAsEndUserState();
2002 case REJECTED:
2003 case REJECTED_RACED:
2004 case TERMINATED_DECLINED_OR_BUSY:
2005 if (isInitiator()) {
2006 return RtpEndUserState.DECLINED_OR_BUSY;
2007 } else {
2008 return RtpEndUserState.ENDED;
2009 }
2010 case TERMINATED_SUCCESS:
2011 case ACCEPTED:
2012 case RETRACTED:
2013 case TERMINATED_CANCEL_OR_TIMEOUT:
2014 return RtpEndUserState.ENDED;
2015 case RETRACTED_RACED:
2016 if (isInitiator()) {
2017 return RtpEndUserState.ENDED;
2018 } else {
2019 return RtpEndUserState.RETRACTED;
2020 }
2021 case TERMINATED_CONNECTIVITY_ERROR:
2022 return zeroDuration()
2023 ? RtpEndUserState.CONNECTIVITY_ERROR
2024 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2025 case TERMINATED_APPLICATION_FAILURE:
2026 return RtpEndUserState.APPLICATION_ERROR;
2027 case TERMINATED_SECURITY_ERROR:
2028 return RtpEndUserState.SECURITY_ERROR;
2029 }
2030 throw new IllegalStateException(
2031 String.format("%s has no equivalent EndUserState", this.state));
2032 }
2033
2034 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2035 final PeerConnection.PeerConnectionState state;
2036 try {
2037 state = webRTCWrapper.getState();
2038 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2039 // We usually close the WebRTCWrapper *before* transitioning so we might still
2040 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2041 return RtpEndUserState.ENDING_CALL;
2042 }
2043 switch (state) {
2044 case CONNECTED:
2045 return RtpEndUserState.CONNECTED;
2046 case NEW:
2047 case CONNECTING:
2048 return RtpEndUserState.CONNECTING;
2049 case CLOSED:
2050 return RtpEndUserState.ENDING_CALL;
2051 default:
2052 return zeroDuration()
2053 ? RtpEndUserState.CONNECTIVITY_ERROR
2054 : RtpEndUserState.RECONNECTING;
2055 }
2056 }
2057
2058 public ContentAddition getPendingContentAddition() {
2059 final RtpContentMap in = this.incomingContentAdd;
2060 final RtpContentMap out = this.outgoingContentAdd;
2061 if (out != null) {
2062 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2063 } else if (in != null) {
2064 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2065 } else {
2066 return null;
2067 }
2068 }
2069
2070 public Set<Media> getMedia() {
2071 final State current = getState();
2072 if (current == State.NULL) {
2073 if (isInitiator()) {
2074 return Preconditions.checkNotNull(
2075 this.proposedMedia, "RTP connection has not been initialized properly");
2076 }
2077 throw new IllegalStateException("RTP connection has not been initialized yet");
2078 }
2079 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2080 return Preconditions.checkNotNull(
2081 this.proposedMedia, "RTP connection has not been initialized properly");
2082 }
2083 final RtpContentMap localContentMap = getLocalContentMap();
2084 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2085 if (localContentMap != null) {
2086 return localContentMap.getMedia();
2087 } else if (initiatorContentMap != null) {
2088 return initiatorContentMap.getMedia();
2089 } else if (isTerminated()) {
2090 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2091 } else {
2092 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2093 }
2094 }
2095
2096 public boolean isVerified() {
2097 final String fingerprint = this.omemoVerification.getFingerprint();
2098 if (fingerprint == null) {
2099 return false;
2100 }
2101 final FingerprintStatus status =
2102 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2103 return status != null && status.isVerified();
2104 }
2105
2106 public boolean addMedia(final Media media) {
2107 final Set<Media> currentMedia = getMedia();
2108 if (currentMedia.contains(media)) {
2109 throw new IllegalStateException(String.format("%s has already been proposed", media));
2110 }
2111 // TODO add state protection - can only add while ACCEPTED or so
2112 Log.d(Config.LOGTAG,"adding media: "+media);
2113 return webRTCWrapper.addTrack(media);
2114 }
2115
2116 public synchronized void acceptCall() {
2117 switch (this.state) {
2118 case PROPOSED:
2119 cancelRingingTimeout();
2120 acceptCallFromProposed();
2121 break;
2122 case SESSION_INITIALIZED:
2123 cancelRingingTimeout();
2124 acceptCallFromSessionInitialized();
2125 break;
2126 case ACCEPTED:
2127 Log.w(
2128 Config.LOGTAG,
2129 id.account.getJid().asBareJid()
2130 + ": the call has already been accepted with another client. UI was just lagging behind");
2131 break;
2132 case PROCEED:
2133 case SESSION_ACCEPTED:
2134 Log.w(
2135 Config.LOGTAG,
2136 id.account.getJid().asBareJid()
2137 + ": the call has already been accepted. user probably double tapped the UI");
2138 break;
2139 default:
2140 throw new IllegalStateException("Can not accept call from " + this.state);
2141 }
2142 }
2143
2144 public void notifyPhoneCall() {
2145 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2146 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2147 rejectCall();
2148 } else {
2149 endCall();
2150 }
2151 }
2152
2153 public synchronized void rejectCall() {
2154 if (isTerminated()) {
2155 Log.w(
2156 Config.LOGTAG,
2157 id.account.getJid().asBareJid()
2158 + ": received rejectCall() when session has already been terminated. nothing to do");
2159 return;
2160 }
2161 switch (this.state) {
2162 case PROPOSED:
2163 rejectCallFromProposed();
2164 break;
2165 case SESSION_INITIALIZED:
2166 rejectCallFromSessionInitiate();
2167 break;
2168 default:
2169 throw new IllegalStateException("Can not reject call from " + this.state);
2170 }
2171 }
2172
2173 public synchronized void endCall() {
2174 if (isTerminated()) {
2175 Log.w(
2176 Config.LOGTAG,
2177 id.account.getJid().asBareJid()
2178 + ": received endCall() when session has already been terminated. nothing to do");
2179 return;
2180 }
2181 if (isInState(State.PROPOSED) && !isInitiator()) {
2182 rejectCallFromProposed();
2183 return;
2184 }
2185 if (isInState(State.PROCEED)) {
2186 if (isInitiator()) {
2187 retractFromProceed();
2188 } else {
2189 rejectCallFromProceed();
2190 }
2191 return;
2192 }
2193 if (isInitiator()
2194 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2195 this.webRTCWrapper.close();
2196 sendSessionTerminate(Reason.CANCEL);
2197 return;
2198 }
2199 if (isInState(State.SESSION_INITIALIZED)) {
2200 rejectCallFromSessionInitiate();
2201 return;
2202 }
2203 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2204 this.webRTCWrapper.close();
2205 sendSessionTerminate(Reason.SUCCESS);
2206 return;
2207 }
2208 if (isInState(
2209 State.TERMINATED_APPLICATION_FAILURE,
2210 State.TERMINATED_CONNECTIVITY_ERROR,
2211 State.TERMINATED_DECLINED_OR_BUSY)) {
2212 Log.d(
2213 Config.LOGTAG,
2214 "ignoring request to end call because already in state " + this.state);
2215 return;
2216 }
2217 throw new IllegalStateException(
2218 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2219 }
2220
2221 private void retractFromProceed() {
2222 Log.d(Config.LOGTAG, "retract from proceed");
2223 this.sendJingleMessage("retract");
2224 closeTransitionLogFinish(State.RETRACTED_RACED);
2225 }
2226
2227 private void closeTransitionLogFinish(final State state) {
2228 this.webRTCWrapper.close();
2229 transitionOrThrow(state);
2230 writeLogMessage(state);
2231 finish();
2232 }
2233
2234 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2235 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2236 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2237 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2238 }
2239
2240 private void acceptCallFromProposed() {
2241 transitionOrThrow(State.PROCEED);
2242 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2243 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2244 this.sendJingleMessage("proceed");
2245 }
2246
2247 private void rejectCallFromProposed() {
2248 transitionOrThrow(State.REJECTED);
2249 writeLogMessageMissed();
2250 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2251 this.sendJingleMessage("reject");
2252 finish();
2253 }
2254
2255 private void rejectCallFromProceed() {
2256 this.sendJingleMessage("reject");
2257 closeTransitionLogFinish(State.REJECTED_RACED);
2258 }
2259
2260 private void rejectCallFromSessionInitiate() {
2261 webRTCWrapper.close();
2262 sendSessionTerminate(Reason.DECLINE);
2263 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2264 }
2265
2266 private void sendJingleMessage(final String action) {
2267 sendJingleMessage(action, id.with);
2268 }
2269
2270 private void sendJingleMessage(final String action, final Jid to) {
2271 final MessagePacket messagePacket = new MessagePacket();
2272 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2273 messagePacket.setTo(to);
2274 final Element intent =
2275 messagePacket
2276 .addChild(action, Namespace.JINGLE_MESSAGE)
2277 .setAttribute("id", id.sessionId);
2278 if ("proceed".equals(action)) {
2279 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2280 if (isOmemoEnabled()) {
2281 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2282 final Element device =
2283 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2284 device.setAttribute("id", deviceId);
2285 }
2286 }
2287 messagePacket.addChild("store", "urn:xmpp:hints");
2288 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2289 }
2290
2291 private boolean isOmemoEnabled() {
2292 final Conversational conversational = message.getConversation();
2293 if (conversational instanceof Conversation) {
2294 return ((Conversation) conversational).getNextEncryption()
2295 == Message.ENCRYPTION_AXOLOTL;
2296 }
2297 return false;
2298 }
2299
2300 private void acceptCallFromSessionInitialized() {
2301 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2302 sendSessionAccept();
2303 }
2304
2305 private synchronized boolean isInState(State... state) {
2306 return Arrays.asList(state).contains(this.state);
2307 }
2308
2309 private boolean transition(final State target) {
2310 return transition(target, null);
2311 }
2312
2313 private synchronized boolean transition(final State target, final Runnable runnable) {
2314 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2315 if (validTransitions != null && validTransitions.contains(target)) {
2316 this.state = target;
2317 if (runnable != null) {
2318 runnable.run();
2319 }
2320 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2321 updateEndUserState();
2322 updateOngoingCallNotification();
2323 return true;
2324 } else {
2325 return false;
2326 }
2327 }
2328
2329 void transitionOrThrow(final State target) {
2330 if (!transition(target)) {
2331 throw new IllegalStateException(
2332 String.format("Unable to transition from %s to %s", this.state, target));
2333 }
2334 }
2335
2336 @Override
2337 public void onIceCandidate(final IceCandidate iceCandidate) {
2338 final RtpContentMap rtpContentMap =
2339 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2340 final IceUdpTransportInfo.Credentials credentials;
2341 try {
2342 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2343 } catch (final IllegalArgumentException e) {
2344 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2345 return;
2346 }
2347 final String uFrag = credentials.ufrag;
2348 final IceUdpTransportInfo.Candidate candidate =
2349 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2350 if (candidate == null) {
2351 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2352 return;
2353 }
2354 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2355 sendTransportInfo(iceCandidate.sdpMid, candidate);
2356 }
2357
2358 @Override
2359 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2360 Log.d(
2361 Config.LOGTAG,
2362 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2363 this.stateHistory.add(newState);
2364 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2365 this.sessionDuration.start();
2366 updateOngoingCallNotification();
2367 } else if (this.sessionDuration.isRunning()) {
2368 this.sessionDuration.stop();
2369 updateOngoingCallNotification();
2370 }
2371
2372 final boolean neverConnected =
2373 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2374
2375 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2376 if (neverConnected) {
2377 if (isTerminated()) {
2378 Log.d(
2379 Config.LOGTAG,
2380 id.account.getJid().asBareJid()
2381 + ": not sending session-terminate after connectivity error because session is already in state "
2382 + this.state);
2383 return;
2384 }
2385 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2386 return;
2387 } else {
2388 this.restartIce();
2389 }
2390 }
2391 updateEndUserState();
2392 }
2393
2394 private void restartIce() {
2395 this.stateHistory.clear();
2396 this.webRTCWrapper.restartIceAsync();
2397 }
2398
2399 @Override
2400 public void onRenegotiationNeeded() {
2401 this.webRTCWrapper.execute(this::renegotiate);
2402 }
2403
2404 private void renegotiate() {
2405 final SessionDescription sessionDescription;
2406 try {
2407 sessionDescription = setLocalSessionDescription();
2408 } catch (final Exception e) {
2409 final Throwable cause = Throwables.getRootCause(e);
2410 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2411 webRTCWrapper.close();
2412 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2413 return;
2414 }
2415 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2416 final RtpContentMap currentContentMap = getLocalContentMap();
2417 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2418 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2419
2420 Log.d(
2421 Config.LOGTAG,
2422 id.getAccount().getJid().asBareJid()
2423 + ": renegotiate. iceRestart="
2424 + iceRestart
2425 + " content id diff="
2426 + diff);
2427
2428 if (diff.hasModifications() && iceRestart) {
2429 webRTCWrapper.close();
2430 sendSessionTerminate(
2431 Reason.FAILED_APPLICATION,
2432 "WebRTC unexpectedly tried to modify content and transport at once");
2433 return;
2434 }
2435
2436 if (iceRestart) {
2437 initiateIceRestart(rtpContentMap);
2438 return;
2439 } else if (diff.isEmpty()) {
2440 Log.d(
2441 Config.LOGTAG,
2442 "renegotiation. nothing to do. SignalingState="
2443 + this.webRTCWrapper.getSignalingState());
2444 }
2445
2446 if (diff.added.size() > 0) {
2447 modifyLocalContentMap(rtpContentMap);
2448 sendContentAdd(rtpContentMap, diff.added);
2449 }
2450 }
2451
2452 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2453 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2454 final JinglePacket jinglePacket =
2455 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2456 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2457 jinglePacket.setTo(id.with);
2458 xmppConnectionService.sendIqPacket(
2459 id.account,
2460 jinglePacket,
2461 (account, response) -> {
2462 if (response.getType() == IqPacket.TYPE.RESULT) {
2463 Log.d(Config.LOGTAG, "received success to our ice restart");
2464 setLocalContentMap(rtpContentMap);
2465 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2466 return;
2467 }
2468 if (response.getType() == IqPacket.TYPE.ERROR) {
2469 if (isTieBreak(response)) {
2470 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2471 return;
2472 }
2473 handleIqErrorResponse(response);
2474 }
2475 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2476 handleIqTimeoutResponse(response);
2477 }
2478 });
2479 }
2480
2481 private boolean isTieBreak(final IqPacket response) {
2482 final Element error = response.findChild("error");
2483 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2484 }
2485
2486 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2487 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2488 this.outgoingContentAdd = contentAdd;
2489 final JinglePacket jinglePacket =
2490 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2491 jinglePacket.setTo(id.with);
2492 xmppConnectionService.sendIqPacket(
2493 id.account,
2494 jinglePacket,
2495 (connection, response) -> {
2496 if (response.getType() == IqPacket.TYPE.RESULT) {
2497 Log.d(
2498 Config.LOGTAG,
2499 id.getAccount().getJid().asBareJid()
2500 + ": received ACK to our content-add");
2501 return;
2502 }
2503 if (response.getType() == IqPacket.TYPE.ERROR) {
2504 if (isTieBreak(response)) {
2505 this.outgoingContentAdd = null;
2506 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2507 return;
2508 }
2509 handleIqErrorResponse(response);
2510 }
2511 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2512 handleIqTimeoutResponse(response);
2513 }
2514 });
2515 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2516 }
2517
2518 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2519 if (isInitiator()) {
2520 this.initiatorRtpContentMap = rtpContentMap;
2521 } else {
2522 this.responderRtpContentMap = rtpContentMap;
2523 }
2524 }
2525
2526 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2527 if (isInitiator()) {
2528 this.responderRtpContentMap = rtpContentMap;
2529 } else {
2530 this.initiatorRtpContentMap = rtpContentMap;
2531 }
2532 }
2533
2534 // this method is to be used for content map modifications that modify media
2535 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2536 final RtpContentMap activeContents = rtpContentMap.activeContents();
2537 setLocalContentMap(activeContents);
2538 this.webRTCWrapper.switchSpeakerPhonePreference(
2539 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2540 updateEndUserState();
2541 }
2542
2543 private SessionDescription setLocalSessionDescription()
2544 throws ExecutionException, InterruptedException {
2545 final org.webrtc.SessionDescription sessionDescription =
2546 this.webRTCWrapper.setLocalDescription().get();
2547 return SessionDescription.parse(sessionDescription.description);
2548 }
2549
2550 private void closeWebRTCSessionAfterFailedConnection() {
2551 this.webRTCWrapper.close();
2552 synchronized (this) {
2553 if (isTerminated()) {
2554 Log.d(
2555 Config.LOGTAG,
2556 id.account.getJid().asBareJid()
2557 + ": no need to send session-terminate after failed connection. Other party already did");
2558 return;
2559 }
2560 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2561 }
2562 }
2563
2564 public boolean zeroDuration() {
2565 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2566 }
2567
2568 public long getCallDuration() {
2569 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2570 }
2571
2572 public AppRTCAudioManager getAudioManager() {
2573 return webRTCWrapper.getAudioManager();
2574 }
2575
2576 public boolean isMicrophoneEnabled() {
2577 return webRTCWrapper.isMicrophoneEnabled();
2578 }
2579
2580 public boolean setMicrophoneEnabled(final boolean enabled) {
2581 return webRTCWrapper.setMicrophoneEnabled(enabled);
2582 }
2583
2584 public boolean isVideoEnabled() {
2585 return webRTCWrapper.isVideoEnabled();
2586 }
2587
2588 public void setVideoEnabled(final boolean enabled) {
2589 webRTCWrapper.setVideoEnabled(enabled);
2590 }
2591
2592 public boolean isCameraSwitchable() {
2593 return webRTCWrapper.isCameraSwitchable();
2594 }
2595
2596 public boolean isFrontCamera() {
2597 return webRTCWrapper.isFrontCamera();
2598 }
2599
2600 public ListenableFuture<Boolean> switchCamera() {
2601 return webRTCWrapper.switchCamera();
2602 }
2603
2604 @Override
2605 public void onAudioDeviceChanged(
2606 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2607 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2608 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2609 selectedAudioDevice, availableAudioDevices);
2610 }
2611
2612 private void updateEndUserState() {
2613 final RtpEndUserState endUserState = getEndUserState();
2614 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2615 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2616 id.account, id.with, id.sessionId, endUserState);
2617 }
2618
2619 private void updateOngoingCallNotification() {
2620 final State state = this.state;
2621 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2622 final boolean reconnecting;
2623 if (state == State.SESSION_ACCEPTED) {
2624 reconnecting =
2625 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2626 } else {
2627 reconnecting = false;
2628 }
2629 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2630 } else {
2631 xmppConnectionService.removeOngoingCall();
2632 }
2633 }
2634
2635 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2636 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2637 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2638 request.setTo(id.account.getDomain());
2639 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2640 xmppConnectionService.sendIqPacket(
2641 id.account,
2642 request,
2643 (account, response) -> {
2644 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2645 new ImmutableList.Builder<>();
2646 if (response.getType() == IqPacket.TYPE.RESULT) {
2647 final Element services =
2648 response.findChild(
2649 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2650 final List<Element> children =
2651 services == null
2652 ? Collections.emptyList()
2653 : services.getChildren();
2654 for (final Element child : children) {
2655 if ("service".equals(child.getName())) {
2656 final String type = child.getAttribute("type");
2657 final String host = child.getAttribute("host");
2658 final String sport = child.getAttribute("port");
2659 final Integer port =
2660 sport == null ? null : Ints.tryParse(sport);
2661 final String transport = child.getAttribute("transport");
2662 final String username = child.getAttribute("username");
2663 final String password = child.getAttribute("password");
2664 if (Strings.isNullOrEmpty(host) || port == null) {
2665 continue;
2666 }
2667 if (port < 0 || port > 65535) {
2668 continue;
2669 }
2670
2671
2672
2673
2674 if (Arrays.asList("stun", "stuns", "turn", "turns")
2675 .contains(type)
2676 && Arrays.asList("udp", "tcp").contains(transport)) {
2677 if (Arrays.asList("stuns", "turns").contains(type)
2678 && "udp".equals(transport)) {
2679 Log.d(
2680 Config.LOGTAG,
2681 id.account.getJid().asBareJid()
2682 + ": skipping invalid combination of udp/tls in external services");
2683 continue;
2684 }
2685
2686 // STUN URLs do not support a query section since M110
2687 final String uri;
2688 if (Arrays.asList("stun","stuns").contains(type)) {
2689 uri = String.format("%s:%s:%s", type, IP.wrapIPv6(host),port);
2690 } else {
2691 uri = String.format(
2692 "%s:%s:%s?transport=%s",
2693 type,
2694 IP.wrapIPv6(host),
2695 port,
2696 transport);
2697 }
2698
2699 final PeerConnection.IceServer.Builder iceServerBuilder =
2700 PeerConnection.IceServer.builder(uri);
2701 iceServerBuilder.setTlsCertPolicy(
2702 PeerConnection.TlsCertPolicy
2703 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2704 if (username != null && password != null) {
2705 iceServerBuilder.setUsername(username);
2706 iceServerBuilder.setPassword(password);
2707 } else if (Arrays.asList("turn", "turns").contains(type)) {
2708 // The WebRTC spec requires throwing an
2709 // InvalidAccessError when username (from libwebrtc
2710 // source coder)
2711 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2712 Log.d(
2713 Config.LOGTAG,
2714 id.account.getJid().asBareJid()
2715 + ": skipping "
2716 + type
2717 + "/"
2718 + transport
2719 + " without username and password");
2720 continue;
2721 }
2722 final PeerConnection.IceServer iceServer =
2723 iceServerBuilder.createIceServer();
2724 Log.d(
2725 Config.LOGTAG,
2726 id.account.getJid().asBareJid()
2727 + ": discovered ICE Server: "
2728 + iceServer);
2729 listBuilder.add(iceServer);
2730 }
2731 }
2732 }
2733 }
2734 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2735 if (iceServers.size() == 0) {
2736 Log.w(
2737 Config.LOGTAG,
2738 id.account.getJid().asBareJid()
2739 + ": no ICE server found "
2740 + response);
2741 }
2742 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2743 });
2744 } else {
2745 Log.w(
2746 Config.LOGTAG,
2747 id.account.getJid().asBareJid() + ": has no external service discovery");
2748 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2749 }
2750 }
2751
2752 private void finish() {
2753 if (isTerminated()) {
2754 this.cancelRingingTimeout();
2755 this.webRTCWrapper.verifyClosed();
2756 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2757 this.jingleConnectionManager.finishConnectionOrThrow(this);
2758 } else {
2759 throw new IllegalStateException(
2760 String.format("Unable to call finish from %s", this.state));
2761 }
2762 }
2763
2764 private void writeLogMessage(final State state) {
2765 final long duration = getCallDuration();
2766 if (state == State.TERMINATED_SUCCESS
2767 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2768 writeLogMessageSuccess(duration);
2769 } else {
2770 writeLogMessageMissed();
2771 }
2772 }
2773
2774 private void writeLogMessageSuccess(final long duration) {
2775 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2776 this.writeMessage();
2777 }
2778
2779 private void writeLogMessageMissed() {
2780 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2781 this.writeMessage();
2782 }
2783
2784 private void writeMessage() {
2785 final Conversational conversational = message.getConversation();
2786 if (conversational instanceof Conversation) {
2787 ((Conversation) conversational).add(this.message);
2788 xmppConnectionService.createMessageAsync(message);
2789 xmppConnectionService.updateConversationUi();
2790 } else {
2791 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2792 }
2793 }
2794
2795 public State getState() {
2796 return this.state;
2797 }
2798
2799 boolean isTerminated() {
2800 return TERMINATED.contains(this.state);
2801 }
2802
2803 public Optional<VideoTrack> getLocalVideoTrack() {
2804 return webRTCWrapper.getLocalVideoTrack();
2805 }
2806
2807 public Optional<VideoTrack> getRemoteVideoTrack() {
2808 return webRTCWrapper.getRemoteVideoTrack();
2809 }
2810
2811 public EglBase.Context getEglBaseContext() {
2812 return webRTCWrapper.getEglBaseContext();
2813 }
2814
2815 void setProposedMedia(final Set<Media> media) {
2816 this.proposedMedia = media;
2817 }
2818
2819 public void fireStateUpdate() {
2820 final RtpEndUserState endUserState = getEndUserState();
2821 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2822 id.account, id.with, id.sessionId, endUserState);
2823 }
2824
2825 public boolean isSwitchToVideoAvailable() {
2826 final boolean prerequisite =
2827 Media.audioOnly(getMedia())
2828 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2829 .contains(getEndUserState());
2830 return prerequisite && remoteHasVideoFeature();
2831 }
2832
2833 private boolean remoteHasVideoFeature() {
2834 final Contact contact = id.getContact();
2835 final Presence presence =
2836 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2837 final ServiceDiscoveryResult serviceDiscoveryResult =
2838 presence == null ? null : presence.getServiceDiscoveryResult();
2839 final List<String> features =
2840 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
2841 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
2842 }
2843
2844 private interface OnIceServersDiscovered {
2845 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2846 }
2847}