1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4import android.os.Environment;
5
6import androidx.annotation.NonNull;
7import androidx.annotation.Nullable;
8
9import com.google.common.base.Joiner;
10import com.google.common.base.Optional;
11import com.google.common.base.Preconditions;
12import com.google.common.base.Stopwatch;
13import com.google.common.base.Strings;
14import com.google.common.base.Throwables;
15import com.google.common.collect.Collections2;
16import com.google.common.collect.ImmutableList;
17import com.google.common.collect.ImmutableMap;
18import com.google.common.collect.ImmutableSet;
19import com.google.common.collect.Sets;
20import com.google.common.primitives.Ints;
21import com.google.common.util.concurrent.FutureCallback;
22import com.google.common.util.concurrent.Futures;
23import com.google.common.util.concurrent.ListenableFuture;
24import com.google.common.util.concurrent.MoreExecutors;
25
26import org.webrtc.DtmfSender;
27import org.webrtc.EglBase;
28import org.webrtc.IceCandidate;
29import org.webrtc.PeerConnection;
30import org.webrtc.VideoTrack;
31
32import java.io.File;
33import java.io.IOException;
34import java.util.Arrays;
35import java.util.Collection;
36import java.util.Collections;
37import java.util.LinkedList;
38import java.util.List;
39import java.util.Map;
40import java.util.Queue;
41import java.util.Set;
42import java.util.concurrent.ExecutionException;
43import java.util.concurrent.ScheduledFuture;
44import java.util.concurrent.TimeUnit;
45
46import eu.siacs.conversations.BuildConfig;
47import eu.siacs.conversations.Config;
48import eu.siacs.conversations.crypto.axolotl.AxolotlService;
49import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
50import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
51import eu.siacs.conversations.entities.Account;
52import eu.siacs.conversations.entities.Contact;
53import eu.siacs.conversations.entities.Conversation;
54import eu.siacs.conversations.entities.Conversational;
55import eu.siacs.conversations.entities.Message;
56import eu.siacs.conversations.entities.Presence;
57import eu.siacs.conversations.entities.RtpSessionStatus;
58import eu.siacs.conversations.entities.ServiceDiscoveryResult;
59import eu.siacs.conversations.services.AppRTCAudioManager;
60import eu.siacs.conversations.utils.IP;
61import eu.siacs.conversations.xml.Element;
62import eu.siacs.conversations.xml.Namespace;
63import eu.siacs.conversations.xmpp.Jid;
64import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
65import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
66import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
67import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
68import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
69import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
70import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
71import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
72import eu.siacs.conversations.xmpp.stanzas.IqPacket;
73import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
74
75public class JingleRtpConnection extends AbstractJingleConnection
76 implements WebRTCWrapper.EventCallback {
77
78 public static final List<State> STATES_SHOWING_ONGOING_CALL =
79 Arrays.asList(
80 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
81 private static final long BUSY_TIME_OUT = 30;
82 private static final List<State> TERMINATED =
83 Arrays.asList(
84 State.ACCEPTED,
85 State.REJECTED,
86 State.REJECTED_RACED,
87 State.RETRACTED,
88 State.RETRACTED_RACED,
89 State.TERMINATED_SUCCESS,
90 State.TERMINATED_DECLINED_OR_BUSY,
91 State.TERMINATED_CONNECTIVITY_ERROR,
92 State.TERMINATED_CANCEL_OR_TIMEOUT,
93 State.TERMINATED_APPLICATION_FAILURE,
94 State.TERMINATED_SECURITY_ERROR);
95
96 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
97
98 static {
99 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
100 new ImmutableMap.Builder<>();
101 transitionBuilder.put(
102 State.NULL,
103 ImmutableList.of(
104 State.PROPOSED,
105 State.SESSION_INITIALIZED,
106 State.TERMINATED_APPLICATION_FAILURE,
107 State.TERMINATED_SECURITY_ERROR));
108 transitionBuilder.put(
109 State.PROPOSED,
110 ImmutableList.of(
111 State.ACCEPTED,
112 State.PROCEED,
113 State.REJECTED,
114 State.RETRACTED,
115 State.TERMINATED_APPLICATION_FAILURE,
116 State.TERMINATED_SECURITY_ERROR,
117 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
118 // rebinds
119 ));
120 transitionBuilder.put(
121 State.PROCEED,
122 ImmutableList.of(
123 State.REJECTED_RACED,
124 State.RETRACTED_RACED,
125 State.SESSION_INITIALIZED_PRE_APPROVED,
126 State.TERMINATED_SUCCESS,
127 State.TERMINATED_APPLICATION_FAILURE,
128 State.TERMINATED_SECURITY_ERROR,
129 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
130 // bounces of the proceed message
131 ));
132 transitionBuilder.put(
133 State.SESSION_INITIALIZED,
134 ImmutableList.of(
135 State.SESSION_ACCEPTED,
136 State.TERMINATED_SUCCESS,
137 State.TERMINATED_DECLINED_OR_BUSY,
138 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
139 // and IQ timeouts
140 State.TERMINATED_CANCEL_OR_TIMEOUT,
141 State.TERMINATED_APPLICATION_FAILURE,
142 State.TERMINATED_SECURITY_ERROR));
143 transitionBuilder.put(
144 State.SESSION_INITIALIZED_PRE_APPROVED,
145 ImmutableList.of(
146 State.SESSION_ACCEPTED,
147 State.TERMINATED_SUCCESS,
148 State.TERMINATED_DECLINED_OR_BUSY,
149 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
150 // and IQ timeouts
151 State.TERMINATED_CANCEL_OR_TIMEOUT,
152 State.TERMINATED_APPLICATION_FAILURE,
153 State.TERMINATED_SECURITY_ERROR));
154 transitionBuilder.put(
155 State.SESSION_ACCEPTED,
156 ImmutableList.of(
157 State.TERMINATED_SUCCESS,
158 State.TERMINATED_DECLINED_OR_BUSY,
159 State.TERMINATED_CONNECTIVITY_ERROR,
160 State.TERMINATED_CANCEL_OR_TIMEOUT,
161 State.TERMINATED_APPLICATION_FAILURE,
162 State.TERMINATED_SECURITY_ERROR));
163 VALID_TRANSITIONS = transitionBuilder.build();
164 }
165
166 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
167 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
168 pendingIceCandidates = new LinkedList<>();
169 private final OmemoVerification omemoVerification = new OmemoVerification();
170 private final Message message;
171 private State state = State.NULL;
172 private Set<Media> proposedMedia;
173 private RtpContentMap initiatorRtpContentMap;
174 private RtpContentMap responderRtpContentMap;
175 private RtpContentMap incomingContentAdd;
176 private RtpContentMap outgoingContentAdd;
177 private IceUdpTransportInfo.Setup peerDtlsSetup;
178 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
179 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
180 private ScheduledFuture<?> ringingTimeoutFuture;
181 private final long created = System.currentTimeMillis() / 1000L;
182
183 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
184 super(jingleConnectionManager, id, initiator);
185 final Conversation conversation =
186 jingleConnectionManager
187 .getXmppConnectionService()
188 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
189 this.message =
190 new Message(
191 conversation,
192 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
193 Message.TYPE_RTP_SESSION,
194 id.sessionId);
195 }
196
197 private static State reasonToState(Reason reason) {
198 switch (reason) {
199 case SUCCESS:
200 return State.TERMINATED_SUCCESS;
201 case DECLINE:
202 case BUSY:
203 return State.TERMINATED_DECLINED_OR_BUSY;
204 case CANCEL:
205 case TIMEOUT:
206 return State.TERMINATED_CANCEL_OR_TIMEOUT;
207 case SECURITY_ERROR:
208 return State.TERMINATED_SECURITY_ERROR;
209 case FAILED_APPLICATION:
210 case UNSUPPORTED_TRANSPORTS:
211 case UNSUPPORTED_APPLICATIONS:
212 return State.TERMINATED_APPLICATION_FAILURE;
213 default:
214 return State.TERMINATED_CONNECTIVITY_ERROR;
215 }
216 }
217
218 @Override
219 synchronized void deliverPacket(final JinglePacket jinglePacket) {
220 switch (jinglePacket.getAction()) {
221 case SESSION_INITIATE:
222 receiveSessionInitiate(jinglePacket);
223 break;
224 case TRANSPORT_INFO:
225 receiveTransportInfo(jinglePacket);
226 break;
227 case SESSION_ACCEPT:
228 receiveSessionAccept(jinglePacket);
229 break;
230 case SESSION_TERMINATE:
231 receiveSessionTerminate(jinglePacket);
232 break;
233 case CONTENT_ADD:
234 receiveContentAdd(jinglePacket);
235 break;
236 case CONTENT_ACCEPT:
237 receiveContentAccept(jinglePacket);
238 break;
239 case CONTENT_REJECT:
240 receiveContentReject(jinglePacket);
241 break;
242 case CONTENT_REMOVE:
243 receiveContentRemove(jinglePacket);
244 break;
245 default:
246 respondOk(jinglePacket);
247 Log.d(
248 Config.LOGTAG,
249 String.format(
250 "%s: received unhandled jingle action %s",
251 id.account.getJid().asBareJid(), jinglePacket.getAction()));
252 break;
253 }
254 }
255
256 @Override
257 synchronized void notifyRebound() {
258 if (isTerminated()) {
259 return;
260 }
261 webRTCWrapper.close();
262 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
263 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
264 }
265 if (isInState(
266 State.SESSION_INITIALIZED,
267 State.SESSION_INITIALIZED_PRE_APPROVED,
268 State.SESSION_ACCEPTED)) {
269 // we might have already changed resources (full jid) at this point; so this might not
270 // even reach the other party
271 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
272 } else {
273 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
274 finish();
275 }
276 }
277
278 public boolean applyDtmfTone(String tone) {
279 return webRTCWrapper.applyDtmfTone(tone);
280 }
281
282 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
283 respondOk(jinglePacket);
284 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
285 final State previous = this.state;
286 Log.d(
287 Config.LOGTAG,
288 id.account.getJid().asBareJid()
289 + ": received session terminate reason="
290 + wrapper.reason
291 + "("
292 + Strings.nullToEmpty(wrapper.text)
293 + ") while in state "
294 + previous);
295 if (TERMINATED.contains(previous)) {
296 Log.d(
297 Config.LOGTAG,
298 id.account.getJid().asBareJid()
299 + ": ignoring session terminate because already in "
300 + previous);
301 return;
302 }
303 webRTCWrapper.close();
304 final State target = reasonToState(wrapper.reason);
305 transitionOrThrow(target);
306 writeLogMessage(target);
307 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
308 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
309 }
310 finish();
311 }
312
313 private void receiveTransportInfo(final JinglePacket jinglePacket) {
314 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
315 // INITIALIZED only after transport-info has been received
316 if (isInState(
317 State.NULL,
318 State.PROCEED,
319 State.SESSION_INITIALIZED,
320 State.SESSION_INITIALIZED_PRE_APPROVED,
321 State.SESSION_ACCEPTED)) {
322 final RtpContentMap contentMap;
323 try {
324 contentMap = RtpContentMap.of(jinglePacket);
325 } catch (final IllegalArgumentException | NullPointerException e) {
326 Log.d(
327 Config.LOGTAG,
328 id.account.getJid().asBareJid()
329 + ": improperly formatted contents; ignoring",
330 e);
331 respondOk(jinglePacket);
332 return;
333 }
334 receiveTransportInfo(jinglePacket, contentMap);
335 } else {
336 if (isTerminated()) {
337 respondOk(jinglePacket);
338 Log.d(
339 Config.LOGTAG,
340 id.account.getJid().asBareJid()
341 + ": ignoring out-of-order transport info; we where already terminated");
342 } else {
343 Log.d(
344 Config.LOGTAG,
345 id.account.getJid().asBareJid()
346 + ": received transport info while in state="
347 + this.state);
348 terminateWithOutOfOrder(jinglePacket);
349 }
350 }
351 }
352
353 private void receiveTransportInfo(
354 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
355 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
356 contentMap.contents.entrySet();
357 if (this.state == State.SESSION_ACCEPTED) {
358 // zero candidates + modified credentials are an ICE restart offer
359 if (checkForIceRestart(jinglePacket, contentMap)) {
360 return;
361 }
362 respondOk(jinglePacket);
363 try {
364 processCandidates(candidates);
365 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
366 Log.w(
367 Config.LOGTAG,
368 id.account.getJid().asBareJid()
369 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
370 }
371 } else {
372 respondOk(jinglePacket);
373 pendingIceCandidates.addAll(candidates);
374 }
375 }
376
377 private void receiveContentAdd(final JinglePacket jinglePacket) {
378 final RtpContentMap modification;
379 try {
380 modification = RtpContentMap.of(jinglePacket);
381 modification.requireContentDescriptions();
382 } catch (final RuntimeException e) {
383 Log.d(
384 Config.LOGTAG,
385 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
386 Throwables.getRootCause(e));
387 respondOk(jinglePacket);
388 webRTCWrapper.close();
389 sendSessionTerminate(Reason.of(e), e.getMessage());
390 return;
391 }
392 if (isInState(State.SESSION_ACCEPTED)) {
393 receiveContentAdd(jinglePacket, modification);
394 } else {
395 terminateWithOutOfOrder(jinglePacket);
396 }
397 }
398
399 private void receiveContentAdd(
400 final JinglePacket jinglePacket, final RtpContentMap modification) {
401 final RtpContentMap remote = getRemoteContentMap();
402 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
403 respondOk(jinglePacket);
404 this.webRTCWrapper.close();
405 sendSessionTerminate(
406 Reason.FAILED_APPLICATION,
407 String.format(
408 "contents with names %s already exists",
409 Joiner.on(", ").join(modification.getNames())));
410 return;
411 }
412 final ContentAddition contentAddition =
413 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
414
415 final RtpContentMap outgoing = this.outgoingContentAdd;
416 final Set<ContentAddition.Summary> outgoingContentAddSummary =
417 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
418
419 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
420 if (isInitiator()) {
421 Log.d(
422 Config.LOGTAG,
423 id.getAccount().getJid().asBareJid()
424 + ": respond with tie break to matching content-add offer");
425 respondWithTieBreak(jinglePacket);
426 } else {
427 Log.d(
428 Config.LOGTAG,
429 id.getAccount().getJid().asBareJid()
430 + ": automatically accept matching content-add offer");
431 acceptContentAdd(contentAddition.summary, modification);
432 }
433 return;
434 }
435
436 // once we can display multiple video tracks we can be more loose with this condition
437 // theoretically it should also be fine to automatically accept audio only contents
438 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
439 Log.d(
440 Config.LOGTAG,
441 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
442 this.incomingContentAdd = modification;
443 respondOk(jinglePacket);
444 updateEndUserState();
445 } else {
446 respondOk(jinglePacket);
447 // TODO do we want to add a reason?
448 rejectContentAdd(modification);
449 }
450 }
451
452 private void receiveContentAccept(final JinglePacket jinglePacket) {
453 final RtpContentMap receivedContentAccept;
454 try {
455 receivedContentAccept = RtpContentMap.of(jinglePacket);
456 receivedContentAccept.requireContentDescriptions();
457 } catch (final RuntimeException e) {
458 Log.d(
459 Config.LOGTAG,
460 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
461 Throwables.getRootCause(e));
462 respondOk(jinglePacket);
463 webRTCWrapper.close();
464 sendSessionTerminate(Reason.of(e), e.getMessage());
465 return;
466 }
467
468 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
469 if (outgoingContentAdd == null) {
470 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
471 terminateWithOutOfOrder(jinglePacket);
472 return;
473 }
474 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
475 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
476 this.outgoingContentAdd = null;
477 respondOk(jinglePacket);
478 receiveContentAccept(receivedContentAccept);
479 } else {
480 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
481 terminateWithOutOfOrder(jinglePacket);
482 }
483 }
484
485 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
486 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
487 final RtpContentMap modifiedContentMap =
488 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
489
490 setRemoteContentMap(modifiedContentMap);
491
492 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
493
494 final org.webrtc.SessionDescription sdp =
495 new org.webrtc.SessionDescription(
496 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
497
498 try {
499 this.webRTCWrapper.setRemoteDescription(sdp).get();
500 } catch (final Exception e) {
501 final Throwable cause = Throwables.getRootCause(e);
502 Log.d(
503 Config.LOGTAG,
504 id.getAccount().getJid().asBareJid()
505 + ": unable to set remote description after receiving content-accept",
506 cause);
507 webRTCWrapper.close();
508 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
509 return;
510 }
511 updateEndUserState();
512 Log.d(
513 Config.LOGTAG,
514 id.getAccount().getJid().asBareJid()
515 + ": remote has accepted content-add "
516 + ContentAddition.summary(receivedContentAccept));
517 }
518
519 private void receiveContentReject(final JinglePacket jinglePacket) {
520 final RtpContentMap receivedContentReject;
521 try {
522 receivedContentReject = RtpContentMap.of(jinglePacket);
523 } catch (final RuntimeException e) {
524 Log.d(
525 Config.LOGTAG,
526 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
527 Throwables.getRootCause(e));
528 respondOk(jinglePacket);
529 this.webRTCWrapper.close();
530 sendSessionTerminate(Reason.of(e), e.getMessage());
531 return;
532 }
533
534 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
535 if (outgoingContentAdd == null) {
536 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
537 terminateWithOutOfOrder(jinglePacket);
538 return;
539 }
540 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
541 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
542 this.outgoingContentAdd = null;
543 respondOk(jinglePacket);
544 Log.d(Config.LOGTAG,jinglePacket.toString());
545 receiveContentReject(ourSummary);
546 } else {
547 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
548 terminateWithOutOfOrder(jinglePacket);
549 }
550 }
551
552 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
553 try {
554 this.webRTCWrapper.removeTrack(Media.VIDEO);
555 final RtpContentMap localContentMap = customRollback();
556 modifyLocalContentMap(localContentMap);
557 } catch (final Exception e) {
558 final Throwable cause = Throwables.getRootCause(e);
559 Log.d(
560 Config.LOGTAG,
561 id.getAccount().getJid().asBareJid()
562 + ": unable to rollback local description after receiving content-reject",
563 cause);
564 webRTCWrapper.close();
565 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
566 return;
567 }
568 Log.d(
569 Config.LOGTAG,
570 id.getAccount().getJid().asBareJid()
571 + ": remote has rejected our content-add "
572 + summary);
573 }
574
575 private void receiveContentRemove(final JinglePacket jinglePacket) {
576 final RtpContentMap receivedContentRemove;
577 try {
578 receivedContentRemove = RtpContentMap.of(jinglePacket);
579 receivedContentRemove.requireContentDescriptions();
580 } catch (final RuntimeException e) {
581 Log.d(
582 Config.LOGTAG,
583 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
584 Throwables.getRootCause(e));
585 respondOk(jinglePacket);
586 this.webRTCWrapper.close();
587 sendSessionTerminate(Reason.of(e), e.getMessage());
588 return;
589 }
590 respondOk(jinglePacket);
591 receiveContentRemove(receivedContentRemove);
592 }
593
594 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
595 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
596 final Set<ContentAddition.Summary> contentAddSummary =
597 incomingContentAdd == null
598 ? Collections.emptySet()
599 : ContentAddition.summary(incomingContentAdd);
600 final Set<ContentAddition.Summary> removeSummary =
601 ContentAddition.summary(receivedContentRemove);
602 if (contentAddSummary.equals(removeSummary)) {
603 this.incomingContentAdd = null;
604 updateEndUserState();
605 } else {
606 webRTCWrapper.close();
607 sendSessionTerminate(
608 Reason.FAILED_APPLICATION,
609 String.format(
610 "%s only supports %s as a means to retract a not yet accepted %s",
611 BuildConfig.APP_NAME,
612 JinglePacket.Action.CONTENT_REMOVE,
613 JinglePacket.Action.CONTENT_ACCEPT));
614 }
615 }
616
617 public synchronized void retractContentAdd() {
618 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
619 if (outgoingContentAdd == null) {
620 throw new IllegalStateException("Not outgoing content add");
621 }
622 try {
623 webRTCWrapper.removeTrack(Media.VIDEO);
624 final RtpContentMap localContentMap = customRollback();
625 modifyLocalContentMap(localContentMap);
626 } catch (final Exception e) {
627 final Throwable cause = Throwables.getRootCause(e);
628 Log.d(
629 Config.LOGTAG,
630 id.getAccount().getJid().asBareJid()
631 + ": unable to rollback local description after trying to retract content-add",
632 cause);
633 webRTCWrapper.close();
634 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
635 return;
636 }
637 this.outgoingContentAdd = null;
638 final JinglePacket retract =
639 outgoingContentAdd
640 .toStub()
641 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
642 this.send(retract);
643 Log.d(
644 Config.LOGTAG,
645 id.getAccount().getJid()
646 + ": retract content-add "
647 + ContentAddition.summary(outgoingContentAdd));
648 }
649
650 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
651 final SessionDescription sdp = setLocalSessionDescription();
652 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
653 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
654 this.webRTCWrapper
655 .setRemoteDescription(
656 new org.webrtc.SessionDescription(
657 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
658 .get();
659 return localRtpContentMap;
660 }
661
662 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
663 final RtpContentMap currentRemote = getRemoteContentMap();
664 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
665 if (diff.isEmpty()) {
666 throw new IllegalStateException(
667 "Unexpected rollback condition. No difference between local and remote");
668 }
669 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
670 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
671 final RtpContentMap nextRemote =
672 currentRemote.addContent(
673 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
674 return SessionDescription.of(nextRemote, !isInitiator());
675 }
676 throw new IllegalStateException(
677 "Unexpected rollback condition. Senders were not uniformly none");
678 }
679
680 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
681 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
682 if (incomingContentAdd == null) {
683 throw new IllegalStateException("No incoming content add");
684 }
685
686 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
687 this.incomingContentAdd = null;
688 acceptContentAdd(contentAddition, incomingContentAdd);
689 } else {
690 throw new IllegalStateException("Accepted content add does not match pending content-add");
691 }
692 }
693
694 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
695 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
696 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
697 this.setRemoteContentMap(modifiedContentMap);
698
699 final SessionDescription offer;
700 try {
701 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
702 } catch (final IllegalArgumentException | NullPointerException e) {
703 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
704 webRTCWrapper.close();
705 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
706 return;
707 }
708 this.incomingContentAdd = null;
709 acceptContentAdd(contentAddition, offer);
710 }
711
712 private void acceptContentAdd(
713 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
714 final org.webrtc.SessionDescription sdp =
715 new org.webrtc.SessionDescription(
716 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
717 try {
718 this.webRTCWrapper.setRemoteDescription(sdp).get();
719
720 // TODO add tracks for 'media' where contentAddition.senders matches
721
722 // TODO if senders.sending(isInitiator())
723
724 this.webRTCWrapper.addTrack(Media.VIDEO);
725
726 // TODO add additional transceivers for recv only cases
727
728 final SessionDescription answer = setLocalSessionDescription();
729 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
730
731 final RtpContentMap contentAcceptMap =
732 rtpContentMap.toContentModification(
733 Collections2.transform(contentAddition, ca -> ca.name));
734 Log.d(
735 Config.LOGTAG,
736 id.getAccount().getJid().asBareJid()
737 + ": sending content-accept "
738 + ContentAddition.summary(contentAcceptMap));
739 modifyLocalContentMap(rtpContentMap);
740 sendContentAccept(contentAcceptMap);
741 } catch (final Exception e) {
742 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
743 webRTCWrapper.close();
744 sendSessionTerminate(Reason.FAILED_APPLICATION);
745 }
746 }
747
748 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
749 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
750 send(jinglePacket);
751 }
752
753 public synchronized void rejectContentAdd() {
754 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
755 if (incomingContentAdd == null) {
756 throw new IllegalStateException("No incoming content add");
757 }
758 this.incomingContentAdd = null;
759 updateEndUserState();
760 rejectContentAdd(incomingContentAdd);
761 }
762
763 private void rejectContentAdd(final RtpContentMap contentMap) {
764 final JinglePacket jinglePacket =
765 contentMap
766 .toStub()
767 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
768 Log.d(
769 Config.LOGTAG,
770 id.getAccount().getJid().asBareJid()
771 + ": rejecting content "
772 + ContentAddition.summary(contentMap));
773 send(jinglePacket);
774 }
775
776 private boolean checkForIceRestart(
777 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
778 final RtpContentMap existing = getRemoteContentMap();
779 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
780 final IceUdpTransportInfo.Credentials newCredentials;
781 try {
782 existingCredentials = existing.getCredentials();
783 newCredentials = rtpContentMap.getDistinctCredentials();
784 } catch (final IllegalStateException e) {
785 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
786 return false;
787 }
788 if (existingCredentials.contains(newCredentials)) {
789 return false;
790 }
791 // TODO an alternative approach is to check if we already got an iq result to our
792 // ICE-restart
793 // and if that's the case we are seeing an answer.
794 // This might be more spec compliant but also more error prone potentially
795 final boolean isOffer = rtpContentMap.emptyCandidates();
796 final RtpContentMap restartContentMap;
797 try {
798 if (isOffer) {
799 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
800 restartContentMap =
801 existing.modifiedCredentials(
802 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
803 } else {
804 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
805 Log.d(
806 Config.LOGTAG,
807 "received confirmation of ICE restart"
808 + newCredentials
809 + " peer_setup="
810 + setup);
811 // DTLS setup attribute needs to be rewritten to reflect current peer state
812 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
813 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
814 }
815 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
816 return isOffer;
817 } else {
818 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
819 respondWithTieBreak(jinglePacket);
820 return true;
821 }
822 } catch (final Exception exception) {
823 respondOk(jinglePacket);
824 final Throwable rootCause = Throwables.getRootCause(exception);
825 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
826 // If this happens a termination is already in progress
827 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
828 return true;
829 }
830 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
831 webRTCWrapper.close();
832 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
833 return true;
834 }
835 }
836
837 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
838 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
839 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
840 throw new IllegalStateException("Invalid peer setup");
841 }
842 return peerSetup;
843 }
844
845 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
846 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
847 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
848 }
849 this.peerDtlsSetup = setup;
850 }
851
852 private boolean applyIceRestart(
853 final JinglePacket jinglePacket,
854 final RtpContentMap restartContentMap,
855 final boolean isOffer)
856 throws ExecutionException, InterruptedException {
857 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
858 final org.webrtc.SessionDescription.Type type =
859 isOffer
860 ? org.webrtc.SessionDescription.Type.OFFER
861 : org.webrtc.SessionDescription.Type.ANSWER;
862 org.webrtc.SessionDescription sdp =
863 new org.webrtc.SessionDescription(type, sessionDescription.toString());
864 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
865 if (isInitiator()) {
866 // We ignore the offer and respond with tie-break. This will clause the responder
867 // not to apply the content map
868 return false;
869 }
870 }
871 webRTCWrapper.setRemoteDescription(sdp).get();
872 setRemoteContentMap(restartContentMap);
873 if (isOffer) {
874 webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
875 final SessionDescription localSessionDescription = setLocalSessionDescription();
876 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
877 // We need to respond OK before sending any candidates
878 respondOk(jinglePacket);
879 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
880 } else {
881 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
882 }
883 return true;
884 }
885
886 private void processCandidates(
887 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
888 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
889 processCandidate(content);
890 }
891 }
892
893 private void processCandidate(
894 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
895 final RtpContentMap rtpContentMap = getRemoteContentMap();
896 final List<String> indices = toIdentificationTags(rtpContentMap);
897 final String sdpMid = content.getKey(); // aka content name
898 final IceUdpTransportInfo transport = content.getValue().transport;
899 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
900
901 // TODO check that credentials remained the same
902
903 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
904 final String sdp;
905 try {
906 sdp = candidate.toSdpAttribute(credentials.ufrag);
907 } catch (final IllegalArgumentException e) {
908 Log.d(
909 Config.LOGTAG,
910 id.account.getJid().asBareJid()
911 + ": ignoring invalid ICE candidate "
912 + e.getMessage());
913 continue;
914 }
915 final int mLineIndex = indices.indexOf(sdpMid);
916 if (mLineIndex < 0) {
917 Log.w(
918 Config.LOGTAG,
919 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
920 }
921 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
922 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
923 this.webRTCWrapper.addIceCandidate(iceCandidate);
924 }
925 }
926
927 private RtpContentMap getRemoteContentMap() {
928 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
929 }
930
931 private RtpContentMap getLocalContentMap() {
932 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
933 }
934
935 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
936 final Group originalGroup = rtpContentMap.group;
937 final List<String> identificationTags =
938 originalGroup == null
939 ? rtpContentMap.getNames()
940 : originalGroup.getIdentificationTags();
941 if (identificationTags.size() == 0) {
942 Log.w(
943 Config.LOGTAG,
944 id.account.getJid().asBareJid()
945 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
946 }
947 return identificationTags;
948 }
949
950 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
951 final JinglePacket jinglePacket, final boolean expectVerification) {
952 final RtpContentMap receivedContentMap;
953 try {
954 receivedContentMap = RtpContentMap.of(jinglePacket);
955 } catch (final Exception e) {
956 return Futures.immediateFailedFuture(e);
957 }
958 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
959 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
960 id.account
961 .getAxolotlService()
962 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
963 return Futures.transform(
964 future,
965 omemoVerifiedPayload -> {
966 // TODO test if an exception here triggers a correct abort
967 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
968 Log.d(
969 Config.LOGTAG,
970 id.account.getJid().asBareJid()
971 + ": received verifiable DTLS fingerprint via "
972 + omemoVerification);
973 return omemoVerifiedPayload.getPayload();
974 },
975 MoreExecutors.directExecutor());
976 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
977 return Futures.immediateFailedFuture(
978 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
979 } else {
980 return Futures.immediateFuture(receivedContentMap);
981 }
982 }
983
984 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
985 if (isInitiator()) {
986 Log.d(
987 Config.LOGTAG,
988 String.format(
989 "%s: received session-initiate even though we were initiating",
990 id.account.getJid().asBareJid()));
991 if (isTerminated()) {
992 Log.d(
993 Config.LOGTAG,
994 String.format(
995 "%s: got a reason to terminate with out-of-order. but already in state %s",
996 id.account.getJid().asBareJid(), getState()));
997 respondWithOutOfOrder(jinglePacket);
998 } else {
999 terminateWithOutOfOrder(jinglePacket);
1000 }
1001 return;
1002 }
1003 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1004 Futures.addCallback(
1005 future,
1006 new FutureCallback<RtpContentMap>() {
1007 @Override
1008 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1009 receiveSessionInitiate(jinglePacket, rtpContentMap);
1010 }
1011
1012 @Override
1013 public void onFailure(@NonNull final Throwable throwable) {
1014 respondOk(jinglePacket);
1015 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1016 }
1017 },
1018 MoreExecutors.directExecutor());
1019 }
1020
1021 private void receiveSessionInitiate(
1022 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1023 try {
1024 contentMap.requireContentDescriptions();
1025 contentMap.requireDTLSFingerprint(true);
1026 } catch (final RuntimeException e) {
1027 Log.d(
1028 Config.LOGTAG,
1029 id.account.getJid().asBareJid() + ": improperly formatted contents",
1030 Throwables.getRootCause(e));
1031 respondOk(jinglePacket);
1032 sendSessionTerminate(Reason.of(e), e.getMessage());
1033 return;
1034 }
1035 Log.d(
1036 Config.LOGTAG,
1037 "processing session-init with " + contentMap.contents.size() + " contents");
1038 final State target;
1039 if (this.state == State.PROCEED) {
1040 Preconditions.checkState(
1041 proposedMedia != null && proposedMedia.size() > 0,
1042 "proposed media must be set when processing pre-approved session-initiate");
1043 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1044 sendSessionTerminate(
1045 Reason.SECURITY_ERROR,
1046 String.format(
1047 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1048 this.proposedMedia, contentMap.getMedia()));
1049 return;
1050 }
1051 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1052 } else {
1053 target = State.SESSION_INITIALIZED;
1054 }
1055 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1056 respondOk(jinglePacket);
1057 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1058 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1059 Log.d(
1060 Config.LOGTAG,
1061 id.account.getJid().asBareJid()
1062 + ": automatically accepting session-initiate");
1063 sendSessionAccept();
1064 } else {
1065 Log.d(
1066 Config.LOGTAG,
1067 id.account.getJid().asBareJid()
1068 + ": received not pre-approved session-initiate. start ringing");
1069 startRinging();
1070 }
1071 } else {
1072 Log.d(
1073 Config.LOGTAG,
1074 String.format(
1075 "%s: received session-initiate while in state %s",
1076 id.account.getJid().asBareJid(), state));
1077 terminateWithOutOfOrder(jinglePacket);
1078 }
1079 }
1080
1081 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1082 if (!isInitiator()) {
1083 Log.d(
1084 Config.LOGTAG,
1085 String.format(
1086 "%s: received session-accept even though we were responding",
1087 id.account.getJid().asBareJid()));
1088 terminateWithOutOfOrder(jinglePacket);
1089 return;
1090 }
1091 final ListenableFuture<RtpContentMap> future =
1092 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1093 Futures.addCallback(
1094 future,
1095 new FutureCallback<RtpContentMap>() {
1096 @Override
1097 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1098 receiveSessionAccept(jinglePacket, rtpContentMap);
1099 }
1100
1101 @Override
1102 public void onFailure(@NonNull final Throwable throwable) {
1103 respondOk(jinglePacket);
1104 Log.d(
1105 Config.LOGTAG,
1106 id.account.getJid().asBareJid()
1107 + ": improperly formatted contents in session-accept",
1108 throwable);
1109 webRTCWrapper.close();
1110 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1111 }
1112 },
1113 MoreExecutors.directExecutor());
1114 }
1115
1116 private void receiveSessionAccept(
1117 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1118 try {
1119 contentMap.requireContentDescriptions();
1120 contentMap.requireDTLSFingerprint();
1121 } catch (final RuntimeException e) {
1122 respondOk(jinglePacket);
1123 Log.d(
1124 Config.LOGTAG,
1125 id.account.getJid().asBareJid()
1126 + ": improperly formatted contents in session-accept",
1127 e);
1128 webRTCWrapper.close();
1129 sendSessionTerminate(Reason.of(e), e.getMessage());
1130 return;
1131 }
1132 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1133 if (!initiatorMedia.equals(contentMap.getMedia())) {
1134 sendSessionTerminate(
1135 Reason.SECURITY_ERROR,
1136 String.format(
1137 "Your session-included included media %s but our session-initiate was %s",
1138 this.proposedMedia, contentMap.getMedia()));
1139 return;
1140 }
1141 Log.d(
1142 Config.LOGTAG,
1143 "processing session-accept with " + contentMap.contents.size() + " contents");
1144 if (transition(State.SESSION_ACCEPTED)) {
1145 respondOk(jinglePacket);
1146 receiveSessionAccept(contentMap);
1147 } else {
1148 Log.d(
1149 Config.LOGTAG,
1150 String.format(
1151 "%s: received session-accept while in state %s",
1152 id.account.getJid().asBareJid(), state));
1153 respondOk(jinglePacket);
1154 }
1155 }
1156
1157 private void receiveSessionAccept(final RtpContentMap contentMap) {
1158 this.responderRtpContentMap = contentMap;
1159 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1160 final SessionDescription sessionDescription;
1161 try {
1162 sessionDescription = SessionDescription.of(contentMap, false);
1163 } catch (final IllegalArgumentException | NullPointerException e) {
1164 Log.d(
1165 Config.LOGTAG,
1166 id.account.getJid().asBareJid()
1167 + ": unable convert offer from session-accept to SDP",
1168 e);
1169 webRTCWrapper.close();
1170 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1171 return;
1172 }
1173 final org.webrtc.SessionDescription answer =
1174 new org.webrtc.SessionDescription(
1175 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1176 try {
1177 this.webRTCWrapper.setRemoteDescription(answer).get();
1178 } catch (final Exception e) {
1179 Log.d(
1180 Config.LOGTAG,
1181 id.account.getJid().asBareJid()
1182 + ": unable to set remote description after receiving session-accept",
1183 Throwables.getRootCause(e));
1184 webRTCWrapper.close();
1185 sendSessionTerminate(
1186 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1187 return;
1188 }
1189 processCandidates(contentMap.contents.entrySet());
1190 }
1191
1192 private void sendSessionAccept() {
1193 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1194 if (rtpContentMap == null) {
1195 throw new IllegalStateException("initiator RTP Content Map has not been set");
1196 }
1197 final SessionDescription offer;
1198 try {
1199 offer = SessionDescription.of(rtpContentMap, true);
1200 } catch (final IllegalArgumentException | NullPointerException e) {
1201 Log.d(
1202 Config.LOGTAG,
1203 id.account.getJid().asBareJid()
1204 + ": unable convert offer from session-initiate to SDP",
1205 e);
1206 webRTCWrapper.close();
1207 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1208 return;
1209 }
1210 sendSessionAccept(rtpContentMap.getMedia(), offer);
1211 }
1212
1213 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1214 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1215 }
1216
1217 private synchronized void sendSessionAccept(
1218 final Set<Media> media,
1219 final SessionDescription offer,
1220 final List<PeerConnection.IceServer> iceServers) {
1221 if (isTerminated()) {
1222 Log.w(
1223 Config.LOGTAG,
1224 id.account.getJid().asBareJid()
1225 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1226 return;
1227 }
1228 try {
1229 setupWebRTC(media, iceServers);
1230 } catch (final WebRTCWrapper.InitializationException e) {
1231 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1232 webRTCWrapper.close();
1233 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1234 return;
1235 }
1236 final org.webrtc.SessionDescription sdp =
1237 new org.webrtc.SessionDescription(
1238 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1239 try {
1240 this.webRTCWrapper.setRemoteDescription(sdp).get();
1241 addIceCandidatesFromBlackLog();
1242 org.webrtc.SessionDescription webRTCSessionDescription =
1243 this.webRTCWrapper.setLocalDescription().get();
1244 prepareSessionAccept(webRTCSessionDescription);
1245 } catch (final Exception e) {
1246 failureToAcceptSession(e);
1247 }
1248 }
1249
1250 private void failureToAcceptSession(final Throwable throwable) {
1251 if (isTerminated()) {
1252 return;
1253 }
1254 final Throwable rootCause = Throwables.getRootCause(throwable);
1255 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1256 webRTCWrapper.close();
1257 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1258 }
1259
1260 private void addIceCandidatesFromBlackLog() {
1261 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1262 while ((foo = this.pendingIceCandidates.poll()) != null) {
1263 processCandidate(foo);
1264 Log.d(
1265 Config.LOGTAG,
1266 id.account.getJid().asBareJid() + ": added candidate from back log");
1267 }
1268 }
1269
1270 private void prepareSessionAccept(
1271 final org.webrtc.SessionDescription webRTCSessionDescription) {
1272 final SessionDescription sessionDescription =
1273 SessionDescription.parse(webRTCSessionDescription.description);
1274 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1275 this.responderRtpContentMap = respondingRtpContentMap;
1276 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1277 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1278 prepareOutgoingContentMap(respondingRtpContentMap);
1279 Futures.addCallback(
1280 outgoingContentMapFuture,
1281 new FutureCallback<RtpContentMap>() {
1282 @Override
1283 public void onSuccess(final RtpContentMap outgoingContentMap) {
1284 sendSessionAccept(outgoingContentMap);
1285 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1286 }
1287
1288 @Override
1289 public void onFailure(@NonNull Throwable throwable) {
1290 failureToAcceptSession(throwable);
1291 }
1292 },
1293 MoreExecutors.directExecutor());
1294 }
1295
1296 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1297 if (isTerminated()) {
1298 Log.w(
1299 Config.LOGTAG,
1300 id.account.getJid().asBareJid()
1301 + ": preparing session accept was too slow. already terminated. nothing to do.");
1302 return;
1303 }
1304 transitionOrThrow(State.SESSION_ACCEPTED);
1305 final JinglePacket sessionAccept =
1306 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1307 send(sessionAccept);
1308 }
1309
1310 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1311 final RtpContentMap rtpContentMap) {
1312 if (this.omemoVerification.hasDeviceId()) {
1313 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1314 verifiedPayloadFuture =
1315 id.account
1316 .getAxolotlService()
1317 .encrypt(
1318 rtpContentMap,
1319 id.with,
1320 omemoVerification.getDeviceId());
1321 return Futures.transform(
1322 verifiedPayloadFuture,
1323 verifiedPayload -> {
1324 omemoVerification.setOrEnsureEqual(verifiedPayload);
1325 return verifiedPayload.getPayload();
1326 },
1327 MoreExecutors.directExecutor());
1328 } else {
1329 return Futures.immediateFuture(rtpContentMap);
1330 }
1331 }
1332
1333 synchronized void deliveryMessage(
1334 final Jid from,
1335 final Element message,
1336 final String serverMessageId,
1337 final long timestamp) {
1338 Log.d(
1339 Config.LOGTAG,
1340 id.account.getJid().asBareJid()
1341 + ": delivered message to JingleRtpConnection "
1342 + message);
1343 switch (message.getName()) {
1344 case "propose":
1345 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1346 break;
1347 case "proceed":
1348 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1349 break;
1350 case "retract":
1351 receiveRetract(from, serverMessageId, timestamp);
1352 break;
1353 case "reject":
1354 receiveReject(from, serverMessageId, timestamp);
1355 break;
1356 case "accept":
1357 receiveAccept(from, serverMessageId, timestamp);
1358 break;
1359 default:
1360 break;
1361 }
1362 }
1363
1364 void deliverFailedProceed(final String message) {
1365 Log.d(
1366 Config.LOGTAG,
1367 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1368 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1369 webRTCWrapper.close();
1370 Log.d(
1371 Config.LOGTAG,
1372 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1373 this.finish();
1374 }
1375 }
1376
1377 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1378 final boolean originatedFromMyself =
1379 from.asBareJid().equals(id.account.getJid().asBareJid());
1380 if (originatedFromMyself) {
1381 if (transition(State.ACCEPTED)) {
1382 acceptedOnOtherDevice(serverMsgId, timestamp);
1383 } else {
1384 Log.d(
1385 Config.LOGTAG,
1386 id.account.getJid().asBareJid()
1387 + ": unable to transition to accept because already in state="
1388 + this.state);
1389 }
1390 } else {
1391 Log.d(
1392 Config.LOGTAG,
1393 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1394 }
1395 }
1396
1397 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1398 if (serverMsgId != null) {
1399 this.message.setServerMsgId(serverMsgId);
1400 }
1401 this.message.setTime(timestamp);
1402 this.message.setCarbon(true); // indicate that call was accepted on other device
1403 this.writeLogMessageSuccess(0);
1404 this.xmppConnectionService
1405 .getNotificationService()
1406 .cancelIncomingCallNotification();
1407 this.finish();
1408 }
1409
1410 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1411 final boolean originatedFromMyself =
1412 from.asBareJid().equals(id.account.getJid().asBareJid());
1413 // reject from another one of my clients
1414 if (originatedFromMyself) {
1415 receiveRejectFromMyself(serverMsgId, timestamp);
1416 } else if (isInitiator()) {
1417 if (from.equals(id.with)) {
1418 receiveRejectFromResponder();
1419 } else {
1420 Log.d(
1421 Config.LOGTAG,
1422 id.account.getJid()
1423 + ": ignoring reject from "
1424 + from
1425 + " for session with "
1426 + id.with);
1427 }
1428 } else {
1429 Log.d(
1430 Config.LOGTAG,
1431 id.account.getJid()
1432 + ": ignoring reject from "
1433 + from
1434 + " for session with "
1435 + id.with);
1436 }
1437 }
1438
1439 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1440 if (transition(State.REJECTED)) {
1441 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1442 this.finish();
1443 if (serverMsgId != null) {
1444 this.message.setServerMsgId(serverMsgId);
1445 }
1446 this.message.setTime(timestamp);
1447 this.message.setCarbon(true); // indicate that call was rejected on other device
1448 writeLogMessageMissed();
1449 } else {
1450 Log.d(
1451 Config.LOGTAG,
1452 "not able to transition into REJECTED because already in " + this.state);
1453 }
1454 }
1455
1456 private void receiveRejectFromResponder() {
1457 if (isInState(State.PROCEED)) {
1458 Log.d(
1459 Config.LOGTAG,
1460 id.account.getJid()
1461 + ": received reject while still in proceed. callee reconsidered");
1462 closeTransitionLogFinish(State.REJECTED_RACED);
1463 return;
1464 }
1465 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1466 Log.d(
1467 Config.LOGTAG,
1468 id.account.getJid()
1469 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1470 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1471 return;
1472 }
1473 Log.d(
1474 Config.LOGTAG,
1475 id.account.getJid()
1476 + ": ignoring reject from responder because already in state "
1477 + this.state);
1478 }
1479
1480 private void receivePropose(
1481 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1482 final boolean originatedFromMyself =
1483 from.asBareJid().equals(id.account.getJid().asBareJid());
1484 if (originatedFromMyself) {
1485 Log.d(
1486 Config.LOGTAG,
1487 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1488 } else if (transition(
1489 State.PROPOSED,
1490 () -> {
1491 final Collection<RtpDescription> descriptions =
1492 Collections2.transform(
1493 Collections2.filter(
1494 propose.getDescriptions(),
1495 d -> d instanceof RtpDescription),
1496 input -> (RtpDescription) input);
1497 final Collection<Media> media =
1498 Collections2.transform(descriptions, RtpDescription::getMedia);
1499 Preconditions.checkState(
1500 !media.contains(Media.UNKNOWN),
1501 "RTP descriptions contain unknown media");
1502 Log.d(
1503 Config.LOGTAG,
1504 id.account.getJid().asBareJid()
1505 + ": received session proposal from "
1506 + from
1507 + " for "
1508 + media);
1509 this.proposedMedia = Sets.newHashSet(media);
1510 })) {
1511 if (serverMsgId != null) {
1512 this.message.setServerMsgId(serverMsgId);
1513 }
1514 this.message.setTime(timestamp);
1515 startRinging();
1516 } else {
1517 Log.d(
1518 Config.LOGTAG,
1519 id.account.getJid()
1520 + ": ignoring session proposal because already in "
1521 + state);
1522 }
1523 }
1524
1525 private void startRinging() {
1526 Log.d(
1527 Config.LOGTAG,
1528 id.account.getJid().asBareJid()
1529 + ": received call from "
1530 + id.with
1531 + ". start ringing");
1532 ringingTimeoutFuture =
1533 jingleConnectionManager.schedule(
1534 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1535 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1536 }
1537
1538 private synchronized void ringingTimeout() {
1539 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1540 switch (this.state) {
1541 case PROPOSED:
1542 message.markUnread();
1543 rejectCallFromProposed();
1544 break;
1545 case SESSION_INITIALIZED:
1546 message.markUnread();
1547 rejectCallFromSessionInitiate();
1548 break;
1549 }
1550 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1551 }
1552
1553 private void cancelRingingTimeout() {
1554 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1555 if (future != null && !future.isCancelled()) {
1556 future.cancel(false);
1557 }
1558 }
1559
1560 private void receiveProceed(
1561 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1562 final Set<Media> media =
1563 Preconditions.checkNotNull(
1564 this.proposedMedia, "Proposed media has to be set before handling proceed");
1565 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1566 if (from.equals(id.with)) {
1567 if (isInitiator()) {
1568 if (transition(State.PROCEED)) {
1569 if (serverMsgId != null) {
1570 this.message.setServerMsgId(serverMsgId);
1571 }
1572 this.message.setTime(timestamp);
1573 final Integer remoteDeviceId = proceed.getDeviceId();
1574 if (isOmemoEnabled()) {
1575 this.omemoVerification.setDeviceId(remoteDeviceId);
1576 } else {
1577 if (remoteDeviceId != null) {
1578 Log.d(
1579 Config.LOGTAG,
1580 id.account.getJid().asBareJid()
1581 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1582 }
1583 this.omemoVerification.setDeviceId(null);
1584 }
1585 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1586 } else {
1587 Log.d(
1588 Config.LOGTAG,
1589 String.format(
1590 "%s: ignoring proceed because already in %s",
1591 id.account.getJid().asBareJid(), this.state));
1592 }
1593 } else {
1594 Log.d(
1595 Config.LOGTAG,
1596 String.format(
1597 "%s: ignoring proceed because we were not initializing",
1598 id.account.getJid().asBareJid()));
1599 }
1600 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1601 if (transition(State.ACCEPTED)) {
1602 Log.d(
1603 Config.LOGTAG,
1604 id.account.getJid().asBareJid()
1605 + ": moved session with "
1606 + id.with
1607 + " into state accepted after received carbon copied proceed");
1608 acceptedOnOtherDevice(serverMsgId, timestamp);
1609 }
1610 } else {
1611 Log.d(
1612 Config.LOGTAG,
1613 String.format(
1614 "%s: ignoring proceed from %s. was expected from %s",
1615 id.account.getJid().asBareJid(), from, id.with));
1616 }
1617 }
1618
1619 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1620 if (from.equals(id.with)) {
1621 final State target =
1622 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1623 if (transition(target)) {
1624 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1625 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1626 Log.d(
1627 Config.LOGTAG,
1628 id.account.getJid().asBareJid()
1629 + ": session with "
1630 + id.with
1631 + " has been retracted (serverMsgId="
1632 + serverMsgId
1633 + ")");
1634 if (serverMsgId != null) {
1635 this.message.setServerMsgId(serverMsgId);
1636 }
1637 this.message.setTime(timestamp);
1638 if (target == State.RETRACTED) {
1639 this.message.markUnread();
1640 }
1641 writeLogMessageMissed();
1642 finish();
1643 } else {
1644 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1645 }
1646 } else {
1647 // TODO parse retract from self
1648 Log.d(
1649 Config.LOGTAG,
1650 id.account.getJid().asBareJid()
1651 + ": received retract from "
1652 + from
1653 + ". expected retract from"
1654 + id.with
1655 + ". ignoring");
1656 }
1657 }
1658
1659 public void sendSessionInitiate() {
1660 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1661 }
1662
1663 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1664 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1665 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1666 }
1667
1668 private synchronized void sendSessionInitiate(
1669 final Set<Media> media,
1670 final State targetState,
1671 final List<PeerConnection.IceServer> iceServers) {
1672 if (isTerminated()) {
1673 Log.w(
1674 Config.LOGTAG,
1675 id.account.getJid().asBareJid()
1676 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1677 return;
1678 }
1679 try {
1680 setupWebRTC(media, iceServers);
1681 } catch (final WebRTCWrapper.InitializationException e) {
1682 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1683 webRTCWrapper.close();
1684 sendRetract(Reason.ofThrowable(e));
1685 return;
1686 }
1687 try {
1688 org.webrtc.SessionDescription webRTCSessionDescription =
1689 this.webRTCWrapper.setLocalDescription().get();
1690 prepareSessionInitiate(webRTCSessionDescription, targetState);
1691 } catch (final Exception e) {
1692 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1693 // exceptions
1694 failureToInitiateSession(e, targetState);
1695 }
1696 }
1697
1698 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1699 if (isTerminated()) {
1700 return;
1701 }
1702 Log.d(
1703 Config.LOGTAG,
1704 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1705 Throwables.getRootCause(throwable));
1706 webRTCWrapper.close();
1707 final Reason reason = Reason.ofThrowable(throwable);
1708 if (isInState(targetState)) {
1709 sendSessionTerminate(reason, throwable.getMessage());
1710 } else {
1711 sendRetract(reason);
1712 }
1713 }
1714
1715 private void sendRetract(final Reason reason) {
1716 // TODO embed reason into retract
1717 sendJingleMessage("retract", id.with.asBareJid());
1718 transitionOrThrow(reasonToState(reason));
1719 this.finish();
1720 }
1721
1722 private void prepareSessionInitiate(
1723 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1724 final SessionDescription sessionDescription =
1725 SessionDescription.parse(webRTCSessionDescription.description);
1726 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1727 this.initiatorRtpContentMap = rtpContentMap;
1728 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1729 encryptSessionInitiate(rtpContentMap);
1730 Futures.addCallback(
1731 outgoingContentMapFuture,
1732 new FutureCallback<RtpContentMap>() {
1733 @Override
1734 public void onSuccess(final RtpContentMap outgoingContentMap) {
1735 sendSessionInitiate(outgoingContentMap, targetState);
1736 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1737 }
1738
1739 @Override
1740 public void onFailure(@NonNull final Throwable throwable) {
1741 failureToInitiateSession(throwable, targetState);
1742 }
1743 },
1744 MoreExecutors.directExecutor());
1745 }
1746
1747 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1748 if (isTerminated()) {
1749 Log.w(
1750 Config.LOGTAG,
1751 id.account.getJid().asBareJid()
1752 + ": preparing session was too slow. already terminated. nothing to do.");
1753 return;
1754 }
1755 this.transitionOrThrow(targetState);
1756 final JinglePacket sessionInitiate =
1757 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1758 send(sessionInitiate);
1759 }
1760
1761 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1762 final RtpContentMap rtpContentMap) {
1763 if (this.omemoVerification.hasDeviceId()) {
1764 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1765 verifiedPayloadFuture =
1766 id.account
1767 .getAxolotlService()
1768 .encrypt(
1769 rtpContentMap,
1770 id.with,
1771 omemoVerification.getDeviceId());
1772 final ListenableFuture<RtpContentMap> future =
1773 Futures.transform(
1774 verifiedPayloadFuture,
1775 verifiedPayload -> {
1776 omemoVerification.setSessionFingerprint(
1777 verifiedPayload.getFingerprint());
1778 return verifiedPayload.getPayload();
1779 },
1780 MoreExecutors.directExecutor());
1781 if (Config.REQUIRE_RTP_VERIFICATION) {
1782 return future;
1783 }
1784 return Futures.catching(
1785 future,
1786 CryptoFailedException.class,
1787 e -> {
1788 Log.w(
1789 Config.LOGTAG,
1790 id.account.getJid().asBareJid()
1791 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1792 e);
1793 return rtpContentMap;
1794 },
1795 MoreExecutors.directExecutor());
1796 } else {
1797 return Futures.immediateFuture(rtpContentMap);
1798 }
1799 }
1800
1801 private void sendSessionTerminate(final Reason reason) {
1802 sendSessionTerminate(reason, null);
1803 }
1804
1805 private void sendSessionTerminate(final Reason reason, final String text) {
1806 final State previous = this.state;
1807 final State target = reasonToState(reason);
1808 transitionOrThrow(target);
1809 if (previous != State.NULL) {
1810 writeLogMessage(target);
1811 }
1812 final JinglePacket jinglePacket =
1813 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1814 jinglePacket.setReason(reason, text);
1815 Log.d(Config.LOGTAG, jinglePacket.toString());
1816 send(jinglePacket);
1817 finish();
1818 }
1819
1820 private void sendTransportInfo(
1821 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1822 final RtpContentMap transportInfo;
1823 try {
1824 final RtpContentMap rtpContentMap =
1825 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1826 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1827 } catch (final Exception e) {
1828 Log.d(
1829 Config.LOGTAG,
1830 id.account.getJid().asBareJid()
1831 + ": unable to prepare transport-info from candidate for content="
1832 + contentName);
1833 return;
1834 }
1835 final JinglePacket jinglePacket =
1836 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1837 send(jinglePacket);
1838 }
1839
1840 private void send(final JinglePacket jinglePacket) {
1841 jinglePacket.setTo(id.with);
1842 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1843 }
1844
1845 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1846 if (response.getType() == IqPacket.TYPE.ERROR) {
1847 handleIqErrorResponse(response);
1848 return;
1849 }
1850 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1851 handleIqTimeoutResponse(response);
1852 }
1853 }
1854
1855 private void handleIqErrorResponse(final IqPacket response) {
1856 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1857 final String errorCondition = response.getErrorCondition();
1858 Log.d(
1859 Config.LOGTAG,
1860 id.account.getJid().asBareJid()
1861 + ": received IQ-error from "
1862 + response.getFrom()
1863 + " in RTP session. "
1864 + errorCondition);
1865 if (isTerminated()) {
1866 Log.i(
1867 Config.LOGTAG,
1868 id.account.getJid().asBareJid()
1869 + ": ignoring error because session was already terminated");
1870 return;
1871 }
1872 this.webRTCWrapper.close();
1873 final State target;
1874 if (Arrays.asList(
1875 "service-unavailable",
1876 "recipient-unavailable",
1877 "remote-server-not-found",
1878 "remote-server-timeout")
1879 .contains(errorCondition)) {
1880 target = State.TERMINATED_CONNECTIVITY_ERROR;
1881 } else {
1882 target = State.TERMINATED_APPLICATION_FAILURE;
1883 }
1884 transitionOrThrow(target);
1885 this.finish();
1886 }
1887
1888 private void handleIqTimeoutResponse(final IqPacket response) {
1889 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1890 Log.d(
1891 Config.LOGTAG,
1892 id.account.getJid().asBareJid()
1893 + ": received IQ timeout in RTP session with "
1894 + id.with
1895 + ". terminating with connectivity error");
1896 if (isTerminated()) {
1897 Log.i(
1898 Config.LOGTAG,
1899 id.account.getJid().asBareJid()
1900 + ": ignoring error because session was already terminated");
1901 return;
1902 }
1903 this.webRTCWrapper.close();
1904 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
1905 this.finish();
1906 }
1907
1908 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1909 Log.d(
1910 Config.LOGTAG,
1911 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1912 this.webRTCWrapper.close();
1913 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1914 respondWithOutOfOrder(jinglePacket);
1915 this.finish();
1916 }
1917
1918 private void respondWithTieBreak(final JinglePacket jinglePacket) {
1919 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
1920 }
1921
1922 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1923 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
1924 }
1925
1926 void respondWithJingleError(
1927 final IqPacket original,
1928 String jingleCondition,
1929 String condition,
1930 String conditionType) {
1931 jingleConnectionManager.respondWithJingleError(
1932 id.account, original, jingleCondition, condition, conditionType);
1933 }
1934
1935 private void respondOk(final JinglePacket jinglePacket) {
1936 xmppConnectionService.sendIqPacket(
1937 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1938 }
1939
1940 public RtpEndUserState getEndUserState() {
1941 switch (this.state) {
1942 case NULL:
1943 case PROPOSED:
1944 case SESSION_INITIALIZED:
1945 if (isInitiator()) {
1946 return RtpEndUserState.RINGING;
1947 } else {
1948 return RtpEndUserState.INCOMING_CALL;
1949 }
1950 case PROCEED:
1951 if (isInitiator()) {
1952 return RtpEndUserState.RINGING;
1953 } else {
1954 return RtpEndUserState.ACCEPTING_CALL;
1955 }
1956 case SESSION_INITIALIZED_PRE_APPROVED:
1957 if (isInitiator()) {
1958 return RtpEndUserState.RINGING;
1959 } else {
1960 return RtpEndUserState.CONNECTING;
1961 }
1962 case SESSION_ACCEPTED:
1963 final ContentAddition ca = getPendingContentAddition();
1964 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
1965 return RtpEndUserState.INCOMING_CONTENT_ADD;
1966 }
1967 return getPeerConnectionStateAsEndUserState();
1968 case REJECTED:
1969 case REJECTED_RACED:
1970 case TERMINATED_DECLINED_OR_BUSY:
1971 if (isInitiator()) {
1972 return RtpEndUserState.DECLINED_OR_BUSY;
1973 } else {
1974 return RtpEndUserState.ENDED;
1975 }
1976 case TERMINATED_SUCCESS:
1977 case ACCEPTED:
1978 case RETRACTED:
1979 case TERMINATED_CANCEL_OR_TIMEOUT:
1980 return RtpEndUserState.ENDED;
1981 case RETRACTED_RACED:
1982 if (isInitiator()) {
1983 return RtpEndUserState.ENDED;
1984 } else {
1985 return RtpEndUserState.RETRACTED;
1986 }
1987 case TERMINATED_CONNECTIVITY_ERROR:
1988 return zeroDuration()
1989 ? RtpEndUserState.CONNECTIVITY_ERROR
1990 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1991 case TERMINATED_APPLICATION_FAILURE:
1992 return RtpEndUserState.APPLICATION_ERROR;
1993 case TERMINATED_SECURITY_ERROR:
1994 return RtpEndUserState.SECURITY_ERROR;
1995 }
1996 throw new IllegalStateException(
1997 String.format("%s has no equivalent EndUserState", this.state));
1998 }
1999
2000 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2001 final PeerConnection.PeerConnectionState state;
2002 try {
2003 state = webRTCWrapper.getState();
2004 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2005 // We usually close the WebRTCWrapper *before* transitioning so we might still
2006 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2007 return RtpEndUserState.ENDING_CALL;
2008 }
2009 switch (state) {
2010 case CONNECTED:
2011 return RtpEndUserState.CONNECTED;
2012 case NEW:
2013 case CONNECTING:
2014 return RtpEndUserState.CONNECTING;
2015 case CLOSED:
2016 return RtpEndUserState.ENDING_CALL;
2017 default:
2018 return zeroDuration()
2019 ? RtpEndUserState.CONNECTIVITY_ERROR
2020 : RtpEndUserState.RECONNECTING;
2021 }
2022 }
2023
2024 public ContentAddition getPendingContentAddition() {
2025 final RtpContentMap in = this.incomingContentAdd;
2026 final RtpContentMap out = this.outgoingContentAdd;
2027 if (out != null) {
2028 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2029 } else if (in != null) {
2030 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2031 } else {
2032 return null;
2033 }
2034 }
2035
2036 public Set<Media> getMedia() {
2037 final State current = getState();
2038 if (current == State.NULL) {
2039 if (isInitiator()) {
2040 return Preconditions.checkNotNull(
2041 this.proposedMedia, "RTP connection has not been initialized properly");
2042 }
2043 throw new IllegalStateException("RTP connection has not been initialized yet");
2044 }
2045 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2046 return Preconditions.checkNotNull(
2047 this.proposedMedia, "RTP connection has not been initialized properly");
2048 }
2049 final RtpContentMap localContentMap = getLocalContentMap();
2050 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2051 if (localContentMap != null) {
2052 return localContentMap.getMedia();
2053 } else if (initiatorContentMap != null) {
2054 return initiatorContentMap.getMedia();
2055 } else if (isTerminated()) {
2056 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2057 } else {
2058 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2059 }
2060 }
2061
2062 public boolean isVerified() {
2063 final String fingerprint = this.omemoVerification.getFingerprint();
2064 if (fingerprint == null) {
2065 return false;
2066 }
2067 final FingerprintStatus status =
2068 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2069 return status != null && status.isVerified();
2070 }
2071
2072 public boolean addMedia(final Media media) {
2073 final Set<Media> currentMedia = getMedia();
2074 if (currentMedia.contains(media)) {
2075 throw new IllegalStateException(String.format("%s has already been proposed", media));
2076 }
2077 // TODO add state protection - can only add while ACCEPTED or so
2078 Log.d(Config.LOGTAG,"adding media: "+media);
2079 return webRTCWrapper.addTrack(media);
2080 }
2081
2082 public synchronized void acceptCall() {
2083 switch (this.state) {
2084 case PROPOSED:
2085 cancelRingingTimeout();
2086 acceptCallFromProposed();
2087 break;
2088 case SESSION_INITIALIZED:
2089 cancelRingingTimeout();
2090 acceptCallFromSessionInitialized();
2091 break;
2092 case ACCEPTED:
2093 Log.w(
2094 Config.LOGTAG,
2095 id.account.getJid().asBareJid()
2096 + ": the call has already been accepted with another client. UI was just lagging behind");
2097 break;
2098 case PROCEED:
2099 case SESSION_ACCEPTED:
2100 Log.w(
2101 Config.LOGTAG,
2102 id.account.getJid().asBareJid()
2103 + ": the call has already been accepted. user probably double tapped the UI");
2104 break;
2105 default:
2106 throw new IllegalStateException("Can not accept call from " + this.state);
2107 }
2108 }
2109
2110 public void notifyPhoneCall() {
2111 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2112 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2113 rejectCall();
2114 } else {
2115 endCall();
2116 }
2117 }
2118
2119 public synchronized void rejectCall() {
2120 if (isTerminated()) {
2121 Log.w(
2122 Config.LOGTAG,
2123 id.account.getJid().asBareJid()
2124 + ": received rejectCall() when session has already been terminated. nothing to do");
2125 return;
2126 }
2127 switch (this.state) {
2128 case PROPOSED:
2129 rejectCallFromProposed();
2130 break;
2131 case SESSION_INITIALIZED:
2132 rejectCallFromSessionInitiate();
2133 break;
2134 default:
2135 throw new IllegalStateException("Can not reject call from " + this.state);
2136 }
2137 }
2138
2139 public synchronized void endCall() {
2140 if (isTerminated()) {
2141 Log.w(
2142 Config.LOGTAG,
2143 id.account.getJid().asBareJid()
2144 + ": received endCall() when session has already been terminated. nothing to do");
2145 return;
2146 }
2147 if (isInState(State.PROPOSED) && !isInitiator()) {
2148 rejectCallFromProposed();
2149 return;
2150 }
2151 if (isInState(State.PROCEED)) {
2152 if (isInitiator()) {
2153 retractFromProceed();
2154 } else {
2155 rejectCallFromProceed();
2156 }
2157 return;
2158 }
2159 if (isInitiator()
2160 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2161 this.webRTCWrapper.close();
2162 sendSessionTerminate(Reason.CANCEL);
2163 return;
2164 }
2165 if (isInState(State.SESSION_INITIALIZED)) {
2166 rejectCallFromSessionInitiate();
2167 return;
2168 }
2169 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2170 this.webRTCWrapper.close();
2171 sendSessionTerminate(Reason.SUCCESS);
2172 return;
2173 }
2174 if (isInState(
2175 State.TERMINATED_APPLICATION_FAILURE,
2176 State.TERMINATED_CONNECTIVITY_ERROR,
2177 State.TERMINATED_DECLINED_OR_BUSY)) {
2178 Log.d(
2179 Config.LOGTAG,
2180 "ignoring request to end call because already in state " + this.state);
2181 return;
2182 }
2183 throw new IllegalStateException(
2184 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2185 }
2186
2187 private void retractFromProceed() {
2188 Log.d(Config.LOGTAG, "retract from proceed");
2189 this.sendJingleMessage("retract");
2190 closeTransitionLogFinish(State.RETRACTED_RACED);
2191 }
2192
2193 private void closeTransitionLogFinish(final State state) {
2194 this.webRTCWrapper.close();
2195 transitionOrThrow(state);
2196 writeLogMessage(state);
2197 finish();
2198 }
2199
2200 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2201 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2202 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2203 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2204 }
2205
2206 private void acceptCallFromProposed() {
2207 transitionOrThrow(State.PROCEED);
2208 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2209 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2210 this.sendJingleMessage("proceed");
2211 }
2212
2213 private void rejectCallFromProposed() {
2214 transitionOrThrow(State.REJECTED);
2215 writeLogMessageMissed();
2216 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2217 this.sendJingleMessage("reject");
2218 finish();
2219 }
2220
2221 private void rejectCallFromProceed() {
2222 this.sendJingleMessage("reject");
2223 closeTransitionLogFinish(State.REJECTED_RACED);
2224 }
2225
2226 private void rejectCallFromSessionInitiate() {
2227 webRTCWrapper.close();
2228 sendSessionTerminate(Reason.DECLINE);
2229 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2230 }
2231
2232 private void sendJingleMessage(final String action) {
2233 sendJingleMessage(action, id.with);
2234 }
2235
2236 private void sendJingleMessage(final String action, final Jid to) {
2237 final MessagePacket messagePacket = new MessagePacket();
2238 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2239 messagePacket.setTo(to);
2240 final Element intent =
2241 messagePacket
2242 .addChild(action, Namespace.JINGLE_MESSAGE)
2243 .setAttribute("id", id.sessionId);
2244 if ("proceed".equals(action)) {
2245 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2246 if (isOmemoEnabled()) {
2247 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2248 final Element device =
2249 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2250 device.setAttribute("id", deviceId);
2251 }
2252 }
2253 messagePacket.addChild("store", "urn:xmpp:hints");
2254 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2255 }
2256
2257 private boolean isOmemoEnabled() {
2258 final Conversational conversational = message.getConversation();
2259 if (conversational instanceof Conversation) {
2260 return ((Conversation) conversational).getNextEncryption()
2261 == Message.ENCRYPTION_AXOLOTL;
2262 }
2263 return false;
2264 }
2265
2266 private void acceptCallFromSessionInitialized() {
2267 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2268 sendSessionAccept();
2269 }
2270
2271 private synchronized boolean isInState(State... state) {
2272 return Arrays.asList(state).contains(this.state);
2273 }
2274
2275 private boolean transition(final State target) {
2276 return transition(target, null);
2277 }
2278
2279 private synchronized boolean transition(final State target, final Runnable runnable) {
2280 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2281 if (validTransitions != null && validTransitions.contains(target)) {
2282 this.state = target;
2283 if (runnable != null) {
2284 runnable.run();
2285 }
2286 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2287 updateEndUserState();
2288 updateOngoingCallNotification();
2289 return true;
2290 } else {
2291 return false;
2292 }
2293 }
2294
2295 void transitionOrThrow(final State target) {
2296 if (!transition(target)) {
2297 throw new IllegalStateException(
2298 String.format("Unable to transition from %s to %s", this.state, target));
2299 }
2300 }
2301
2302 @Override
2303 public void onIceCandidate(final IceCandidate iceCandidate) {
2304 final RtpContentMap rtpContentMap =
2305 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2306 final IceUdpTransportInfo.Credentials credentials;
2307 try {
2308 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2309 } catch (final IllegalArgumentException e) {
2310 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2311 return;
2312 }
2313 final String uFrag = credentials.ufrag;
2314 final IceUdpTransportInfo.Candidate candidate =
2315 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2316 if (candidate == null) {
2317 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2318 return;
2319 }
2320 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2321 sendTransportInfo(iceCandidate.sdpMid, candidate);
2322 }
2323
2324 @Override
2325 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2326 Log.d(
2327 Config.LOGTAG,
2328 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2329 this.stateHistory.add(newState);
2330 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2331 this.sessionDuration.start();
2332 updateOngoingCallNotification();
2333 } else if (this.sessionDuration.isRunning()) {
2334 this.sessionDuration.stop();
2335 updateOngoingCallNotification();
2336 }
2337
2338 final boolean neverConnected =
2339 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2340
2341 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2342 if (neverConnected) {
2343 if (isTerminated()) {
2344 Log.d(
2345 Config.LOGTAG,
2346 id.account.getJid().asBareJid()
2347 + ": not sending session-terminate after connectivity error because session is already in state "
2348 + this.state);
2349 return;
2350 }
2351 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2352 return;
2353 } else {
2354 this.restartIce();
2355 }
2356 }
2357 updateEndUserState();
2358 }
2359
2360 private void restartIce() {
2361 this.stateHistory.clear();
2362 this.webRTCWrapper.restartIce();
2363 }
2364
2365 @Override
2366 public void onRenegotiationNeeded() {
2367 this.webRTCWrapper.execute(this::renegotiate);
2368 }
2369
2370 private void renegotiate() {
2371 final SessionDescription sessionDescription;
2372 try {
2373 sessionDescription = setLocalSessionDescription();
2374 } catch (final Exception e) {
2375 final Throwable cause = Throwables.getRootCause(e);
2376 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2377 webRTCWrapper.close();
2378 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2379 return;
2380 }
2381 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2382 final RtpContentMap currentContentMap = getLocalContentMap();
2383 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2384 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2385
2386 Log.d(
2387 Config.LOGTAG,
2388 id.getAccount().getJid().asBareJid()
2389 + ": renegotiate. iceRestart="
2390 + iceRestart
2391 + " content id diff="
2392 + diff);
2393
2394 if (diff.hasModifications() && iceRestart) {
2395 webRTCWrapper.close();
2396 sendSessionTerminate(
2397 Reason.FAILED_APPLICATION,
2398 "WebRTC unexpectedly tried to modify content and transport at once");
2399 return;
2400 }
2401
2402 if (iceRestart) {
2403 initiateIceRestart(rtpContentMap);
2404 return;
2405 } else if (diff.isEmpty()) {
2406 Log.d(
2407 Config.LOGTAG,
2408 "renegotiation. nothing to do. SignalingState="
2409 + this.webRTCWrapper.getSignalingState());
2410 }
2411
2412 if (diff.added.size() > 0) {
2413 modifyLocalContentMap(rtpContentMap);
2414 sendContentAdd(rtpContentMap, diff.added);
2415 }
2416 }
2417
2418 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2419 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2420 final JinglePacket jinglePacket =
2421 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2422 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2423 jinglePacket.setTo(id.with);
2424 xmppConnectionService.sendIqPacket(
2425 id.account,
2426 jinglePacket,
2427 (account, response) -> {
2428 if (response.getType() == IqPacket.TYPE.RESULT) {
2429 Log.d(Config.LOGTAG, "received success to our ice restart");
2430 setLocalContentMap(rtpContentMap);
2431 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2432 return;
2433 }
2434 if (response.getType() == IqPacket.TYPE.ERROR) {
2435 if (isTieBreak(response)) {
2436 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2437 return;
2438 }
2439 handleIqErrorResponse(response);
2440 }
2441 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2442 handleIqTimeoutResponse(response);
2443 }
2444 });
2445 }
2446
2447 private boolean isTieBreak(final IqPacket response) {
2448 final Element error = response.findChild("error");
2449 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2450 }
2451
2452 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2453 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2454 this.outgoingContentAdd = contentAdd;
2455 final JinglePacket jinglePacket =
2456 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2457 jinglePacket.setTo(id.with);
2458 xmppConnectionService.sendIqPacket(
2459 id.account,
2460 jinglePacket,
2461 (connection, response) -> {
2462 if (response.getType() == IqPacket.TYPE.RESULT) {
2463 Log.d(
2464 Config.LOGTAG,
2465 id.getAccount().getJid().asBareJid()
2466 + ": received ACK to our content-add");
2467 return;
2468 }
2469 if (response.getType() == IqPacket.TYPE.ERROR) {
2470 if (isTieBreak(response)) {
2471 this.outgoingContentAdd = null;
2472 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2473 return;
2474 }
2475 handleIqErrorResponse(response);
2476 }
2477 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2478 handleIqTimeoutResponse(response);
2479 }
2480 });
2481 }
2482
2483 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2484 if (isInitiator()) {
2485 this.initiatorRtpContentMap = rtpContentMap;
2486 } else {
2487 this.responderRtpContentMap = rtpContentMap;
2488 }
2489 }
2490
2491 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2492 if (isInitiator()) {
2493 this.responderRtpContentMap = rtpContentMap;
2494 } else {
2495 this.initiatorRtpContentMap = rtpContentMap;
2496 }
2497 }
2498
2499 // this method is to be used for content map modifications that modify media
2500 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2501 final RtpContentMap activeContents = rtpContentMap.activeContents();
2502 setLocalContentMap(activeContents);
2503 this.webRTCWrapper.switchSpeakerPhonePreference(
2504 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2505 updateEndUserState();
2506 }
2507
2508 private SessionDescription setLocalSessionDescription()
2509 throws ExecutionException, InterruptedException {
2510 final org.webrtc.SessionDescription sessionDescription =
2511 this.webRTCWrapper.setLocalDescription().get();
2512 return SessionDescription.parse(sessionDescription.description);
2513 }
2514
2515 private void closeWebRTCSessionAfterFailedConnection() {
2516 this.webRTCWrapper.close();
2517 synchronized (this) {
2518 if (isTerminated()) {
2519 Log.d(
2520 Config.LOGTAG,
2521 id.account.getJid().asBareJid()
2522 + ": no need to send session-terminate after failed connection. Other party already did");
2523 return;
2524 }
2525 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2526 }
2527 }
2528
2529 public boolean zeroDuration() {
2530 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2531 }
2532
2533 public long getCallDuration() {
2534 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2535 }
2536
2537 public AppRTCAudioManager getAudioManager() {
2538 return webRTCWrapper.getAudioManager();
2539 }
2540
2541 public boolean isMicrophoneEnabled() {
2542 return webRTCWrapper.isMicrophoneEnabled();
2543 }
2544
2545 public boolean setMicrophoneEnabled(final boolean enabled) {
2546 return webRTCWrapper.setMicrophoneEnabled(enabled);
2547 }
2548
2549 public boolean isVideoEnabled() {
2550 return webRTCWrapper.isVideoEnabled();
2551 }
2552
2553 public void setVideoEnabled(final boolean enabled) {
2554 webRTCWrapper.setVideoEnabled(enabled);
2555 }
2556
2557 public boolean isCameraSwitchable() {
2558 return webRTCWrapper.isCameraSwitchable();
2559 }
2560
2561 public boolean isFrontCamera() {
2562 return webRTCWrapper.isFrontCamera();
2563 }
2564
2565 public ListenableFuture<Boolean> switchCamera() {
2566 return webRTCWrapper.switchCamera();
2567 }
2568
2569 @Override
2570 public void onAudioDeviceChanged(
2571 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2572 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2573 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2574 selectedAudioDevice, availableAudioDevices);
2575 }
2576
2577 private void updateEndUserState() {
2578 final RtpEndUserState endUserState = getEndUserState();
2579 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2580 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2581 id.account, id.with, id.sessionId, endUserState);
2582 }
2583
2584 private void updateOngoingCallNotification() {
2585 final State state = this.state;
2586 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2587 final boolean reconnecting;
2588 if (state == State.SESSION_ACCEPTED) {
2589 reconnecting =
2590 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2591 } else {
2592 reconnecting = false;
2593 }
2594 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2595 } else {
2596 xmppConnectionService.removeOngoingCall();
2597 }
2598 }
2599
2600 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2601 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2602 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2603 request.setTo(id.account.getDomain());
2604 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2605 xmppConnectionService.sendIqPacket(
2606 id.account,
2607 request,
2608 (account, response) -> {
2609 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2610 new ImmutableList.Builder<>();
2611 if (response.getType() == IqPacket.TYPE.RESULT) {
2612 final Element services =
2613 response.findChild(
2614 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2615 final List<Element> children =
2616 services == null
2617 ? Collections.emptyList()
2618 : services.getChildren();
2619 for (final Element child : children) {
2620 if ("service".equals(child.getName())) {
2621 final String type = child.getAttribute("type");
2622 final String host = child.getAttribute("host");
2623 final String sport = child.getAttribute("port");
2624 final Integer port =
2625 sport == null ? null : Ints.tryParse(sport);
2626 final String transport = child.getAttribute("transport");
2627 final String username = child.getAttribute("username");
2628 final String password = child.getAttribute("password");
2629 if (Strings.isNullOrEmpty(host) || port == null) {
2630 continue;
2631 }
2632 if (port < 0 || port > 65535) {
2633 continue;
2634 }
2635
2636
2637
2638
2639 if (Arrays.asList("stun", "stuns", "turn", "turns")
2640 .contains(type)
2641 && Arrays.asList("udp", "tcp").contains(transport)) {
2642 if (Arrays.asList("stuns", "turns").contains(type)
2643 && "udp".equals(transport)) {
2644 Log.d(
2645 Config.LOGTAG,
2646 id.account.getJid().asBareJid()
2647 + ": skipping invalid combination of udp/tls in external services");
2648 continue;
2649 }
2650
2651 // STUN URLs do not support a query section since M110
2652 final String uri;
2653 if (Arrays.asList("stun","stuns").contains(type)) {
2654 uri = String.format("%s:%s%s", type, IP.wrapIPv6(host),port);
2655 } else {
2656 uri = String.format(
2657 "%s:%s:%s?transport=%s",
2658 type,
2659 IP.wrapIPv6(host),
2660 port,
2661 transport);
2662 }
2663
2664 final PeerConnection.IceServer.Builder iceServerBuilder =
2665 PeerConnection.IceServer.builder(uri);
2666 iceServerBuilder.setTlsCertPolicy(
2667 PeerConnection.TlsCertPolicy
2668 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2669 if (username != null && password != null) {
2670 iceServerBuilder.setUsername(username);
2671 iceServerBuilder.setPassword(password);
2672 } else if (Arrays.asList("turn", "turns").contains(type)) {
2673 // The WebRTC spec requires throwing an
2674 // InvalidAccessError when username (from libwebrtc
2675 // source coder)
2676 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2677 Log.d(
2678 Config.LOGTAG,
2679 id.account.getJid().asBareJid()
2680 + ": skipping "
2681 + type
2682 + "/"
2683 + transport
2684 + " without username and password");
2685 continue;
2686 }
2687 final PeerConnection.IceServer iceServer =
2688 iceServerBuilder.createIceServer();
2689 Log.d(
2690 Config.LOGTAG,
2691 id.account.getJid().asBareJid()
2692 + ": discovered ICE Server: "
2693 + iceServer);
2694 listBuilder.add(iceServer);
2695 }
2696 }
2697 }
2698 }
2699 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2700 if (iceServers.size() == 0) {
2701 Log.w(
2702 Config.LOGTAG,
2703 id.account.getJid().asBareJid()
2704 + ": no ICE server found "
2705 + response);
2706 }
2707 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2708 });
2709 } else {
2710 Log.w(
2711 Config.LOGTAG,
2712 id.account.getJid().asBareJid() + ": has no external service discovery");
2713 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2714 }
2715 }
2716
2717 private void finish() {
2718 if (isTerminated()) {
2719 this.cancelRingingTimeout();
2720 this.webRTCWrapper.verifyClosed();
2721 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2722 this.jingleConnectionManager.finishConnectionOrThrow(this);
2723 try {
2724 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2725 log.getParentFile().mkdirs();
2726 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2727 } catch (final IOException e) { }
2728 } else {
2729 throw new IllegalStateException(
2730 String.format("Unable to call finish from %s", this.state));
2731 }
2732 }
2733
2734 private void writeLogMessage(final State state) {
2735 final long duration = getCallDuration();
2736 if (state == State.TERMINATED_SUCCESS
2737 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2738 writeLogMessageSuccess(duration);
2739 } else {
2740 writeLogMessageMissed();
2741 }
2742 }
2743
2744 private void writeLogMessageSuccess(final long duration) {
2745 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2746 this.writeMessage();
2747 }
2748
2749 private void writeLogMessageMissed() {
2750 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2751 this.writeMessage();
2752 }
2753
2754 private void writeMessage() {
2755 final Conversational conversational = message.getConversation();
2756 if (conversational instanceof Conversation) {
2757 ((Conversation) conversational).add(this.message);
2758 xmppConnectionService.createMessageAsync(message);
2759 xmppConnectionService.updateConversationUi();
2760 } else {
2761 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2762 }
2763 }
2764
2765 public State getState() {
2766 return this.state;
2767 }
2768
2769 boolean isTerminated() {
2770 return TERMINATED.contains(this.state);
2771 }
2772
2773 public Optional<VideoTrack> getLocalVideoTrack() {
2774 return webRTCWrapper.getLocalVideoTrack();
2775 }
2776
2777 public Optional<VideoTrack> getRemoteVideoTrack() {
2778 return webRTCWrapper.getRemoteVideoTrack();
2779 }
2780
2781 public EglBase.Context getEglBaseContext() {
2782 return webRTCWrapper.getEglBaseContext();
2783 }
2784
2785 void setProposedMedia(final Set<Media> media) {
2786 this.proposedMedia = media;
2787 }
2788
2789 public void fireStateUpdate() {
2790 final RtpEndUserState endUserState = getEndUserState();
2791 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2792 id.account, id.with, id.sessionId, endUserState);
2793 }
2794
2795 public boolean isSwitchToVideoAvailable() {
2796 final boolean prerequisite =
2797 Media.audioOnly(getMedia())
2798 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2799 .contains(getEndUserState());
2800 return prerequisite && remoteHasVideoFeature();
2801 }
2802
2803 private boolean remoteHasVideoFeature() {
2804 final Contact contact = id.getContact();
2805 final Presence presence =
2806 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2807 final ServiceDiscoveryResult serviceDiscoveryResult =
2808 presence == null ? null : presence.getServiceDiscoveryResult();
2809 final List<String> features =
2810 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
2811 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
2812 }
2813
2814 private interface OnIceServersDiscovered {
2815 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2816 }
2817}