1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4import android.os.Environment;
5
6import androidx.annotation.NonNull;
7import androidx.annotation.Nullable;
8
9import com.google.common.base.Joiner;
10import com.google.common.base.Optional;
11import com.google.common.base.Preconditions;
12import com.google.common.base.Stopwatch;
13import com.google.common.base.Strings;
14import com.google.common.base.Throwables;
15import com.google.common.collect.Collections2;
16import com.google.common.collect.ImmutableList;
17import com.google.common.collect.ImmutableMap;
18import com.google.common.collect.ImmutableSet;
19import com.google.common.collect.Sets;
20import com.google.common.primitives.Ints;
21import com.google.common.util.concurrent.FutureCallback;
22import com.google.common.util.concurrent.Futures;
23import com.google.common.util.concurrent.ListenableFuture;
24import com.google.common.util.concurrent.MoreExecutors;
25
26import org.webrtc.DtmfSender;
27import org.webrtc.EglBase;
28import org.webrtc.IceCandidate;
29import org.webrtc.PeerConnection;
30import org.webrtc.VideoTrack;
31
32import java.io.File;
33import java.io.IOException;
34import java.util.Arrays;
35import java.util.Collection;
36import java.util.Collections;
37import java.util.LinkedList;
38import java.util.List;
39import java.util.Map;
40import java.util.Queue;
41import java.util.Set;
42import java.util.concurrent.ExecutionException;
43import java.util.concurrent.ScheduledFuture;
44import java.util.concurrent.TimeUnit;
45
46import eu.siacs.conversations.BuildConfig;
47import eu.siacs.conversations.Config;
48import eu.siacs.conversations.crypto.axolotl.AxolotlService;
49import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
50import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
51import eu.siacs.conversations.entities.Account;
52import eu.siacs.conversations.entities.Contact;
53import eu.siacs.conversations.entities.Conversation;
54import eu.siacs.conversations.entities.Conversational;
55import eu.siacs.conversations.entities.Message;
56import eu.siacs.conversations.entities.Presence;
57import eu.siacs.conversations.entities.RtpSessionStatus;
58import eu.siacs.conversations.entities.ServiceDiscoveryResult;
59import eu.siacs.conversations.services.AppRTCAudioManager;
60import eu.siacs.conversations.utils.IP;
61import eu.siacs.conversations.xml.Element;
62import eu.siacs.conversations.xml.Namespace;
63import eu.siacs.conversations.xmpp.Jid;
64import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
65import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
66import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
67import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
68import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
69import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
70import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
71import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
72import eu.siacs.conversations.xmpp.stanzas.IqPacket;
73import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
74
75public class JingleRtpConnection extends AbstractJingleConnection
76 implements WebRTCWrapper.EventCallback {
77
78 public static final List<State> STATES_SHOWING_ONGOING_CALL =
79 Arrays.asList(
80 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
81 private static final long BUSY_TIME_OUT = 30;
82 private static final List<State> TERMINATED =
83 Arrays.asList(
84 State.ACCEPTED,
85 State.REJECTED,
86 State.REJECTED_RACED,
87 State.RETRACTED,
88 State.RETRACTED_RACED,
89 State.TERMINATED_SUCCESS,
90 State.TERMINATED_DECLINED_OR_BUSY,
91 State.TERMINATED_CONNECTIVITY_ERROR,
92 State.TERMINATED_CANCEL_OR_TIMEOUT,
93 State.TERMINATED_APPLICATION_FAILURE,
94 State.TERMINATED_SECURITY_ERROR);
95
96 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
97
98 static {
99 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
100 new ImmutableMap.Builder<>();
101 transitionBuilder.put(
102 State.NULL,
103 ImmutableList.of(
104 State.PROPOSED,
105 State.SESSION_INITIALIZED,
106 State.TERMINATED_APPLICATION_FAILURE,
107 State.TERMINATED_SECURITY_ERROR));
108 transitionBuilder.put(
109 State.PROPOSED,
110 ImmutableList.of(
111 State.ACCEPTED,
112 State.PROCEED,
113 State.REJECTED,
114 State.RETRACTED,
115 State.TERMINATED_APPLICATION_FAILURE,
116 State.TERMINATED_SECURITY_ERROR,
117 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
118 // rebinds
119 ));
120 transitionBuilder.put(
121 State.PROCEED,
122 ImmutableList.of(
123 State.REJECTED_RACED,
124 State.RETRACTED_RACED,
125 State.SESSION_INITIALIZED_PRE_APPROVED,
126 State.TERMINATED_SUCCESS,
127 State.TERMINATED_APPLICATION_FAILURE,
128 State.TERMINATED_SECURITY_ERROR,
129 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
130 // bounces of the proceed message
131 ));
132 transitionBuilder.put(
133 State.SESSION_INITIALIZED,
134 ImmutableList.of(
135 State.SESSION_ACCEPTED,
136 State.TERMINATED_SUCCESS,
137 State.TERMINATED_DECLINED_OR_BUSY,
138 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
139 // and IQ timeouts
140 State.TERMINATED_CANCEL_OR_TIMEOUT,
141 State.TERMINATED_APPLICATION_FAILURE,
142 State.TERMINATED_SECURITY_ERROR));
143 transitionBuilder.put(
144 State.SESSION_INITIALIZED_PRE_APPROVED,
145 ImmutableList.of(
146 State.SESSION_ACCEPTED,
147 State.TERMINATED_SUCCESS,
148 State.TERMINATED_DECLINED_OR_BUSY,
149 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
150 // and IQ timeouts
151 State.TERMINATED_CANCEL_OR_TIMEOUT,
152 State.TERMINATED_APPLICATION_FAILURE,
153 State.TERMINATED_SECURITY_ERROR));
154 transitionBuilder.put(
155 State.SESSION_ACCEPTED,
156 ImmutableList.of(
157 State.TERMINATED_SUCCESS,
158 State.TERMINATED_DECLINED_OR_BUSY,
159 State.TERMINATED_CONNECTIVITY_ERROR,
160 State.TERMINATED_CANCEL_OR_TIMEOUT,
161 State.TERMINATED_APPLICATION_FAILURE,
162 State.TERMINATED_SECURITY_ERROR));
163 VALID_TRANSITIONS = transitionBuilder.build();
164 }
165
166 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
167 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
168 pendingIceCandidates = new LinkedList<>();
169 private final OmemoVerification omemoVerification = new OmemoVerification();
170 private final Message message;
171 private State state = State.NULL;
172 private Set<Media> proposedMedia;
173 private RtpContentMap initiatorRtpContentMap;
174 private RtpContentMap responderRtpContentMap;
175 private RtpContentMap incomingContentAdd;
176 private RtpContentMap outgoingContentAdd;
177 private IceUdpTransportInfo.Setup peerDtlsSetup;
178 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
179 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
180 private ScheduledFuture<?> ringingTimeoutFuture;
181 private final long created = System.currentTimeMillis() / 1000L;
182
183 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
184 super(jingleConnectionManager, id, initiator);
185 final Conversation conversation =
186 jingleConnectionManager
187 .getXmppConnectionService()
188 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
189 this.message =
190 new Message(
191 conversation,
192 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
193 Message.TYPE_RTP_SESSION,
194 id.sessionId);
195 }
196
197 private static State reasonToState(Reason reason) {
198 switch (reason) {
199 case SUCCESS:
200 return State.TERMINATED_SUCCESS;
201 case DECLINE:
202 case BUSY:
203 return State.TERMINATED_DECLINED_OR_BUSY;
204 case CANCEL:
205 case TIMEOUT:
206 return State.TERMINATED_CANCEL_OR_TIMEOUT;
207 case SECURITY_ERROR:
208 return State.TERMINATED_SECURITY_ERROR;
209 case FAILED_APPLICATION:
210 case UNSUPPORTED_TRANSPORTS:
211 case UNSUPPORTED_APPLICATIONS:
212 return State.TERMINATED_APPLICATION_FAILURE;
213 default:
214 return State.TERMINATED_CONNECTIVITY_ERROR;
215 }
216 }
217
218 @Override
219 synchronized void deliverPacket(final JinglePacket jinglePacket) {
220 switch (jinglePacket.getAction()) {
221 case SESSION_INITIATE:
222 receiveSessionInitiate(jinglePacket);
223 break;
224 case TRANSPORT_INFO:
225 receiveTransportInfo(jinglePacket);
226 break;
227 case SESSION_ACCEPT:
228 receiveSessionAccept(jinglePacket);
229 break;
230 case SESSION_TERMINATE:
231 receiveSessionTerminate(jinglePacket);
232 break;
233 case CONTENT_ADD:
234 receiveContentAdd(jinglePacket);
235 break;
236 case CONTENT_ACCEPT:
237 receiveContentAccept(jinglePacket);
238 break;
239 case CONTENT_REJECT:
240 receiveContentReject(jinglePacket);
241 break;
242 case CONTENT_REMOVE:
243 receiveContentRemove(jinglePacket);
244 break;
245 default:
246 respondOk(jinglePacket);
247 Log.d(
248 Config.LOGTAG,
249 String.format(
250 "%s: received unhandled jingle action %s",
251 id.account.getJid().asBareJid(), jinglePacket.getAction()));
252 break;
253 }
254 }
255
256 @Override
257 synchronized void notifyRebound() {
258 if (isTerminated()) {
259 return;
260 }
261 webRTCWrapper.close();
262 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
263 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
264 }
265 if (isInState(
266 State.SESSION_INITIALIZED,
267 State.SESSION_INITIALIZED_PRE_APPROVED,
268 State.SESSION_ACCEPTED)) {
269 // we might have already changed resources (full jid) at this point; so this might not
270 // even reach the other party
271 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
272 } else {
273 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
274 finish();
275 }
276 }
277
278 public boolean applyDtmfTone(String tone) {
279 return webRTCWrapper.applyDtmfTone(tone);
280 }
281
282 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
283 respondOk(jinglePacket);
284 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
285 final State previous = this.state;
286 Log.d(
287 Config.LOGTAG,
288 id.account.getJid().asBareJid()
289 + ": received session terminate reason="
290 + wrapper.reason
291 + "("
292 + Strings.nullToEmpty(wrapper.text)
293 + ") while in state "
294 + previous);
295 if (TERMINATED.contains(previous)) {
296 Log.d(
297 Config.LOGTAG,
298 id.account.getJid().asBareJid()
299 + ": ignoring session terminate because already in "
300 + previous);
301 return;
302 }
303 webRTCWrapper.close();
304 final State target = reasonToState(wrapper.reason);
305 transitionOrThrow(target);
306 writeLogMessage(target);
307 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
308 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
309 }
310 finish();
311 }
312
313 private void receiveTransportInfo(final JinglePacket jinglePacket) {
314 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
315 // INITIALIZED only after transport-info has been received
316 if (isInState(
317 State.NULL,
318 State.PROCEED,
319 State.SESSION_INITIALIZED,
320 State.SESSION_INITIALIZED_PRE_APPROVED,
321 State.SESSION_ACCEPTED)) {
322 final RtpContentMap contentMap;
323 try {
324 contentMap = RtpContentMap.of(jinglePacket);
325 } catch (final IllegalArgumentException | NullPointerException e) {
326 Log.d(
327 Config.LOGTAG,
328 id.account.getJid().asBareJid()
329 + ": improperly formatted contents; ignoring",
330 e);
331 respondOk(jinglePacket);
332 return;
333 }
334 receiveTransportInfo(jinglePacket, contentMap);
335 } else {
336 if (isTerminated()) {
337 respondOk(jinglePacket);
338 Log.d(
339 Config.LOGTAG,
340 id.account.getJid().asBareJid()
341 + ": ignoring out-of-order transport info; we where already terminated");
342 } else {
343 Log.d(
344 Config.LOGTAG,
345 id.account.getJid().asBareJid()
346 + ": received transport info while in state="
347 + this.state);
348 terminateWithOutOfOrder(jinglePacket);
349 }
350 }
351 }
352
353 private void receiveTransportInfo(
354 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
355 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
356 contentMap.contents.entrySet();
357 if (this.state == State.SESSION_ACCEPTED) {
358 // zero candidates + modified credentials are an ICE restart offer
359 if (checkForIceRestart(jinglePacket, contentMap)) {
360 return;
361 }
362 respondOk(jinglePacket);
363 try {
364 processCandidates(candidates);
365 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
366 Log.w(
367 Config.LOGTAG,
368 id.account.getJid().asBareJid()
369 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
370 }
371 } else {
372 respondOk(jinglePacket);
373 pendingIceCandidates.addAll(candidates);
374 }
375 }
376
377 private void receiveContentAdd(final JinglePacket jinglePacket) {
378 final RtpContentMap modification;
379 try {
380 modification = RtpContentMap.of(jinglePacket);
381 modification.requireContentDescriptions();
382 } catch (final RuntimeException e) {
383 Log.d(
384 Config.LOGTAG,
385 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
386 Throwables.getRootCause(e));
387 respondOk(jinglePacket);
388 webRTCWrapper.close();
389 sendSessionTerminate(Reason.of(e), e.getMessage());
390 return;
391 }
392 if (isInState(State.SESSION_ACCEPTED)) {
393 receiveContentAdd(jinglePacket, modification);
394 } else {
395 terminateWithOutOfOrder(jinglePacket);
396 }
397 }
398
399 private void receiveContentAdd(
400 final JinglePacket jinglePacket, final RtpContentMap modification) {
401 final RtpContentMap remote = getRemoteContentMap();
402 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
403 respondOk(jinglePacket);
404 this.webRTCWrapper.close();
405 sendSessionTerminate(
406 Reason.FAILED_APPLICATION,
407 String.format(
408 "contents with names %s already exists",
409 Joiner.on(", ").join(modification.getNames())));
410 return;
411 }
412 final ContentAddition contentAddition =
413 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
414
415 final RtpContentMap outgoing = this.outgoingContentAdd;
416 final Set<ContentAddition.Summary> outgoingContentAddSummary =
417 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
418
419 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
420 if (isInitiator()) {
421 Log.d(
422 Config.LOGTAG,
423 id.getAccount().getJid().asBareJid()
424 + ": respond with tie break to matching content-add offer");
425 respondWithTieBreak(jinglePacket);
426 } else {
427 Log.d(
428 Config.LOGTAG,
429 id.getAccount().getJid().asBareJid()
430 + ": automatically accept matching content-add offer");
431 acceptContentAdd(contentAddition.summary, modification);
432 }
433 return;
434 }
435
436 // once we can display multiple video tracks we can be more loose with this condition
437 // theoretically it should also be fine to automatically accept audio only contents
438 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
439 Log.d(
440 Config.LOGTAG,
441 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
442 this.incomingContentAdd = modification;
443 respondOk(jinglePacket);
444 updateEndUserState();
445 } else {
446 respondOk(jinglePacket);
447 // TODO do we want to add a reason?
448 rejectContentAdd(modification);
449 }
450 }
451
452 private void receiveContentAccept(final JinglePacket jinglePacket) {
453 final RtpContentMap receivedContentAccept;
454 try {
455 receivedContentAccept = RtpContentMap.of(jinglePacket);
456 receivedContentAccept.requireContentDescriptions();
457 } catch (final RuntimeException e) {
458 Log.d(
459 Config.LOGTAG,
460 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
461 Throwables.getRootCause(e));
462 respondOk(jinglePacket);
463 webRTCWrapper.close();
464 sendSessionTerminate(Reason.of(e), e.getMessage());
465 return;
466 }
467
468 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
469 if (outgoingContentAdd == null) {
470 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
471 terminateWithOutOfOrder(jinglePacket);
472 return;
473 }
474 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
475 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
476 this.outgoingContentAdd = null;
477 respondOk(jinglePacket);
478 receiveContentAccept(receivedContentAccept);
479 } else {
480 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
481 terminateWithOutOfOrder(jinglePacket);
482 }
483 }
484
485 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
486 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
487 final RtpContentMap modifiedContentMap =
488 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
489
490 setRemoteContentMap(modifiedContentMap);
491
492 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
493
494 final org.webrtc.SessionDescription sdp =
495 new org.webrtc.SessionDescription(
496 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
497
498 try {
499 this.webRTCWrapper.setRemoteDescription(sdp).get();
500 } catch (final Exception e) {
501 final Throwable cause = Throwables.getRootCause(e);
502 Log.d(
503 Config.LOGTAG,
504 id.getAccount().getJid().asBareJid()
505 + ": unable to set remote description after receiving content-accept",
506 cause);
507 webRTCWrapper.close();
508 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
509 return;
510 }
511 updateEndUserState();
512 Log.d(
513 Config.LOGTAG,
514 id.getAccount().getJid().asBareJid()
515 + ": remote has accepted content-add "
516 + ContentAddition.summary(receivedContentAccept));
517 }
518
519 private void receiveContentReject(final JinglePacket jinglePacket) {
520 final RtpContentMap receivedContentReject;
521 try {
522 receivedContentReject = RtpContentMap.of(jinglePacket);
523 } catch (final RuntimeException e) {
524 Log.d(
525 Config.LOGTAG,
526 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
527 Throwables.getRootCause(e));
528 respondOk(jinglePacket);
529 this.webRTCWrapper.close();
530 sendSessionTerminate(Reason.of(e), e.getMessage());
531 return;
532 }
533
534 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
535 if (outgoingContentAdd == null) {
536 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
537 terminateWithOutOfOrder(jinglePacket);
538 return;
539 }
540 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
541 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
542 this.outgoingContentAdd = null;
543 respondOk(jinglePacket);
544 Log.d(Config.LOGTAG,jinglePacket.toString());
545 receiveContentReject(ourSummary);
546 } else {
547 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
548 terminateWithOutOfOrder(jinglePacket);
549 }
550 }
551
552 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
553 try {
554 this.webRTCWrapper.removeTrack(Media.VIDEO);
555 final RtpContentMap localContentMap = customRollback();
556 modifyLocalContentMap(localContentMap);
557 } catch (final Exception e) {
558 final Throwable cause = Throwables.getRootCause(e);
559 Log.d(
560 Config.LOGTAG,
561 id.getAccount().getJid().asBareJid()
562 + ": unable to rollback local description after receiving content-reject",
563 cause);
564 webRTCWrapper.close();
565 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
566 return;
567 }
568 Log.d(
569 Config.LOGTAG,
570 id.getAccount().getJid().asBareJid()
571 + ": remote has rejected our content-add "
572 + summary);
573 }
574
575 private void receiveContentRemove(final JinglePacket jinglePacket) {
576 final RtpContentMap receivedContentRemove;
577 try {
578 receivedContentRemove = RtpContentMap.of(jinglePacket);
579 receivedContentRemove.requireContentDescriptions();
580 } catch (final RuntimeException e) {
581 Log.d(
582 Config.LOGTAG,
583 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
584 Throwables.getRootCause(e));
585 respondOk(jinglePacket);
586 this.webRTCWrapper.close();
587 sendSessionTerminate(Reason.of(e), e.getMessage());
588 return;
589 }
590 respondOk(jinglePacket);
591 receiveContentRemove(receivedContentRemove);
592 }
593
594 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
595 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
596 final Set<ContentAddition.Summary> contentAddSummary =
597 incomingContentAdd == null
598 ? Collections.emptySet()
599 : ContentAddition.summary(incomingContentAdd);
600 final Set<ContentAddition.Summary> removeSummary =
601 ContentAddition.summary(receivedContentRemove);
602 if (contentAddSummary.equals(removeSummary)) {
603 this.incomingContentAdd = null;
604 updateEndUserState();
605 } else {
606 webRTCWrapper.close();
607 sendSessionTerminate(
608 Reason.FAILED_APPLICATION,
609 String.format(
610 "%s only supports %s as a means to retract a not yet accepted %s",
611 BuildConfig.APP_NAME,
612 JinglePacket.Action.CONTENT_REMOVE,
613 JinglePacket.Action.CONTENT_ACCEPT));
614 }
615 }
616
617 public synchronized void retractContentAdd() {
618 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
619 if (outgoingContentAdd == null) {
620 throw new IllegalStateException("Not outgoing content add");
621 }
622 try {
623 webRTCWrapper.removeTrack(Media.VIDEO);
624 final RtpContentMap localContentMap = customRollback();
625 modifyLocalContentMap(localContentMap);
626 } catch (final Exception e) {
627 final Throwable cause = Throwables.getRootCause(e);
628 Log.d(
629 Config.LOGTAG,
630 id.getAccount().getJid().asBareJid()
631 + ": unable to rollback local description after trying to retract content-add",
632 cause);
633 webRTCWrapper.close();
634 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
635 return;
636 }
637 this.outgoingContentAdd = null;
638 final JinglePacket retract =
639 outgoingContentAdd
640 .toStub()
641 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
642 this.send(retract);
643 Log.d(
644 Config.LOGTAG,
645 id.getAccount().getJid()
646 + ": retract content-add "
647 + ContentAddition.summary(outgoingContentAdd));
648 }
649
650 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
651 final SessionDescription sdp = setLocalSessionDescription();
652 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
653 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
654 this.webRTCWrapper
655 .setRemoteDescription(
656 new org.webrtc.SessionDescription(
657 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
658 .get();
659 return localRtpContentMap;
660 }
661
662 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
663 final RtpContentMap currentRemote = getRemoteContentMap();
664 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
665 if (diff.isEmpty()) {
666 throw new IllegalStateException(
667 "Unexpected rollback condition. No difference between local and remote");
668 }
669 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
670 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
671 final RtpContentMap nextRemote =
672 currentRemote.addContent(
673 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
674 return SessionDescription.of(nextRemote, !isInitiator());
675 }
676 throw new IllegalStateException(
677 "Unexpected rollback condition. Senders were not uniformly none");
678 }
679
680 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
681 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
682 if (incomingContentAdd == null) {
683 throw new IllegalStateException("No incoming content add");
684 }
685
686 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
687 this.incomingContentAdd = null;
688 acceptContentAdd(contentAddition, incomingContentAdd);
689 } else {
690 throw new IllegalStateException("Accepted content add does not match pending content-add");
691 }
692 }
693
694 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
695 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
696 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
697 this.setRemoteContentMap(modifiedContentMap);
698
699 final SessionDescription offer;
700 try {
701 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
702 } catch (final IllegalArgumentException | NullPointerException e) {
703 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
704 webRTCWrapper.close();
705 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
706 return;
707 }
708 this.incomingContentAdd = null;
709 acceptContentAdd(contentAddition, offer);
710 }
711
712 private void acceptContentAdd(
713 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
714 final org.webrtc.SessionDescription sdp =
715 new org.webrtc.SessionDescription(
716 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
717 try {
718 this.webRTCWrapper.setRemoteDescription(sdp).get();
719
720 // TODO add tracks for 'media' where contentAddition.senders matches
721
722 // TODO if senders.sending(isInitiator())
723
724 this.webRTCWrapper.addTrack(Media.VIDEO);
725
726 // TODO add additional transceivers for recv only cases
727
728 final SessionDescription answer = setLocalSessionDescription();
729 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
730
731 final RtpContentMap contentAcceptMap =
732 rtpContentMap.toContentModification(
733 Collections2.transform(contentAddition, ca -> ca.name));
734 Log.d(
735 Config.LOGTAG,
736 id.getAccount().getJid().asBareJid()
737 + ": sending content-accept "
738 + ContentAddition.summary(contentAcceptMap));
739 modifyLocalContentMap(rtpContentMap);
740 sendContentAccept(contentAcceptMap);
741 } catch (final Exception e) {
742 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
743 webRTCWrapper.close();
744 sendSessionTerminate(Reason.FAILED_APPLICATION);
745 }
746 }
747
748 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
749 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
750 send(jinglePacket);
751 }
752
753 public synchronized void rejectContentAdd() {
754 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
755 if (incomingContentAdd == null) {
756 throw new IllegalStateException("No incoming content add");
757 }
758 this.incomingContentAdd = null;
759 updateEndUserState();
760 rejectContentAdd(incomingContentAdd);
761 }
762
763 private void rejectContentAdd(final RtpContentMap contentMap) {
764 final JinglePacket jinglePacket =
765 contentMap
766 .toStub()
767 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
768 Log.d(
769 Config.LOGTAG,
770 id.getAccount().getJid().asBareJid()
771 + ": rejecting content "
772 + ContentAddition.summary(contentMap));
773 send(jinglePacket);
774 }
775
776 private boolean checkForIceRestart(
777 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
778 final RtpContentMap existing = getRemoteContentMap();
779 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
780 final IceUdpTransportInfo.Credentials newCredentials;
781 try {
782 existingCredentials = existing.getCredentials();
783 newCredentials = rtpContentMap.getDistinctCredentials();
784 } catch (final IllegalStateException e) {
785 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
786 return false;
787 }
788 if (existingCredentials.contains(newCredentials)) {
789 return false;
790 }
791 // TODO an alternative approach is to check if we already got an iq result to our
792 // ICE-restart
793 // and if that's the case we are seeing an answer.
794 // This might be more spec compliant but also more error prone potentially
795 final boolean isOffer = rtpContentMap.emptyCandidates();
796 final RtpContentMap restartContentMap;
797 try {
798 if (isOffer) {
799 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
800 restartContentMap =
801 existing.modifiedCredentials(
802 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
803 } else {
804 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
805 Log.d(
806 Config.LOGTAG,
807 "received confirmation of ICE restart"
808 + newCredentials
809 + " peer_setup="
810 + setup);
811 // DTLS setup attribute needs to be rewritten to reflect current peer state
812 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
813 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
814 }
815 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
816 return isOffer;
817 } else {
818 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
819 respondWithTieBreak(jinglePacket);
820 return true;
821 }
822 } catch (final Exception exception) {
823 respondOk(jinglePacket);
824 final Throwable rootCause = Throwables.getRootCause(exception);
825 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
826 // If this happens a termination is already in progress
827 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
828 return true;
829 }
830 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
831 webRTCWrapper.close();
832 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
833 return true;
834 }
835 }
836
837 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
838 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
839 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
840 throw new IllegalStateException("Invalid peer setup");
841 }
842 return peerSetup;
843 }
844
845 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
846 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
847 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
848 }
849 this.peerDtlsSetup = setup;
850 }
851
852 private boolean applyIceRestart(
853 final JinglePacket jinglePacket,
854 final RtpContentMap restartContentMap,
855 final boolean isOffer)
856 throws ExecutionException, InterruptedException {
857 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
858 final org.webrtc.SessionDescription.Type type =
859 isOffer
860 ? org.webrtc.SessionDescription.Type.OFFER
861 : org.webrtc.SessionDescription.Type.ANSWER;
862 org.webrtc.SessionDescription sdp =
863 new org.webrtc.SessionDescription(type, sessionDescription.toString());
864 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
865 if (isInitiator()) {
866 // We ignore the offer and respond with tie-break. This will clause the responder
867 // not to apply the content map
868 return false;
869 }
870 }
871 webRTCWrapper.setRemoteDescription(sdp).get();
872 setRemoteContentMap(restartContentMap);
873 if (isOffer) {
874 webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
875 final SessionDescription localSessionDescription = setLocalSessionDescription();
876 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
877 // We need to respond OK before sending any candidates
878 respondOk(jinglePacket);
879 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
880 } else {
881 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
882 }
883 return true;
884 }
885
886 private void processCandidates(
887 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
888 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
889 processCandidate(content);
890 }
891 }
892
893 private void processCandidate(
894 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
895 final RtpContentMap rtpContentMap = getRemoteContentMap();
896 final List<String> indices = toIdentificationTags(rtpContentMap);
897 final String sdpMid = content.getKey(); // aka content name
898 final IceUdpTransportInfo transport = content.getValue().transport;
899 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
900
901 // TODO check that credentials remained the same
902
903 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
904 final String sdp;
905 try {
906 sdp = candidate.toSdpAttribute(credentials.ufrag);
907 } catch (final IllegalArgumentException e) {
908 Log.d(
909 Config.LOGTAG,
910 id.account.getJid().asBareJid()
911 + ": ignoring invalid ICE candidate "
912 + e.getMessage());
913 continue;
914 }
915 final int mLineIndex = indices.indexOf(sdpMid);
916 if (mLineIndex < 0) {
917 Log.w(
918 Config.LOGTAG,
919 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
920 }
921 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
922 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
923 this.webRTCWrapper.addIceCandidate(iceCandidate);
924 }
925 }
926
927 private RtpContentMap getRemoteContentMap() {
928 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
929 }
930
931 private RtpContentMap getLocalContentMap() {
932 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
933 }
934
935 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
936 final Group originalGroup = rtpContentMap.group;
937 final List<String> identificationTags =
938 originalGroup == null
939 ? rtpContentMap.getNames()
940 : originalGroup.getIdentificationTags();
941 if (identificationTags.size() == 0) {
942 Log.w(
943 Config.LOGTAG,
944 id.account.getJid().asBareJid()
945 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
946 }
947 return identificationTags;
948 }
949
950 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
951 final JinglePacket jinglePacket, final boolean expectVerification) {
952 final RtpContentMap receivedContentMap;
953 try {
954 receivedContentMap = RtpContentMap.of(jinglePacket);
955 } catch (final Exception e) {
956 return Futures.immediateFailedFuture(e);
957 }
958 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
959 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
960 id.account
961 .getAxolotlService()
962 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
963 return Futures.transform(
964 future,
965 omemoVerifiedPayload -> {
966 // TODO test if an exception here triggers a correct abort
967 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
968 Log.d(
969 Config.LOGTAG,
970 id.account.getJid().asBareJid()
971 + ": received verifiable DTLS fingerprint via "
972 + omemoVerification);
973 return omemoVerifiedPayload.getPayload();
974 },
975 MoreExecutors.directExecutor());
976 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
977 return Futures.immediateFailedFuture(
978 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
979 } else {
980 return Futures.immediateFuture(receivedContentMap);
981 }
982 }
983
984 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
985 if (isInitiator()) {
986 Log.d(
987 Config.LOGTAG,
988 String.format(
989 "%s: received session-initiate even though we were initiating",
990 id.account.getJid().asBareJid()));
991 if (isTerminated()) {
992 Log.d(
993 Config.LOGTAG,
994 String.format(
995 "%s: got a reason to terminate with out-of-order. but already in state %s",
996 id.account.getJid().asBareJid(), getState()));
997 respondWithOutOfOrder(jinglePacket);
998 } else {
999 terminateWithOutOfOrder(jinglePacket);
1000 }
1001 return;
1002 }
1003 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1004 Futures.addCallback(
1005 future,
1006 new FutureCallback<RtpContentMap>() {
1007 @Override
1008 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1009 receiveSessionInitiate(jinglePacket, rtpContentMap);
1010 }
1011
1012 @Override
1013 public void onFailure(@NonNull final Throwable throwable) {
1014 respondOk(jinglePacket);
1015 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1016 }
1017 },
1018 MoreExecutors.directExecutor());
1019 }
1020
1021 private void receiveSessionInitiate(
1022 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1023 try {
1024 contentMap.requireContentDescriptions();
1025 contentMap.requireDTLSFingerprint(true);
1026 } catch (final RuntimeException e) {
1027 Log.d(
1028 Config.LOGTAG,
1029 id.account.getJid().asBareJid() + ": improperly formatted contents",
1030 Throwables.getRootCause(e));
1031 respondOk(jinglePacket);
1032 sendSessionTerminate(Reason.of(e), e.getMessage());
1033 return;
1034 }
1035 Log.d(
1036 Config.LOGTAG,
1037 "processing session-init with " + contentMap.contents.size() + " contents");
1038 final State target;
1039 if (this.state == State.PROCEED) {
1040 Preconditions.checkState(
1041 proposedMedia != null && proposedMedia.size() > 0,
1042 "proposed media must be set when processing pre-approved session-initiate");
1043 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1044 sendSessionTerminate(
1045 Reason.SECURITY_ERROR,
1046 String.format(
1047 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1048 this.proposedMedia, contentMap.getMedia()));
1049 return;
1050 }
1051 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1052 } else {
1053 target = State.SESSION_INITIALIZED;
1054 }
1055 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1056 respondOk(jinglePacket);
1057 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1058 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1059 Log.d(
1060 Config.LOGTAG,
1061 id.account.getJid().asBareJid()
1062 + ": automatically accepting session-initiate");
1063 sendSessionAccept();
1064 } else {
1065 Log.d(
1066 Config.LOGTAG,
1067 id.account.getJid().asBareJid()
1068 + ": received not pre-approved session-initiate. start ringing");
1069 startRinging();
1070 }
1071 } else {
1072 Log.d(
1073 Config.LOGTAG,
1074 String.format(
1075 "%s: received session-initiate while in state %s",
1076 id.account.getJid().asBareJid(), state));
1077 terminateWithOutOfOrder(jinglePacket);
1078 }
1079 }
1080
1081 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1082 if (!isInitiator()) {
1083 Log.d(
1084 Config.LOGTAG,
1085 String.format(
1086 "%s: received session-accept even though we were responding",
1087 id.account.getJid().asBareJid()));
1088 terminateWithOutOfOrder(jinglePacket);
1089 return;
1090 }
1091 final ListenableFuture<RtpContentMap> future =
1092 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1093 Futures.addCallback(
1094 future,
1095 new FutureCallback<RtpContentMap>() {
1096 @Override
1097 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1098 receiveSessionAccept(jinglePacket, rtpContentMap);
1099 }
1100
1101 @Override
1102 public void onFailure(@NonNull final Throwable throwable) {
1103 respondOk(jinglePacket);
1104 Log.d(
1105 Config.LOGTAG,
1106 id.account.getJid().asBareJid()
1107 + ": improperly formatted contents in session-accept",
1108 throwable);
1109 webRTCWrapper.close();
1110 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1111 }
1112 },
1113 MoreExecutors.directExecutor());
1114 }
1115
1116 private void receiveSessionAccept(
1117 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1118 try {
1119 contentMap.requireContentDescriptions();
1120 contentMap.requireDTLSFingerprint();
1121 } catch (final RuntimeException e) {
1122 respondOk(jinglePacket);
1123 Log.d(
1124 Config.LOGTAG,
1125 id.account.getJid().asBareJid()
1126 + ": improperly formatted contents in session-accept",
1127 e);
1128 webRTCWrapper.close();
1129 sendSessionTerminate(Reason.of(e), e.getMessage());
1130 return;
1131 }
1132 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1133 if (!initiatorMedia.equals(contentMap.getMedia())) {
1134 sendSessionTerminate(
1135 Reason.SECURITY_ERROR,
1136 String.format(
1137 "Your session-included included media %s but our session-initiate was %s",
1138 this.proposedMedia, contentMap.getMedia()));
1139 return;
1140 }
1141 Log.d(
1142 Config.LOGTAG,
1143 "processing session-accept with " + contentMap.contents.size() + " contents");
1144 if (transition(State.SESSION_ACCEPTED)) {
1145 respondOk(jinglePacket);
1146 receiveSessionAccept(contentMap);
1147 } else {
1148 Log.d(
1149 Config.LOGTAG,
1150 String.format(
1151 "%s: received session-accept while in state %s",
1152 id.account.getJid().asBareJid(), state));
1153 respondOk(jinglePacket);
1154 }
1155 }
1156
1157 private void receiveSessionAccept(final RtpContentMap contentMap) {
1158 this.responderRtpContentMap = contentMap;
1159 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1160 final SessionDescription sessionDescription;
1161 try {
1162 sessionDescription = SessionDescription.of(contentMap, false);
1163 } catch (final IllegalArgumentException | NullPointerException e) {
1164 Log.d(
1165 Config.LOGTAG,
1166 id.account.getJid().asBareJid()
1167 + ": unable convert offer from session-accept to SDP",
1168 e);
1169 webRTCWrapper.close();
1170 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1171 return;
1172 }
1173 final org.webrtc.SessionDescription answer =
1174 new org.webrtc.SessionDescription(
1175 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1176 try {
1177 this.webRTCWrapper.setRemoteDescription(answer).get();
1178 } catch (final Exception e) {
1179 Log.d(
1180 Config.LOGTAG,
1181 id.account.getJid().asBareJid()
1182 + ": unable to set remote description after receiving session-accept",
1183 Throwables.getRootCause(e));
1184 webRTCWrapper.close();
1185 sendSessionTerminate(
1186 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1187 return;
1188 }
1189 processCandidates(contentMap.contents.entrySet());
1190 }
1191
1192 private void sendSessionAccept() {
1193 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1194 if (rtpContentMap == null) {
1195 throw new IllegalStateException("initiator RTP Content Map has not been set");
1196 }
1197 final SessionDescription offer;
1198 try {
1199 offer = SessionDescription.of(rtpContentMap, true);
1200 } catch (final IllegalArgumentException | NullPointerException e) {
1201 Log.d(
1202 Config.LOGTAG,
1203 id.account.getJid().asBareJid()
1204 + ": unable convert offer from session-initiate to SDP",
1205 e);
1206 webRTCWrapper.close();
1207 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1208 return;
1209 }
1210 sendSessionAccept(rtpContentMap.getMedia(), offer);
1211 }
1212
1213 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1214 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1215 }
1216
1217 private synchronized void sendSessionAccept(
1218 final Set<Media> media,
1219 final SessionDescription offer,
1220 final List<PeerConnection.IceServer> iceServers) {
1221 if (isTerminated()) {
1222 Log.w(
1223 Config.LOGTAG,
1224 id.account.getJid().asBareJid()
1225 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1226 return;
1227 }
1228 try {
1229 setupWebRTC(media, iceServers);
1230 } catch (final WebRTCWrapper.InitializationException e) {
1231 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1232 webRTCWrapper.close();
1233 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1234 return;
1235 }
1236 final org.webrtc.SessionDescription sdp =
1237 new org.webrtc.SessionDescription(
1238 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1239 try {
1240 this.webRTCWrapper.setRemoteDescription(sdp).get();
1241 addIceCandidatesFromBlackLog();
1242 org.webrtc.SessionDescription webRTCSessionDescription =
1243 this.webRTCWrapper.setLocalDescription().get();
1244 prepareSessionAccept(webRTCSessionDescription);
1245 } catch (final Exception e) {
1246 failureToAcceptSession(e);
1247 }
1248 }
1249
1250 private void failureToAcceptSession(final Throwable throwable) {
1251 if (isTerminated()) {
1252 return;
1253 }
1254 final Throwable rootCause = Throwables.getRootCause(throwable);
1255 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1256 webRTCWrapper.close();
1257 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1258 }
1259
1260 private void addIceCandidatesFromBlackLog() {
1261 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1262 while ((foo = this.pendingIceCandidates.poll()) != null) {
1263 processCandidate(foo);
1264 Log.d(
1265 Config.LOGTAG,
1266 id.account.getJid().asBareJid() + ": added candidate from back log");
1267 }
1268 }
1269
1270 private void prepareSessionAccept(
1271 final org.webrtc.SessionDescription webRTCSessionDescription) {
1272 final SessionDescription sessionDescription =
1273 SessionDescription.parse(webRTCSessionDescription.description);
1274 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1275 this.responderRtpContentMap = respondingRtpContentMap;
1276 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1277 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1278 prepareOutgoingContentMap(respondingRtpContentMap);
1279 Futures.addCallback(
1280 outgoingContentMapFuture,
1281 new FutureCallback<RtpContentMap>() {
1282 @Override
1283 public void onSuccess(final RtpContentMap outgoingContentMap) {
1284 sendSessionAccept(outgoingContentMap);
1285 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1286 }
1287
1288 @Override
1289 public void onFailure(@NonNull Throwable throwable) {
1290 failureToAcceptSession(throwable);
1291 }
1292 },
1293 MoreExecutors.directExecutor());
1294 }
1295
1296 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1297 if (isTerminated()) {
1298 Log.w(
1299 Config.LOGTAG,
1300 id.account.getJid().asBareJid()
1301 + ": preparing session accept was too slow. already terminated. nothing to do.");
1302 return;
1303 }
1304 transitionOrThrow(State.SESSION_ACCEPTED);
1305 final JinglePacket sessionAccept =
1306 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1307 send(sessionAccept);
1308 }
1309
1310 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1311 final RtpContentMap rtpContentMap) {
1312 if (this.omemoVerification.hasDeviceId()) {
1313 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1314 verifiedPayloadFuture =
1315 id.account
1316 .getAxolotlService()
1317 .encrypt(
1318 rtpContentMap,
1319 id.with,
1320 omemoVerification.getDeviceId());
1321 return Futures.transform(
1322 verifiedPayloadFuture,
1323 verifiedPayload -> {
1324 omemoVerification.setOrEnsureEqual(verifiedPayload);
1325 return verifiedPayload.getPayload();
1326 },
1327 MoreExecutors.directExecutor());
1328 } else {
1329 return Futures.immediateFuture(rtpContentMap);
1330 }
1331 }
1332
1333 synchronized void deliveryMessage(
1334 final Jid from,
1335 final Element message,
1336 final String serverMessageId,
1337 final long timestamp) {
1338 Log.d(
1339 Config.LOGTAG,
1340 id.account.getJid().asBareJid()
1341 + ": delivered message to JingleRtpConnection "
1342 + message);
1343 switch (message.getName()) {
1344 case "propose":
1345 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1346 break;
1347 case "proceed":
1348 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1349 break;
1350 case "retract":
1351 receiveRetract(from, serverMessageId, timestamp);
1352 break;
1353 case "reject":
1354 receiveReject(from, serverMessageId, timestamp);
1355 break;
1356 case "accept":
1357 receiveAccept(from, serverMessageId, timestamp);
1358 break;
1359 default:
1360 break;
1361 }
1362 }
1363
1364 void deliverFailedProceed(final String message) {
1365 Log.d(
1366 Config.LOGTAG,
1367 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1368 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1369 webRTCWrapper.close();
1370 Log.d(
1371 Config.LOGTAG,
1372 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1373 this.finish();
1374 }
1375 }
1376
1377 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1378 final boolean originatedFromMyself =
1379 from.asBareJid().equals(id.account.getJid().asBareJid());
1380 if (originatedFromMyself) {
1381 if (transition(State.ACCEPTED)) {
1382 acceptedOnOtherDevice(serverMsgId, timestamp);
1383 } else {
1384 Log.d(
1385 Config.LOGTAG,
1386 id.account.getJid().asBareJid()
1387 + ": unable to transition to accept because already in state="
1388 + this.state);
1389 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1390 }
1391 } else {
1392 Log.d(
1393 Config.LOGTAG,
1394 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1395 }
1396 }
1397
1398 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1399 if (serverMsgId != null) {
1400 this.message.setServerMsgId(serverMsgId);
1401 }
1402 this.message.setTime(timestamp);
1403 this.message.setCarbon(true); // indicate that call was accepted on other device
1404 this.writeLogMessageSuccess(0);
1405 this.xmppConnectionService
1406 .getNotificationService()
1407 .cancelIncomingCallNotification();
1408 this.finish();
1409 }
1410
1411 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1412 final boolean originatedFromMyself =
1413 from.asBareJid().equals(id.account.getJid().asBareJid());
1414 // reject from another one of my clients
1415 if (originatedFromMyself) {
1416 receiveRejectFromMyself(serverMsgId, timestamp);
1417 } else if (isInitiator()) {
1418 if (from.equals(id.with)) {
1419 receiveRejectFromResponder();
1420 } else {
1421 Log.d(
1422 Config.LOGTAG,
1423 id.account.getJid()
1424 + ": ignoring reject from "
1425 + from
1426 + " for session with "
1427 + id.with);
1428 }
1429 } else {
1430 Log.d(
1431 Config.LOGTAG,
1432 id.account.getJid()
1433 + ": ignoring reject from "
1434 + from
1435 + " for session with "
1436 + id.with);
1437 }
1438 }
1439
1440 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1441 if (transition(State.REJECTED)) {
1442 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1443 this.finish();
1444 if (serverMsgId != null) {
1445 this.message.setServerMsgId(serverMsgId);
1446 }
1447 this.message.setTime(timestamp);
1448 this.message.setCarbon(true); // indicate that call was rejected on other device
1449 writeLogMessageMissed();
1450 } else {
1451 Log.d(
1452 Config.LOGTAG,
1453 "not able to transition into REJECTED because already in " + this.state);
1454 }
1455 }
1456
1457 private void receiveRejectFromResponder() {
1458 if (isInState(State.PROCEED)) {
1459 Log.d(
1460 Config.LOGTAG,
1461 id.account.getJid()
1462 + ": received reject while still in proceed. callee reconsidered");
1463 closeTransitionLogFinish(State.REJECTED_RACED);
1464 return;
1465 }
1466 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1467 Log.d(
1468 Config.LOGTAG,
1469 id.account.getJid()
1470 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1471 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1472 return;
1473 }
1474 Log.d(
1475 Config.LOGTAG,
1476 id.account.getJid()
1477 + ": ignoring reject from responder because already in state "
1478 + this.state);
1479 }
1480
1481 private void receivePropose(
1482 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1483 final boolean originatedFromMyself =
1484 from.asBareJid().equals(id.account.getJid().asBareJid());
1485 if (originatedFromMyself) {
1486 Log.d(
1487 Config.LOGTAG,
1488 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1489 } else if (transition(
1490 State.PROPOSED,
1491 () -> {
1492 final Collection<RtpDescription> descriptions =
1493 Collections2.transform(
1494 Collections2.filter(
1495 propose.getDescriptions(),
1496 d -> d instanceof RtpDescription),
1497 input -> (RtpDescription) input);
1498 final Collection<Media> media =
1499 Collections2.transform(descriptions, RtpDescription::getMedia);
1500 Preconditions.checkState(
1501 !media.contains(Media.UNKNOWN),
1502 "RTP descriptions contain unknown media");
1503 Log.d(
1504 Config.LOGTAG,
1505 id.account.getJid().asBareJid()
1506 + ": received session proposal from "
1507 + from
1508 + " for "
1509 + media);
1510 this.proposedMedia = Sets.newHashSet(media);
1511 })) {
1512 if (serverMsgId != null) {
1513 this.message.setServerMsgId(serverMsgId);
1514 }
1515 this.message.setTime(timestamp);
1516 startRinging();
1517 } else {
1518 Log.d(
1519 Config.LOGTAG,
1520 id.account.getJid()
1521 + ": ignoring session proposal because already in "
1522 + state);
1523 }
1524 }
1525
1526 private void startRinging() {
1527 Log.d(
1528 Config.LOGTAG,
1529 id.account.getJid().asBareJid()
1530 + ": received call from "
1531 + id.with
1532 + ". start ringing");
1533 ringingTimeoutFuture =
1534 jingleConnectionManager.schedule(
1535 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1536 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1537 }
1538
1539 private synchronized void ringingTimeout() {
1540 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1541 switch (this.state) {
1542 case PROPOSED:
1543 message.markUnread();
1544 rejectCallFromProposed();
1545 break;
1546 case SESSION_INITIALIZED:
1547 message.markUnread();
1548 rejectCallFromSessionInitiate();
1549 break;
1550 }
1551 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1552 }
1553
1554 private void cancelRingingTimeout() {
1555 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1556 if (future != null && !future.isCancelled()) {
1557 future.cancel(false);
1558 }
1559 }
1560
1561 private void receiveProceed(
1562 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1563 final Set<Media> media =
1564 Preconditions.checkNotNull(
1565 this.proposedMedia, "Proposed media has to be set before handling proceed");
1566 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1567 if (from.equals(id.with)) {
1568 if (isInitiator()) {
1569 if (transition(State.PROCEED)) {
1570 if (serverMsgId != null) {
1571 this.message.setServerMsgId(serverMsgId);
1572 }
1573 this.message.setTime(timestamp);
1574 final Integer remoteDeviceId = proceed.getDeviceId();
1575 if (isOmemoEnabled()) {
1576 this.omemoVerification.setDeviceId(remoteDeviceId);
1577 } else {
1578 if (remoteDeviceId != null) {
1579 Log.d(
1580 Config.LOGTAG,
1581 id.account.getJid().asBareJid()
1582 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1583 }
1584 this.omemoVerification.setDeviceId(null);
1585 }
1586 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1587 } else {
1588 Log.d(
1589 Config.LOGTAG,
1590 String.format(
1591 "%s: ignoring proceed because already in %s",
1592 id.account.getJid().asBareJid(), this.state));
1593 }
1594 } else {
1595 Log.d(
1596 Config.LOGTAG,
1597 String.format(
1598 "%s: ignoring proceed because we were not initializing",
1599 id.account.getJid().asBareJid()));
1600 }
1601 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1602 if (transition(State.ACCEPTED)) {
1603 Log.d(
1604 Config.LOGTAG,
1605 id.account.getJid().asBareJid()
1606 + ": moved session with "
1607 + id.with
1608 + " into state accepted after received carbon copied proceed");
1609 acceptedOnOtherDevice(serverMsgId, timestamp);
1610 }
1611 } else {
1612 Log.d(
1613 Config.LOGTAG,
1614 String.format(
1615 "%s: ignoring proceed from %s. was expected from %s",
1616 id.account.getJid().asBareJid(), from, id.with));
1617 }
1618 }
1619
1620 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1621 if (from.equals(id.with)) {
1622 final State target =
1623 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1624 if (transition(target)) {
1625 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1626 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1627 Log.d(
1628 Config.LOGTAG,
1629 id.account.getJid().asBareJid()
1630 + ": session with "
1631 + id.with
1632 + " has been retracted (serverMsgId="
1633 + serverMsgId
1634 + ")");
1635 if (serverMsgId != null) {
1636 this.message.setServerMsgId(serverMsgId);
1637 }
1638 this.message.setTime(timestamp);
1639 if (target == State.RETRACTED) {
1640 this.message.markUnread();
1641 }
1642 writeLogMessageMissed();
1643 finish();
1644 } else {
1645 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1646 }
1647 } else {
1648 // TODO parse retract from self
1649 Log.d(
1650 Config.LOGTAG,
1651 id.account.getJid().asBareJid()
1652 + ": received retract from "
1653 + from
1654 + ". expected retract from"
1655 + id.with
1656 + ". ignoring");
1657 }
1658 }
1659
1660 public void sendSessionInitiate() {
1661 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1662 }
1663
1664 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1665 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1666 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1667 }
1668
1669 private synchronized void sendSessionInitiate(
1670 final Set<Media> media,
1671 final State targetState,
1672 final List<PeerConnection.IceServer> iceServers) {
1673 if (isTerminated()) {
1674 Log.w(
1675 Config.LOGTAG,
1676 id.account.getJid().asBareJid()
1677 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1678 return;
1679 }
1680 try {
1681 setupWebRTC(media, iceServers);
1682 } catch (final WebRTCWrapper.InitializationException e) {
1683 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1684 webRTCWrapper.close();
1685 sendRetract(Reason.ofThrowable(e));
1686 return;
1687 }
1688 try {
1689 org.webrtc.SessionDescription webRTCSessionDescription =
1690 this.webRTCWrapper.setLocalDescription().get();
1691 prepareSessionInitiate(webRTCSessionDescription, targetState);
1692 } catch (final Exception e) {
1693 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1694 // exceptions
1695 failureToInitiateSession(e, targetState);
1696 }
1697 }
1698
1699 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1700 if (isTerminated()) {
1701 return;
1702 }
1703 Log.d(
1704 Config.LOGTAG,
1705 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1706 Throwables.getRootCause(throwable));
1707 webRTCWrapper.close();
1708 final Reason reason = Reason.ofThrowable(throwable);
1709 if (isInState(targetState)) {
1710 sendSessionTerminate(reason, throwable.getMessage());
1711 } else {
1712 sendRetract(reason);
1713 }
1714 }
1715
1716 private void sendRetract(final Reason reason) {
1717 // TODO embed reason into retract
1718 sendJingleMessage("retract", id.with.asBareJid());
1719 transitionOrThrow(reasonToState(reason));
1720 this.finish();
1721 }
1722
1723 private void prepareSessionInitiate(
1724 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1725 final SessionDescription sessionDescription =
1726 SessionDescription.parse(webRTCSessionDescription.description);
1727 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1728 this.initiatorRtpContentMap = rtpContentMap;
1729 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1730 encryptSessionInitiate(rtpContentMap);
1731 Futures.addCallback(
1732 outgoingContentMapFuture,
1733 new FutureCallback<RtpContentMap>() {
1734 @Override
1735 public void onSuccess(final RtpContentMap outgoingContentMap) {
1736 sendSessionInitiate(outgoingContentMap, targetState);
1737 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1738 }
1739
1740 @Override
1741 public void onFailure(@NonNull final Throwable throwable) {
1742 failureToInitiateSession(throwable, targetState);
1743 }
1744 },
1745 MoreExecutors.directExecutor());
1746 }
1747
1748 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1749 if (isTerminated()) {
1750 Log.w(
1751 Config.LOGTAG,
1752 id.account.getJid().asBareJid()
1753 + ": preparing session was too slow. already terminated. nothing to do.");
1754 return;
1755 }
1756 this.transitionOrThrow(targetState);
1757 final JinglePacket sessionInitiate =
1758 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1759 send(sessionInitiate);
1760 }
1761
1762 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1763 final RtpContentMap rtpContentMap) {
1764 if (this.omemoVerification.hasDeviceId()) {
1765 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1766 verifiedPayloadFuture =
1767 id.account
1768 .getAxolotlService()
1769 .encrypt(
1770 rtpContentMap,
1771 id.with,
1772 omemoVerification.getDeviceId());
1773 final ListenableFuture<RtpContentMap> future =
1774 Futures.transform(
1775 verifiedPayloadFuture,
1776 verifiedPayload -> {
1777 omemoVerification.setSessionFingerprint(
1778 verifiedPayload.getFingerprint());
1779 return verifiedPayload.getPayload();
1780 },
1781 MoreExecutors.directExecutor());
1782 if (Config.REQUIRE_RTP_VERIFICATION) {
1783 return future;
1784 }
1785 return Futures.catching(
1786 future,
1787 CryptoFailedException.class,
1788 e -> {
1789 Log.w(
1790 Config.LOGTAG,
1791 id.account.getJid().asBareJid()
1792 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1793 e);
1794 return rtpContentMap;
1795 },
1796 MoreExecutors.directExecutor());
1797 } else {
1798 return Futures.immediateFuture(rtpContentMap);
1799 }
1800 }
1801
1802 private void sendSessionTerminate(final Reason reason) {
1803 sendSessionTerminate(reason, null);
1804 }
1805
1806 private void sendSessionTerminate(final Reason reason, final String text) {
1807 final State previous = this.state;
1808 final State target = reasonToState(reason);
1809 transitionOrThrow(target);
1810 if (previous != State.NULL) {
1811 writeLogMessage(target);
1812 }
1813 final JinglePacket jinglePacket =
1814 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1815 jinglePacket.setReason(reason, text);
1816 Log.d(Config.LOGTAG, jinglePacket.toString());
1817 send(jinglePacket);
1818 finish();
1819 }
1820
1821 private void sendTransportInfo(
1822 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1823 final RtpContentMap transportInfo;
1824 try {
1825 final RtpContentMap rtpContentMap =
1826 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1827 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1828 } catch (final Exception e) {
1829 Log.d(
1830 Config.LOGTAG,
1831 id.account.getJid().asBareJid()
1832 + ": unable to prepare transport-info from candidate for content="
1833 + contentName);
1834 return;
1835 }
1836 final JinglePacket jinglePacket =
1837 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1838 send(jinglePacket);
1839 }
1840
1841 private void send(final JinglePacket jinglePacket) {
1842 jinglePacket.setTo(id.with);
1843 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1844 }
1845
1846 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1847 if (response.getType() == IqPacket.TYPE.ERROR) {
1848 handleIqErrorResponse(response);
1849 return;
1850 }
1851 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1852 handleIqTimeoutResponse(response);
1853 }
1854 }
1855
1856 private void handleIqErrorResponse(final IqPacket response) {
1857 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1858 final String errorCondition = response.getErrorCondition();
1859 Log.d(
1860 Config.LOGTAG,
1861 id.account.getJid().asBareJid()
1862 + ": received IQ-error from "
1863 + response.getFrom()
1864 + " in RTP session. "
1865 + errorCondition);
1866 if (isTerminated()) {
1867 Log.i(
1868 Config.LOGTAG,
1869 id.account.getJid().asBareJid()
1870 + ": ignoring error because session was already terminated");
1871 return;
1872 }
1873 this.webRTCWrapper.close();
1874 final State target;
1875 if (Arrays.asList(
1876 "service-unavailable",
1877 "recipient-unavailable",
1878 "remote-server-not-found",
1879 "remote-server-timeout")
1880 .contains(errorCondition)) {
1881 target = State.TERMINATED_CONNECTIVITY_ERROR;
1882 } else {
1883 target = State.TERMINATED_APPLICATION_FAILURE;
1884 }
1885 transitionOrThrow(target);
1886 this.finish();
1887 }
1888
1889 private void handleIqTimeoutResponse(final IqPacket response) {
1890 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1891 Log.d(
1892 Config.LOGTAG,
1893 id.account.getJid().asBareJid()
1894 + ": received IQ timeout in RTP session with "
1895 + id.with
1896 + ". terminating with connectivity error");
1897 if (isTerminated()) {
1898 Log.i(
1899 Config.LOGTAG,
1900 id.account.getJid().asBareJid()
1901 + ": ignoring error because session was already terminated");
1902 return;
1903 }
1904 this.webRTCWrapper.close();
1905 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
1906 this.finish();
1907 }
1908
1909 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1910 Log.d(
1911 Config.LOGTAG,
1912 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1913 this.webRTCWrapper.close();
1914 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1915 respondWithOutOfOrder(jinglePacket);
1916 this.finish();
1917 }
1918
1919 private void respondWithTieBreak(final JinglePacket jinglePacket) {
1920 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
1921 }
1922
1923 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1924 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
1925 }
1926
1927 void respondWithJingleError(
1928 final IqPacket original,
1929 String jingleCondition,
1930 String condition,
1931 String conditionType) {
1932 jingleConnectionManager.respondWithJingleError(
1933 id.account, original, jingleCondition, condition, conditionType);
1934 }
1935
1936 private void respondOk(final JinglePacket jinglePacket) {
1937 xmppConnectionService.sendIqPacket(
1938 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1939 }
1940
1941 public RtpEndUserState getEndUserState() {
1942 switch (this.state) {
1943 case NULL:
1944 case PROPOSED:
1945 case SESSION_INITIALIZED:
1946 if (isInitiator()) {
1947 return RtpEndUserState.RINGING;
1948 } else {
1949 return RtpEndUserState.INCOMING_CALL;
1950 }
1951 case PROCEED:
1952 if (isInitiator()) {
1953 return RtpEndUserState.RINGING;
1954 } else {
1955 return RtpEndUserState.ACCEPTING_CALL;
1956 }
1957 case SESSION_INITIALIZED_PRE_APPROVED:
1958 if (isInitiator()) {
1959 return RtpEndUserState.RINGING;
1960 } else {
1961 return RtpEndUserState.CONNECTING;
1962 }
1963 case SESSION_ACCEPTED:
1964 final ContentAddition ca = getPendingContentAddition();
1965 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
1966 return RtpEndUserState.INCOMING_CONTENT_ADD;
1967 }
1968 return getPeerConnectionStateAsEndUserState();
1969 case REJECTED:
1970 case REJECTED_RACED:
1971 case TERMINATED_DECLINED_OR_BUSY:
1972 if (isInitiator()) {
1973 return RtpEndUserState.DECLINED_OR_BUSY;
1974 } else {
1975 return RtpEndUserState.ENDED;
1976 }
1977 case TERMINATED_SUCCESS:
1978 case ACCEPTED:
1979 case RETRACTED:
1980 case TERMINATED_CANCEL_OR_TIMEOUT:
1981 return RtpEndUserState.ENDED;
1982 case RETRACTED_RACED:
1983 if (isInitiator()) {
1984 return RtpEndUserState.ENDED;
1985 } else {
1986 return RtpEndUserState.RETRACTED;
1987 }
1988 case TERMINATED_CONNECTIVITY_ERROR:
1989 return zeroDuration()
1990 ? RtpEndUserState.CONNECTIVITY_ERROR
1991 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1992 case TERMINATED_APPLICATION_FAILURE:
1993 return RtpEndUserState.APPLICATION_ERROR;
1994 case TERMINATED_SECURITY_ERROR:
1995 return RtpEndUserState.SECURITY_ERROR;
1996 }
1997 throw new IllegalStateException(
1998 String.format("%s has no equivalent EndUserState", this.state));
1999 }
2000
2001 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2002 final PeerConnection.PeerConnectionState state;
2003 try {
2004 state = webRTCWrapper.getState();
2005 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2006 // We usually close the WebRTCWrapper *before* transitioning so we might still
2007 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2008 return RtpEndUserState.ENDING_CALL;
2009 }
2010 switch (state) {
2011 case CONNECTED:
2012 return RtpEndUserState.CONNECTED;
2013 case NEW:
2014 case CONNECTING:
2015 return RtpEndUserState.CONNECTING;
2016 case CLOSED:
2017 return RtpEndUserState.ENDING_CALL;
2018 default:
2019 return zeroDuration()
2020 ? RtpEndUserState.CONNECTIVITY_ERROR
2021 : RtpEndUserState.RECONNECTING;
2022 }
2023 }
2024
2025 public ContentAddition getPendingContentAddition() {
2026 final RtpContentMap in = this.incomingContentAdd;
2027 final RtpContentMap out = this.outgoingContentAdd;
2028 if (out != null) {
2029 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2030 } else if (in != null) {
2031 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2032 } else {
2033 return null;
2034 }
2035 }
2036
2037 public Set<Media> getMedia() {
2038 final State current = getState();
2039 if (current == State.NULL) {
2040 if (isInitiator()) {
2041 return Preconditions.checkNotNull(
2042 this.proposedMedia, "RTP connection has not been initialized properly");
2043 }
2044 throw new IllegalStateException("RTP connection has not been initialized yet");
2045 }
2046 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2047 return Preconditions.checkNotNull(
2048 this.proposedMedia, "RTP connection has not been initialized properly");
2049 }
2050 final RtpContentMap localContentMap = getLocalContentMap();
2051 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2052 if (localContentMap != null) {
2053 return localContentMap.getMedia();
2054 } else if (initiatorContentMap != null) {
2055 return initiatorContentMap.getMedia();
2056 } else if (isTerminated()) {
2057 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2058 } else {
2059 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2060 }
2061 }
2062
2063 public boolean isVerified() {
2064 final String fingerprint = this.omemoVerification.getFingerprint();
2065 if (fingerprint == null) {
2066 return false;
2067 }
2068 final FingerprintStatus status =
2069 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2070 return status != null && status.isVerified();
2071 }
2072
2073 public boolean addMedia(final Media media) {
2074 final Set<Media> currentMedia = getMedia();
2075 if (currentMedia.contains(media)) {
2076 throw new IllegalStateException(String.format("%s has already been proposed", media));
2077 }
2078 // TODO add state protection - can only add while ACCEPTED or so
2079 Log.d(Config.LOGTAG,"adding media: "+media);
2080 return webRTCWrapper.addTrack(media);
2081 }
2082
2083 public synchronized void acceptCall() {
2084 switch (this.state) {
2085 case PROPOSED:
2086 cancelRingingTimeout();
2087 acceptCallFromProposed();
2088 break;
2089 case SESSION_INITIALIZED:
2090 cancelRingingTimeout();
2091 acceptCallFromSessionInitialized();
2092 break;
2093 case ACCEPTED:
2094 Log.w(
2095 Config.LOGTAG,
2096 id.account.getJid().asBareJid()
2097 + ": the call has already been accepted with another client. UI was just lagging behind");
2098 break;
2099 case PROCEED:
2100 case SESSION_ACCEPTED:
2101 Log.w(
2102 Config.LOGTAG,
2103 id.account.getJid().asBareJid()
2104 + ": the call has already been accepted. user probably double tapped the UI");
2105 break;
2106 default:
2107 throw new IllegalStateException("Can not accept call from " + this.state);
2108 }
2109 }
2110
2111 public void notifyPhoneCall() {
2112 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2113 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2114 rejectCall();
2115 } else {
2116 endCall();
2117 }
2118 }
2119
2120 public synchronized void rejectCall() {
2121 if (isTerminated()) {
2122 Log.w(
2123 Config.LOGTAG,
2124 id.account.getJid().asBareJid()
2125 + ": received rejectCall() when session has already been terminated. nothing to do");
2126 return;
2127 }
2128 switch (this.state) {
2129 case PROPOSED:
2130 rejectCallFromProposed();
2131 break;
2132 case SESSION_INITIALIZED:
2133 rejectCallFromSessionInitiate();
2134 break;
2135 default:
2136 throw new IllegalStateException("Can not reject call from " + this.state);
2137 }
2138 }
2139
2140 public synchronized void endCall() {
2141 if (isTerminated()) {
2142 Log.w(
2143 Config.LOGTAG,
2144 id.account.getJid().asBareJid()
2145 + ": received endCall() when session has already been terminated. nothing to do");
2146 return;
2147 }
2148 if (isInState(State.PROPOSED) && !isInitiator()) {
2149 rejectCallFromProposed();
2150 return;
2151 }
2152 if (isInState(State.PROCEED)) {
2153 if (isInitiator()) {
2154 retractFromProceed();
2155 } else {
2156 rejectCallFromProceed();
2157 }
2158 return;
2159 }
2160 if (isInitiator()
2161 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2162 this.webRTCWrapper.close();
2163 sendSessionTerminate(Reason.CANCEL);
2164 return;
2165 }
2166 if (isInState(State.SESSION_INITIALIZED)) {
2167 rejectCallFromSessionInitiate();
2168 return;
2169 }
2170 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2171 this.webRTCWrapper.close();
2172 sendSessionTerminate(Reason.SUCCESS);
2173 return;
2174 }
2175 if (isInState(
2176 State.TERMINATED_APPLICATION_FAILURE,
2177 State.TERMINATED_CONNECTIVITY_ERROR,
2178 State.TERMINATED_DECLINED_OR_BUSY)) {
2179 Log.d(
2180 Config.LOGTAG,
2181 "ignoring request to end call because already in state " + this.state);
2182 return;
2183 }
2184 throw new IllegalStateException(
2185 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2186 }
2187
2188 private void retractFromProceed() {
2189 Log.d(Config.LOGTAG, "retract from proceed");
2190 this.sendJingleMessage("retract");
2191 closeTransitionLogFinish(State.RETRACTED_RACED);
2192 }
2193
2194 private void closeTransitionLogFinish(final State state) {
2195 this.webRTCWrapper.close();
2196 transitionOrThrow(state);
2197 writeLogMessage(state);
2198 finish();
2199 }
2200
2201 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2202 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2203 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2204 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2205 }
2206
2207 private void acceptCallFromProposed() {
2208 transitionOrThrow(State.PROCEED);
2209 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2210 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2211 this.sendJingleMessage("proceed");
2212 }
2213
2214 private void rejectCallFromProposed() {
2215 transitionOrThrow(State.REJECTED);
2216 writeLogMessageMissed();
2217 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2218 this.sendJingleMessage("reject");
2219 finish();
2220 }
2221
2222 private void rejectCallFromProceed() {
2223 this.sendJingleMessage("reject");
2224 closeTransitionLogFinish(State.REJECTED_RACED);
2225 }
2226
2227 private void rejectCallFromSessionInitiate() {
2228 webRTCWrapper.close();
2229 sendSessionTerminate(Reason.DECLINE);
2230 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2231 }
2232
2233 private void sendJingleMessage(final String action) {
2234 sendJingleMessage(action, id.with);
2235 }
2236
2237 private void sendJingleMessage(final String action, final Jid to) {
2238 final MessagePacket messagePacket = new MessagePacket();
2239 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2240 messagePacket.setTo(to);
2241 final Element intent =
2242 messagePacket
2243 .addChild(action, Namespace.JINGLE_MESSAGE)
2244 .setAttribute("id", id.sessionId);
2245 if ("proceed".equals(action)) {
2246 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2247 if (isOmemoEnabled()) {
2248 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2249 final Element device =
2250 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2251 device.setAttribute("id", deviceId);
2252 }
2253 }
2254 messagePacket.addChild("store", "urn:xmpp:hints");
2255 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2256 }
2257
2258 private boolean isOmemoEnabled() {
2259 final Conversational conversational = message.getConversation();
2260 if (conversational instanceof Conversation) {
2261 return ((Conversation) conversational).getNextEncryption()
2262 == Message.ENCRYPTION_AXOLOTL;
2263 }
2264 return false;
2265 }
2266
2267 private void acceptCallFromSessionInitialized() {
2268 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2269 sendSessionAccept();
2270 }
2271
2272 private synchronized boolean isInState(State... state) {
2273 return Arrays.asList(state).contains(this.state);
2274 }
2275
2276 private boolean transition(final State target) {
2277 return transition(target, null);
2278 }
2279
2280 private synchronized boolean transition(final State target, final Runnable runnable) {
2281 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2282 if (validTransitions != null && validTransitions.contains(target)) {
2283 this.state = target;
2284 if (runnable != null) {
2285 runnable.run();
2286 }
2287 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2288 updateEndUserState();
2289 updateOngoingCallNotification();
2290 return true;
2291 } else {
2292 return false;
2293 }
2294 }
2295
2296 void transitionOrThrow(final State target) {
2297 if (!transition(target)) {
2298 throw new IllegalStateException(
2299 String.format("Unable to transition from %s to %s", this.state, target));
2300 }
2301 }
2302
2303 @Override
2304 public void onIceCandidate(final IceCandidate iceCandidate) {
2305 final RtpContentMap rtpContentMap =
2306 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2307 final IceUdpTransportInfo.Credentials credentials;
2308 try {
2309 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2310 } catch (final IllegalArgumentException e) {
2311 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2312 return;
2313 }
2314 final String uFrag = credentials.ufrag;
2315 final IceUdpTransportInfo.Candidate candidate =
2316 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2317 if (candidate == null) {
2318 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2319 return;
2320 }
2321 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2322 sendTransportInfo(iceCandidate.sdpMid, candidate);
2323 }
2324
2325 @Override
2326 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2327 Log.d(
2328 Config.LOGTAG,
2329 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2330 this.stateHistory.add(newState);
2331 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2332 this.sessionDuration.start();
2333 updateOngoingCallNotification();
2334 } else if (this.sessionDuration.isRunning()) {
2335 this.sessionDuration.stop();
2336 updateOngoingCallNotification();
2337 }
2338
2339 final boolean neverConnected =
2340 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2341
2342 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2343 if (neverConnected) {
2344 if (isTerminated()) {
2345 Log.d(
2346 Config.LOGTAG,
2347 id.account.getJid().asBareJid()
2348 + ": not sending session-terminate after connectivity error because session is already in state "
2349 + this.state);
2350 return;
2351 }
2352 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2353 return;
2354 } else {
2355 this.restartIce();
2356 }
2357 }
2358 updateEndUserState();
2359 }
2360
2361 private void restartIce() {
2362 this.stateHistory.clear();
2363 this.webRTCWrapper.restartIceAsync();
2364 }
2365
2366 @Override
2367 public void onRenegotiationNeeded() {
2368 this.webRTCWrapper.execute(this::renegotiate);
2369 }
2370
2371 private void renegotiate() {
2372 final SessionDescription sessionDescription;
2373 try {
2374 sessionDescription = setLocalSessionDescription();
2375 } catch (final Exception e) {
2376 final Throwable cause = Throwables.getRootCause(e);
2377 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2378 webRTCWrapper.close();
2379 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2380 return;
2381 }
2382 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2383 final RtpContentMap currentContentMap = getLocalContentMap();
2384 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2385 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2386
2387 Log.d(
2388 Config.LOGTAG,
2389 id.getAccount().getJid().asBareJid()
2390 + ": renegotiate. iceRestart="
2391 + iceRestart
2392 + " content id diff="
2393 + diff);
2394
2395 if (diff.hasModifications() && iceRestart) {
2396 webRTCWrapper.close();
2397 sendSessionTerminate(
2398 Reason.FAILED_APPLICATION,
2399 "WebRTC unexpectedly tried to modify content and transport at once");
2400 return;
2401 }
2402
2403 if (iceRestart) {
2404 initiateIceRestart(rtpContentMap);
2405 return;
2406 } else if (diff.isEmpty()) {
2407 Log.d(
2408 Config.LOGTAG,
2409 "renegotiation. nothing to do. SignalingState="
2410 + this.webRTCWrapper.getSignalingState());
2411 }
2412
2413 if (diff.added.size() > 0) {
2414 modifyLocalContentMap(rtpContentMap);
2415 sendContentAdd(rtpContentMap, diff.added);
2416 }
2417 }
2418
2419 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2420 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2421 final JinglePacket jinglePacket =
2422 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2423 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2424 jinglePacket.setTo(id.with);
2425 xmppConnectionService.sendIqPacket(
2426 id.account,
2427 jinglePacket,
2428 (account, response) -> {
2429 if (response.getType() == IqPacket.TYPE.RESULT) {
2430 Log.d(Config.LOGTAG, "received success to our ice restart");
2431 setLocalContentMap(rtpContentMap);
2432 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2433 return;
2434 }
2435 if (response.getType() == IqPacket.TYPE.ERROR) {
2436 if (isTieBreak(response)) {
2437 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2438 return;
2439 }
2440 handleIqErrorResponse(response);
2441 }
2442 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2443 handleIqTimeoutResponse(response);
2444 }
2445 });
2446 }
2447
2448 private boolean isTieBreak(final IqPacket response) {
2449 final Element error = response.findChild("error");
2450 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2451 }
2452
2453 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2454 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2455 this.outgoingContentAdd = contentAdd;
2456 final JinglePacket jinglePacket =
2457 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2458 jinglePacket.setTo(id.with);
2459 xmppConnectionService.sendIqPacket(
2460 id.account,
2461 jinglePacket,
2462 (connection, response) -> {
2463 if (response.getType() == IqPacket.TYPE.RESULT) {
2464 Log.d(
2465 Config.LOGTAG,
2466 id.getAccount().getJid().asBareJid()
2467 + ": received ACK to our content-add");
2468 return;
2469 }
2470 if (response.getType() == IqPacket.TYPE.ERROR) {
2471 if (isTieBreak(response)) {
2472 this.outgoingContentAdd = null;
2473 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2474 return;
2475 }
2476 handleIqErrorResponse(response);
2477 }
2478 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2479 handleIqTimeoutResponse(response);
2480 }
2481 });
2482 }
2483
2484 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2485 if (isInitiator()) {
2486 this.initiatorRtpContentMap = rtpContentMap;
2487 } else {
2488 this.responderRtpContentMap = rtpContentMap;
2489 }
2490 }
2491
2492 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2493 if (isInitiator()) {
2494 this.responderRtpContentMap = rtpContentMap;
2495 } else {
2496 this.initiatorRtpContentMap = rtpContentMap;
2497 }
2498 }
2499
2500 // this method is to be used for content map modifications that modify media
2501 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2502 final RtpContentMap activeContents = rtpContentMap.activeContents();
2503 setLocalContentMap(activeContents);
2504 this.webRTCWrapper.switchSpeakerPhonePreference(
2505 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2506 updateEndUserState();
2507 }
2508
2509 private SessionDescription setLocalSessionDescription()
2510 throws ExecutionException, InterruptedException {
2511 final org.webrtc.SessionDescription sessionDescription =
2512 this.webRTCWrapper.setLocalDescription().get();
2513 return SessionDescription.parse(sessionDescription.description);
2514 }
2515
2516 private void closeWebRTCSessionAfterFailedConnection() {
2517 this.webRTCWrapper.close();
2518 synchronized (this) {
2519 if (isTerminated()) {
2520 Log.d(
2521 Config.LOGTAG,
2522 id.account.getJid().asBareJid()
2523 + ": no need to send session-terminate after failed connection. Other party already did");
2524 return;
2525 }
2526 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2527 }
2528 }
2529
2530 public boolean zeroDuration() {
2531 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2532 }
2533
2534 public long getCallDuration() {
2535 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2536 }
2537
2538 public AppRTCAudioManager getAudioManager() {
2539 return webRTCWrapper.getAudioManager();
2540 }
2541
2542 public boolean isMicrophoneEnabled() {
2543 return webRTCWrapper.isMicrophoneEnabled();
2544 }
2545
2546 public boolean setMicrophoneEnabled(final boolean enabled) {
2547 return webRTCWrapper.setMicrophoneEnabled(enabled);
2548 }
2549
2550 public boolean isVideoEnabled() {
2551 return webRTCWrapper.isVideoEnabled();
2552 }
2553
2554 public void setVideoEnabled(final boolean enabled) {
2555 webRTCWrapper.setVideoEnabled(enabled);
2556 }
2557
2558 public boolean isCameraSwitchable() {
2559 return webRTCWrapper.isCameraSwitchable();
2560 }
2561
2562 public boolean isFrontCamera() {
2563 return webRTCWrapper.isFrontCamera();
2564 }
2565
2566 public ListenableFuture<Boolean> switchCamera() {
2567 return webRTCWrapper.switchCamera();
2568 }
2569
2570 @Override
2571 public void onAudioDeviceChanged(
2572 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2573 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2574 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2575 selectedAudioDevice, availableAudioDevices);
2576 }
2577
2578 private void updateEndUserState() {
2579 final RtpEndUserState endUserState = getEndUserState();
2580 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2581 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2582 id.account, id.with, id.sessionId, endUserState);
2583 }
2584
2585 private void updateOngoingCallNotification() {
2586 final State state = this.state;
2587 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2588 final boolean reconnecting;
2589 if (state == State.SESSION_ACCEPTED) {
2590 reconnecting =
2591 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2592 } else {
2593 reconnecting = false;
2594 }
2595 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2596 } else {
2597 xmppConnectionService.removeOngoingCall();
2598 }
2599 }
2600
2601 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2602 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2603 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2604 request.setTo(id.account.getDomain());
2605 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2606 xmppConnectionService.sendIqPacket(
2607 id.account,
2608 request,
2609 (account, response) -> {
2610 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2611 new ImmutableList.Builder<>();
2612 if (response.getType() == IqPacket.TYPE.RESULT) {
2613 final Element services =
2614 response.findChild(
2615 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2616 final List<Element> children =
2617 services == null
2618 ? Collections.emptyList()
2619 : services.getChildren();
2620 for (final Element child : children) {
2621 if ("service".equals(child.getName())) {
2622 final String type = child.getAttribute("type");
2623 final String host = child.getAttribute("host");
2624 final String sport = child.getAttribute("port");
2625 final Integer port =
2626 sport == null ? null : Ints.tryParse(sport);
2627 final String transport = child.getAttribute("transport");
2628 final String username = child.getAttribute("username");
2629 final String password = child.getAttribute("password");
2630 if (Strings.isNullOrEmpty(host) || port == null) {
2631 continue;
2632 }
2633 if (port < 0 || port > 65535) {
2634 continue;
2635 }
2636
2637
2638
2639
2640 if (Arrays.asList("stun", "stuns", "turn", "turns")
2641 .contains(type)
2642 && Arrays.asList("udp", "tcp").contains(transport)) {
2643 if (Arrays.asList("stuns", "turns").contains(type)
2644 && "udp".equals(transport)) {
2645 Log.d(
2646 Config.LOGTAG,
2647 id.account.getJid().asBareJid()
2648 + ": skipping invalid combination of udp/tls in external services");
2649 continue;
2650 }
2651
2652 // STUN URLs do not support a query section since M110
2653 final String uri;
2654 if (Arrays.asList("stun","stuns").contains(type)) {
2655 uri = String.format("%s:%s%s", type, IP.wrapIPv6(host),port);
2656 } else {
2657 uri = String.format(
2658 "%s:%s:%s?transport=%s",
2659 type,
2660 IP.wrapIPv6(host),
2661 port,
2662 transport);
2663 }
2664
2665 final PeerConnection.IceServer.Builder iceServerBuilder =
2666 PeerConnection.IceServer.builder(uri);
2667 iceServerBuilder.setTlsCertPolicy(
2668 PeerConnection.TlsCertPolicy
2669 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2670 if (username != null && password != null) {
2671 iceServerBuilder.setUsername(username);
2672 iceServerBuilder.setPassword(password);
2673 } else if (Arrays.asList("turn", "turns").contains(type)) {
2674 // The WebRTC spec requires throwing an
2675 // InvalidAccessError when username (from libwebrtc
2676 // source coder)
2677 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2678 Log.d(
2679 Config.LOGTAG,
2680 id.account.getJid().asBareJid()
2681 + ": skipping "
2682 + type
2683 + "/"
2684 + transport
2685 + " without username and password");
2686 continue;
2687 }
2688 final PeerConnection.IceServer iceServer =
2689 iceServerBuilder.createIceServer();
2690 Log.d(
2691 Config.LOGTAG,
2692 id.account.getJid().asBareJid()
2693 + ": discovered ICE Server: "
2694 + iceServer);
2695 listBuilder.add(iceServer);
2696 }
2697 }
2698 }
2699 }
2700 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2701 if (iceServers.size() == 0) {
2702 Log.w(
2703 Config.LOGTAG,
2704 id.account.getJid().asBareJid()
2705 + ": no ICE server found "
2706 + response);
2707 }
2708 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2709 });
2710 } else {
2711 Log.w(
2712 Config.LOGTAG,
2713 id.account.getJid().asBareJid() + ": has no external service discovery");
2714 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2715 }
2716 }
2717
2718 private void finish() {
2719 if (isTerminated()) {
2720 this.cancelRingingTimeout();
2721 this.webRTCWrapper.verifyClosed();
2722 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2723 this.jingleConnectionManager.finishConnectionOrThrow(this);
2724 try {
2725 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2726 log.getParentFile().mkdirs();
2727 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2728 } catch (final IOException e) { }
2729 } else {
2730 throw new IllegalStateException(
2731 String.format("Unable to call finish from %s", this.state));
2732 }
2733 }
2734
2735 private void writeLogMessage(final State state) {
2736 final long duration = getCallDuration();
2737 if (state == State.TERMINATED_SUCCESS
2738 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2739 writeLogMessageSuccess(duration);
2740 } else {
2741 writeLogMessageMissed();
2742 }
2743 }
2744
2745 private void writeLogMessageSuccess(final long duration) {
2746 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2747 this.writeMessage();
2748 }
2749
2750 private void writeLogMessageMissed() {
2751 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2752 this.writeMessage();
2753 }
2754
2755 private void writeMessage() {
2756 final Conversational conversational = message.getConversation();
2757 if (conversational instanceof Conversation) {
2758 ((Conversation) conversational).add(this.message);
2759 xmppConnectionService.createMessageAsync(message);
2760 xmppConnectionService.updateConversationUi();
2761 } else {
2762 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2763 }
2764 }
2765
2766 public State getState() {
2767 return this.state;
2768 }
2769
2770 boolean isTerminated() {
2771 return TERMINATED.contains(this.state);
2772 }
2773
2774 public Optional<VideoTrack> getLocalVideoTrack() {
2775 return webRTCWrapper.getLocalVideoTrack();
2776 }
2777
2778 public Optional<VideoTrack> getRemoteVideoTrack() {
2779 return webRTCWrapper.getRemoteVideoTrack();
2780 }
2781
2782 public EglBase.Context getEglBaseContext() {
2783 return webRTCWrapper.getEglBaseContext();
2784 }
2785
2786 void setProposedMedia(final Set<Media> media) {
2787 this.proposedMedia = media;
2788 }
2789
2790 public void fireStateUpdate() {
2791 final RtpEndUserState endUserState = getEndUserState();
2792 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2793 id.account, id.with, id.sessionId, endUserState);
2794 }
2795
2796 public boolean isSwitchToVideoAvailable() {
2797 final boolean prerequisite =
2798 Media.audioOnly(getMedia())
2799 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2800 .contains(getEndUserState());
2801 return prerequisite && remoteHasVideoFeature();
2802 }
2803
2804 private boolean remoteHasVideoFeature() {
2805 final Contact contact = id.getContact();
2806 final Presence presence =
2807 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2808 final ServiceDiscoveryResult serviceDiscoveryResult =
2809 presence == null ? null : presence.getServiceDiscoveryResult();
2810 final List<String> features =
2811 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
2812 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
2813 }
2814
2815 private interface OnIceServersDiscovered {
2816 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2817 }
2818}