1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4import android.os.Environment;
5
6import androidx.annotation.NonNull;
7import androidx.annotation.Nullable;
8
9import com.google.common.base.Joiner;
10import com.google.common.base.Optional;
11import com.google.common.base.Preconditions;
12import com.google.common.base.Stopwatch;
13import com.google.common.base.Strings;
14import com.google.common.base.Throwables;
15import com.google.common.collect.Collections2;
16import com.google.common.collect.ImmutableList;
17import com.google.common.collect.ImmutableMap;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Multimap;
23import com.google.common.collect.Sets;
24import com.google.common.primitives.Ints;
25import com.google.common.util.concurrent.FutureCallback;
26import com.google.common.util.concurrent.Futures;
27import com.google.common.util.concurrent.ListenableFuture;
28import com.google.common.util.concurrent.MoreExecutors;
29import com.google.common.util.concurrent.SettableFuture;
30
31import org.webrtc.DtmfSender;
32import org.webrtc.EglBase;
33import org.webrtc.IceCandidate;
34import org.webrtc.PeerConnection;
35import org.webrtc.VideoTrack;
36
37import java.io.File;
38import java.io.IOException;
39import java.util.Arrays;
40import java.util.Collection;
41import java.util.Collections;
42import java.util.LinkedList;
43import java.util.List;
44import java.util.Map;
45import java.util.Queue;
46import java.util.Set;
47import java.util.concurrent.ExecutionException;
48import java.util.concurrent.ScheduledFuture;
49import java.util.concurrent.TimeUnit;
50
51import eu.siacs.conversations.BuildConfig;
52import eu.siacs.conversations.Config;
53import eu.siacs.conversations.crypto.axolotl.AxolotlService;
54import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
55import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
56import eu.siacs.conversations.entities.Account;
57import eu.siacs.conversations.entities.Contact;
58import eu.siacs.conversations.entities.Conversation;
59import eu.siacs.conversations.entities.Conversational;
60import eu.siacs.conversations.entities.Message;
61import eu.siacs.conversations.entities.Presence;
62import eu.siacs.conversations.entities.RtpSessionStatus;
63import eu.siacs.conversations.entities.ServiceDiscoveryResult;
64import eu.siacs.conversations.services.AppRTCAudioManager;
65import eu.siacs.conversations.utils.IP;
66import eu.siacs.conversations.xml.Element;
67import eu.siacs.conversations.xml.Namespace;
68import eu.siacs.conversations.xmpp.Jid;
69import eu.siacs.conversations.xmpp.OnIqPacketReceived;
70import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
71import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
72import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
73import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
74import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
75import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
76import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
77import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
78import eu.siacs.conversations.xmpp.stanzas.IqPacket;
79import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
80
81public class JingleRtpConnection extends AbstractJingleConnection
82 implements WebRTCWrapper.EventCallback {
83
84 public static final List<State> STATES_SHOWING_ONGOING_CALL =
85 Arrays.asList(
86 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
87 private static final long BUSY_TIME_OUT = 30;
88 private static final List<State> TERMINATED =
89 Arrays.asList(
90 State.ACCEPTED,
91 State.REJECTED,
92 State.REJECTED_RACED,
93 State.RETRACTED,
94 State.RETRACTED_RACED,
95 State.TERMINATED_SUCCESS,
96 State.TERMINATED_DECLINED_OR_BUSY,
97 State.TERMINATED_CONNECTIVITY_ERROR,
98 State.TERMINATED_CANCEL_OR_TIMEOUT,
99 State.TERMINATED_APPLICATION_FAILURE,
100 State.TERMINATED_SECURITY_ERROR);
101
102 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
103
104 static {
105 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
106 new ImmutableMap.Builder<>();
107 transitionBuilder.put(
108 State.NULL,
109 ImmutableList.of(
110 State.PROPOSED,
111 State.SESSION_INITIALIZED,
112 State.TERMINATED_APPLICATION_FAILURE,
113 State.TERMINATED_SECURITY_ERROR));
114 transitionBuilder.put(
115 State.PROPOSED,
116 ImmutableList.of(
117 State.ACCEPTED,
118 State.PROCEED,
119 State.REJECTED,
120 State.RETRACTED,
121 State.TERMINATED_APPLICATION_FAILURE,
122 State.TERMINATED_SECURITY_ERROR,
123 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
124 // rebinds
125 ));
126 transitionBuilder.put(
127 State.PROCEED,
128 ImmutableList.of(
129 State.REJECTED_RACED,
130 State.RETRACTED_RACED,
131 State.SESSION_INITIALIZED_PRE_APPROVED,
132 State.TERMINATED_SUCCESS,
133 State.TERMINATED_APPLICATION_FAILURE,
134 State.TERMINATED_SECURITY_ERROR,
135 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
136 // bounces of the proceed message
137 ));
138 transitionBuilder.put(
139 State.SESSION_INITIALIZED,
140 ImmutableList.of(
141 State.SESSION_ACCEPTED,
142 State.TERMINATED_SUCCESS,
143 State.TERMINATED_DECLINED_OR_BUSY,
144 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
145 // and IQ timeouts
146 State.TERMINATED_CANCEL_OR_TIMEOUT,
147 State.TERMINATED_APPLICATION_FAILURE,
148 State.TERMINATED_SECURITY_ERROR));
149 transitionBuilder.put(
150 State.SESSION_INITIALIZED_PRE_APPROVED,
151 ImmutableList.of(
152 State.SESSION_ACCEPTED,
153 State.TERMINATED_SUCCESS,
154 State.TERMINATED_DECLINED_OR_BUSY,
155 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
156 // and IQ timeouts
157 State.TERMINATED_CANCEL_OR_TIMEOUT,
158 State.TERMINATED_APPLICATION_FAILURE,
159 State.TERMINATED_SECURITY_ERROR));
160 transitionBuilder.put(
161 State.SESSION_ACCEPTED,
162 ImmutableList.of(
163 State.TERMINATED_SUCCESS,
164 State.TERMINATED_DECLINED_OR_BUSY,
165 State.TERMINATED_CONNECTIVITY_ERROR,
166 State.TERMINATED_CANCEL_OR_TIMEOUT,
167 State.TERMINATED_APPLICATION_FAILURE,
168 State.TERMINATED_SECURITY_ERROR));
169 VALID_TRANSITIONS = transitionBuilder.build();
170 }
171
172 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
173 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
174 pendingIceCandidates = new LinkedList<>();
175 private final OmemoVerification omemoVerification = new OmemoVerification();
176 private final Message message;
177 private State state = State.NULL;
178 private Set<Media> proposedMedia;
179 private RtpContentMap initiatorRtpContentMap;
180 private RtpContentMap responderRtpContentMap;
181 private RtpContentMap incomingContentAdd;
182 private RtpContentMap outgoingContentAdd;
183 private IceUdpTransportInfo.Setup peerDtlsSetup;
184 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
185 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
186 private ScheduledFuture<?> ringingTimeoutFuture;
187 private final long created = System.currentTimeMillis() / 1000L;
188 private final SettableFuture<Collection<IceCandidate>> iceGatheringComplete = SettableFuture.create();
189
190 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
191 super(jingleConnectionManager, id, initiator);
192 final Conversation conversation =
193 jingleConnectionManager
194 .getXmppConnectionService()
195 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
196 this.message =
197 new Message(
198 conversation,
199 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
200 Message.TYPE_RTP_SESSION,
201 id.sessionId);
202 }
203
204 private static State reasonToState(Reason reason) {
205 switch (reason) {
206 case SUCCESS:
207 return State.TERMINATED_SUCCESS;
208 case DECLINE:
209 case BUSY:
210 return State.TERMINATED_DECLINED_OR_BUSY;
211 case CANCEL:
212 case TIMEOUT:
213 return State.TERMINATED_CANCEL_OR_TIMEOUT;
214 case SECURITY_ERROR:
215 return State.TERMINATED_SECURITY_ERROR;
216 case FAILED_APPLICATION:
217 case UNSUPPORTED_TRANSPORTS:
218 case UNSUPPORTED_APPLICATIONS:
219 return State.TERMINATED_APPLICATION_FAILURE;
220 default:
221 return State.TERMINATED_CONNECTIVITY_ERROR;
222 }
223 }
224
225 @Override
226 synchronized void deliverPacket(final JinglePacket jinglePacket) {
227 switch (jinglePacket.getAction()) {
228 case SESSION_INITIATE:
229 receiveSessionInitiate(jinglePacket);
230 break;
231 case TRANSPORT_INFO:
232 receiveTransportInfo(jinglePacket);
233 break;
234 case SESSION_ACCEPT:
235 receiveSessionAccept(jinglePacket);
236 break;
237 case SESSION_TERMINATE:
238 receiveSessionTerminate(jinglePacket);
239 break;
240 case CONTENT_ADD:
241 receiveContentAdd(jinglePacket);
242 break;
243 case CONTENT_ACCEPT:
244 receiveContentAccept(jinglePacket);
245 break;
246 case CONTENT_REJECT:
247 receiveContentReject(jinglePacket);
248 break;
249 case CONTENT_REMOVE:
250 receiveContentRemove(jinglePacket);
251 break;
252 case CONTENT_MODIFY:
253 receiveContentModify(jinglePacket);
254 break;
255 default:
256 respondOk(jinglePacket);
257 Log.d(
258 Config.LOGTAG,
259 String.format(
260 "%s: received unhandled jingle action %s",
261 id.account.getJid().asBareJid(), jinglePacket.getAction()));
262 break;
263 }
264 }
265
266 @Override
267 synchronized void notifyRebound() {
268 if (isTerminated()) {
269 return;
270 }
271 webRTCWrapper.close();
272 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
273 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
274 }
275 if (isInState(
276 State.SESSION_INITIALIZED,
277 State.SESSION_INITIALIZED_PRE_APPROVED,
278 State.SESSION_ACCEPTED)) {
279 // we might have already changed resources (full jid) at this point; so this might not
280 // even reach the other party
281 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
282 } else {
283 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
284 finish();
285 }
286 }
287
288 public boolean applyDtmfTone(String tone) {
289 return webRTCWrapper.applyDtmfTone(tone);
290 }
291
292 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
293 respondOk(jinglePacket);
294 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
295 final State previous = this.state;
296 Log.d(
297 Config.LOGTAG,
298 id.account.getJid().asBareJid()
299 + ": received session terminate reason="
300 + wrapper.reason
301 + "("
302 + Strings.nullToEmpty(wrapper.text)
303 + ") while in state "
304 + previous);
305 if (TERMINATED.contains(previous)) {
306 Log.d(
307 Config.LOGTAG,
308 id.account.getJid().asBareJid()
309 + ": ignoring session terminate because already in "
310 + previous);
311 return;
312 }
313 webRTCWrapper.close();
314 final State target = reasonToState(wrapper.reason);
315 transitionOrThrow(target);
316 writeLogMessage(target);
317 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
318 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
319 }
320 finish();
321 }
322
323 private void receiveTransportInfo(final JinglePacket jinglePacket) {
324 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
325 // INITIALIZED only after transport-info has been received
326 if (isInState(
327 State.NULL,
328 State.PROCEED,
329 State.SESSION_INITIALIZED,
330 State.SESSION_INITIALIZED_PRE_APPROVED,
331 State.SESSION_ACCEPTED)) {
332 final RtpContentMap contentMap;
333 try {
334 contentMap = RtpContentMap.of(jinglePacket);
335 } catch (final IllegalArgumentException | NullPointerException e) {
336 Log.d(
337 Config.LOGTAG,
338 id.account.getJid().asBareJid()
339 + ": improperly formatted contents; ignoring",
340 e);
341 respondOk(jinglePacket);
342 return;
343 }
344 receiveTransportInfo(jinglePacket, contentMap);
345 } else {
346 if (isTerminated()) {
347 respondOk(jinglePacket);
348 Log.d(
349 Config.LOGTAG,
350 id.account.getJid().asBareJid()
351 + ": ignoring out-of-order transport info; we where already terminated");
352 } else {
353 Log.d(
354 Config.LOGTAG,
355 id.account.getJid().asBareJid()
356 + ": received transport info while in state="
357 + this.state);
358 terminateWithOutOfOrder(jinglePacket);
359 }
360 }
361 }
362
363 private void receiveTransportInfo(
364 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
365 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
366 contentMap.contents.entrySet();
367 final RtpContentMap remote = getRemoteContentMap();
368 final Set<String> remoteContentIds = remote == null ? Collections.emptySet() : remote.contents.keySet();
369 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
370 Log.d(Config.LOGTAG,"received transport-info for unknown contents "+contentMap.contents.keySet()+" (known: "+remoteContentIds+")");
371 respondOk(jinglePacket);
372 pendingIceCandidates.addAll(candidates);
373 return;
374 }
375 if (this.state != State.SESSION_ACCEPTED) {
376 Log.d(Config.LOGTAG,"received transport-info prematurely. adding to backlog");
377 respondOk(jinglePacket);
378 pendingIceCandidates.addAll(candidates);
379 return;
380 }
381 // zero candidates + modified credentials are an ICE restart offer
382 if (checkForIceRestart(jinglePacket, contentMap)) {
383 return;
384 }
385 respondOk(jinglePacket);
386 try {
387 processCandidates(candidates);
388 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
389 Log.w(
390 Config.LOGTAG,
391 id.account.getJid().asBareJid()
392 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
393 }
394 }
395
396 private void receiveContentAdd(final JinglePacket jinglePacket) {
397 final RtpContentMap modification;
398 try {
399 modification = RtpContentMap.of(jinglePacket);
400 modification.requireContentDescriptions();
401 } catch (final RuntimeException e) {
402 Log.d(
403 Config.LOGTAG,
404 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
405 Throwables.getRootCause(e));
406 respondOk(jinglePacket);
407 webRTCWrapper.close();
408 sendSessionTerminate(Reason.of(e), e.getMessage());
409 return;
410 }
411 if (isInState(State.SESSION_ACCEPTED)) {
412 receiveContentAdd(jinglePacket, modification);
413 } else {
414 terminateWithOutOfOrder(jinglePacket);
415 }
416 }
417
418 private void receiveContentAdd(
419 final JinglePacket jinglePacket, final RtpContentMap modification) {
420 final RtpContentMap remote = getRemoteContentMap();
421 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
422 respondOk(jinglePacket);
423 this.webRTCWrapper.close();
424 sendSessionTerminate(
425 Reason.FAILED_APPLICATION,
426 String.format(
427 "contents with names %s already exists",
428 Joiner.on(", ").join(modification.getNames())));
429 return;
430 }
431 final ContentAddition contentAddition =
432 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
433
434 final RtpContentMap outgoing = this.outgoingContentAdd;
435 final Set<ContentAddition.Summary> outgoingContentAddSummary =
436 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
437
438 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
439 if (isInitiator()) {
440 Log.d(
441 Config.LOGTAG,
442 id.getAccount().getJid().asBareJid()
443 + ": respond with tie break to matching content-add offer");
444 respondWithTieBreak(jinglePacket);
445 } else {
446 Log.d(
447 Config.LOGTAG,
448 id.getAccount().getJid().asBareJid()
449 + ": automatically accept matching content-add offer");
450 acceptContentAdd(contentAddition.summary, modification);
451 }
452 return;
453 }
454
455 // once we can display multiple video tracks we can be more loose with this condition
456 // theoretically it should also be fine to automatically accept audio only contents
457 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
458 Log.d(
459 Config.LOGTAG,
460 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
461 this.incomingContentAdd = modification;
462 respondOk(jinglePacket);
463 updateEndUserState();
464 } else {
465 respondOk(jinglePacket);
466 // TODO do we want to add a reason?
467 rejectContentAdd(modification);
468 }
469 }
470
471 private void receiveContentAccept(final JinglePacket jinglePacket) {
472 final RtpContentMap receivedContentAccept;
473 try {
474 receivedContentAccept = RtpContentMap.of(jinglePacket);
475 receivedContentAccept.requireContentDescriptions();
476 } catch (final RuntimeException e) {
477 Log.d(
478 Config.LOGTAG,
479 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
480 Throwables.getRootCause(e));
481 respondOk(jinglePacket);
482 webRTCWrapper.close();
483 sendSessionTerminate(Reason.of(e), e.getMessage());
484 return;
485 }
486
487 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
488 if (outgoingContentAdd == null) {
489 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
490 terminateWithOutOfOrder(jinglePacket);
491 return;
492 }
493 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
494 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
495 this.outgoingContentAdd = null;
496 respondOk(jinglePacket);
497 receiveContentAccept(receivedContentAccept);
498 } else {
499 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
500 terminateWithOutOfOrder(jinglePacket);
501 }
502 }
503
504 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
505 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
506 final RtpContentMap modifiedContentMap =
507 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
508
509 setRemoteContentMap(modifiedContentMap);
510
511 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
512
513 final org.webrtc.SessionDescription sdp =
514 new org.webrtc.SessionDescription(
515 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
516
517 try {
518 this.webRTCWrapper.setRemoteDescription(sdp).get();
519 } catch (final Exception e) {
520 final Throwable cause = Throwables.getRootCause(e);
521 Log.d(
522 Config.LOGTAG,
523 id.getAccount().getJid().asBareJid()
524 + ": unable to set remote description after receiving content-accept",
525 cause);
526 webRTCWrapper.close();
527 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
528 return;
529 }
530 processCandidates(receivedContentAccept.contents.entrySet());
531 updateEndUserState();
532 Log.d(
533 Config.LOGTAG,
534 id.getAccount().getJid().asBareJid()
535 + ": remote has accepted content-add "
536 + ContentAddition.summary(receivedContentAccept));
537 }
538
539 private void receiveContentModify(final JinglePacket jinglePacket) {
540 // TODO check session accepted
541 final Map<String, Content.Senders> modification =
542 Maps.transformEntries(
543 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
544 final boolean isInitiator = isInitiator();
545 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
546 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
547 final Set<String> currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet();
548 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
549 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
550 respondOk(jinglePacket);
551 final RtpContentMap modifiedContentMap;
552 try {
553 modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification);
554 } catch (final IllegalArgumentException e) {
555 webRTCWrapper.close();
556 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
557 return;
558 }
559 this.outgoingContentAdd = modifiedContentMap;
560 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add");
561 } else if (remoteContentMap != null && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
562 respondOk(jinglePacket);
563 final RtpContentMap modifiedRemoteContentMap;
564 try {
565 modifiedRemoteContentMap = remoteContentMap.modifiedSendersChecked(isInitiator, modification);
566 } catch (final IllegalArgumentException e) {
567 webRTCWrapper.close();
568 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
569 return;
570 }
571 final SessionDescription offer;
572 try {
573 offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator());
574 } catch (final IllegalArgumentException | NullPointerException e) {
575 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-modify to SDP", e);
576 webRTCWrapper.close();
577 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
578 return;
579 }
580 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": auto accepting content-modification");
581 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
582 } else {
583 Log.d(Config.LOGTAG,"received unsupported content modification "+modification);
584 respondWithItemNotFound(jinglePacket);
585 }
586 }
587
588 private void autoAcceptContentModify(final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
589 this.setRemoteContentMap(modifiedRemoteContentMap);
590 final org.webrtc.SessionDescription sdp =
591 new org.webrtc.SessionDescription(
592 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
593 try {
594 this.webRTCWrapper.setRemoteDescription(sdp).get();
595 // auto accept is only done when we already have tracks
596 final SessionDescription answer = setLocalSessionDescription();
597 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
598 modifyLocalContentMap(rtpContentMap);
599 // we do not need to send an answer but do we have to resend the candidates currently in SDP?
600 //resendCandidatesFromSdp(answer);
601 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
602 } catch (final Exception e) {
603 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
604 webRTCWrapper.close();
605 sendSessionTerminate(Reason.FAILED_APPLICATION);
606 }
607 }
608
609 private void resendCandidatesFromSdp(final SessionDescription answer) {
610 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
611 for(final SessionDescription.Media media : answer.media) {
612 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
613 if (Strings.isNullOrEmpty(mid)) {
614 continue;
615 }
616 for(final String sdpCandidate : media.attributes.get("candidate")) {
617 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
618 if (candidate != null) {
619 candidateBuilder.put(mid,candidate);
620 }
621 }
622 }
623 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build();
624 sendTransportInfo(candidates);
625 }
626
627 private void receiveContentReject(final JinglePacket jinglePacket) {
628 final RtpContentMap receivedContentReject;
629 try {
630 receivedContentReject = RtpContentMap.of(jinglePacket);
631 } catch (final RuntimeException e) {
632 Log.d(
633 Config.LOGTAG,
634 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
635 Throwables.getRootCause(e));
636 respondOk(jinglePacket);
637 this.webRTCWrapper.close();
638 sendSessionTerminate(Reason.of(e), e.getMessage());
639 return;
640 }
641
642 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
643 if (outgoingContentAdd == null) {
644 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
645 terminateWithOutOfOrder(jinglePacket);
646 return;
647 }
648 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
649 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
650 this.outgoingContentAdd = null;
651 respondOk(jinglePacket);
652 Log.d(Config.LOGTAG,jinglePacket.toString());
653 receiveContentReject(ourSummary);
654 } else {
655 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
656 terminateWithOutOfOrder(jinglePacket);
657 }
658 }
659
660 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
661 try {
662 this.webRTCWrapper.removeTrack(Media.VIDEO);
663 final RtpContentMap localContentMap = customRollback();
664 modifyLocalContentMap(localContentMap);
665 } catch (final Exception e) {
666 final Throwable cause = Throwables.getRootCause(e);
667 Log.d(
668 Config.LOGTAG,
669 id.getAccount().getJid().asBareJid()
670 + ": unable to rollback local description after receiving content-reject",
671 cause);
672 webRTCWrapper.close();
673 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
674 return;
675 }
676 Log.d(
677 Config.LOGTAG,
678 id.getAccount().getJid().asBareJid()
679 + ": remote has rejected our content-add "
680 + summary);
681 }
682
683 private void receiveContentRemove(final JinglePacket jinglePacket) {
684 final RtpContentMap receivedContentRemove;
685 try {
686 receivedContentRemove = RtpContentMap.of(jinglePacket);
687 receivedContentRemove.requireContentDescriptions();
688 } catch (final RuntimeException e) {
689 Log.d(
690 Config.LOGTAG,
691 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
692 Throwables.getRootCause(e));
693 respondOk(jinglePacket);
694 this.webRTCWrapper.close();
695 sendSessionTerminate(Reason.of(e), e.getMessage());
696 return;
697 }
698 respondOk(jinglePacket);
699 receiveContentRemove(receivedContentRemove);
700 }
701
702 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
703 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
704 final Set<ContentAddition.Summary> contentAddSummary =
705 incomingContentAdd == null
706 ? Collections.emptySet()
707 : ContentAddition.summary(incomingContentAdd);
708 final Set<ContentAddition.Summary> removeSummary =
709 ContentAddition.summary(receivedContentRemove);
710 if (contentAddSummary.equals(removeSummary)) {
711 this.incomingContentAdd = null;
712 updateEndUserState();
713 } else {
714 webRTCWrapper.close();
715 sendSessionTerminate(
716 Reason.FAILED_APPLICATION,
717 String.format(
718 "%s only supports %s as a means to retract a not yet accepted %s",
719 BuildConfig.APP_NAME,
720 JinglePacket.Action.CONTENT_REMOVE,
721 JinglePacket.Action.CONTENT_ADD));
722 }
723 }
724
725 public synchronized void retractContentAdd() {
726 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
727 if (outgoingContentAdd == null) {
728 throw new IllegalStateException("Not outgoing content add");
729 }
730 try {
731 webRTCWrapper.removeTrack(Media.VIDEO);
732 final RtpContentMap localContentMap = customRollback();
733 modifyLocalContentMap(localContentMap);
734 } catch (final Exception e) {
735 final Throwable cause = Throwables.getRootCause(e);
736 Log.d(
737 Config.LOGTAG,
738 id.getAccount().getJid().asBareJid()
739 + ": unable to rollback local description after trying to retract content-add",
740 cause);
741 webRTCWrapper.close();
742 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
743 return;
744 }
745 this.outgoingContentAdd = null;
746 final JinglePacket retract =
747 outgoingContentAdd
748 .toStub()
749 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
750 this.send(retract);
751 Log.d(
752 Config.LOGTAG,
753 id.getAccount().getJid()
754 + ": retract content-add "
755 + ContentAddition.summary(outgoingContentAdd));
756 }
757
758 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
759 final SessionDescription sdp = setLocalSessionDescription();
760 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
761 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
762 this.webRTCWrapper
763 .setRemoteDescription(
764 new org.webrtc.SessionDescription(
765 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
766 .get();
767 return localRtpContentMap;
768 }
769
770 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
771 final RtpContentMap currentRemote = getRemoteContentMap();
772 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
773 if (diff.isEmpty()) {
774 throw new IllegalStateException(
775 "Unexpected rollback condition. No difference between local and remote");
776 }
777 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
778 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
779 final RtpContentMap nextRemote =
780 currentRemote.addContent(
781 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
782 return SessionDescription.of(nextRemote, !isInitiator());
783 }
784 throw new IllegalStateException(
785 "Unexpected rollback condition. Senders were not uniformly none");
786 }
787
788 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
789 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
790 if (incomingContentAdd == null) {
791 throw new IllegalStateException("No incoming content add");
792 }
793
794 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
795 this.incomingContentAdd = null;
796 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
797 Log.d(Config.LOGTAG,"senders of incoming content-add: "+senders);
798 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
799 Log.d(Config.LOGTAG,"content addition is receive only. we want to upgrade to 'both'");
800 final RtpContentMap modifiedSenders = incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
801 final JinglePacket proposedContentModification = modifiedSenders.toStub().toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
802 proposedContentModification.setTo(id.with);
803 xmppConnectionService.sendIqPacket(id.account, proposedContentModification, (account, response) -> {
804 if (response.getType() == IqPacket.TYPE.RESULT) {
805 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has accepted our upgrade to senders=both");
806 acceptContentAdd(ContentAddition.summary(modifiedSenders), modifiedSenders);
807 } else {
808 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has rejected our upgrade to senders=both");
809 acceptContentAdd(contentAddition, incomingContentAdd);
810 }
811 });
812 }
813 } else {
814 throw new IllegalStateException("Accepted content add does not match pending content-add");
815 }
816 }
817
818 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
819 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
820 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
821 this.setRemoteContentMap(modifiedContentMap);
822
823 final SessionDescription offer;
824 try {
825 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
826 } catch (final IllegalArgumentException | NullPointerException e) {
827 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
828 webRTCWrapper.close();
829 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
830 return;
831 }
832 this.incomingContentAdd = null;
833 acceptContentAdd(contentAddition, offer);
834 }
835
836 private void acceptContentAdd(
837 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
838 final org.webrtc.SessionDescription sdp =
839 new org.webrtc.SessionDescription(
840 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
841 try {
842 this.webRTCWrapper.setRemoteDescription(sdp).get();
843
844 // TODO add tracks for 'media' where contentAddition.senders matches
845
846 // TODO if senders.sending(isInitiator())
847
848 this.webRTCWrapper.addTrack(Media.VIDEO);
849
850 // TODO add additional transceivers for recv only cases
851
852 final SessionDescription answer = setLocalSessionDescription();
853 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
854
855 final RtpContentMap contentAcceptMap =
856 rtpContentMap.toContentModification(
857 Collections2.transform(contentAddition, ca -> ca.name));
858 Log.d(
859 Config.LOGTAG,
860 id.getAccount().getJid().asBareJid()
861 + ": sending content-accept "
862 + ContentAddition.summary(contentAcceptMap));
863
864 addIceCandidatesFromBlackLog();
865
866 modifyLocalContentMap(rtpContentMap);
867 sendContentAccept(contentAcceptMap);
868 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
869 } catch (final Exception e) {
870 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
871 webRTCWrapper.close();
872 sendSessionTerminate(Reason.FAILED_APPLICATION);
873 }
874 }
875
876 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
877 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
878 send(jinglePacket);
879 }
880
881 public synchronized void rejectContentAdd() {
882 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
883 if (incomingContentAdd == null) {
884 throw new IllegalStateException("No incoming content add");
885 }
886 this.incomingContentAdd = null;
887 updateEndUserState();
888 rejectContentAdd(incomingContentAdd);
889 }
890
891 private void rejectContentAdd(final RtpContentMap contentMap) {
892 final JinglePacket jinglePacket =
893 contentMap
894 .toStub()
895 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
896 Log.d(
897 Config.LOGTAG,
898 id.getAccount().getJid().asBareJid()
899 + ": rejecting content "
900 + ContentAddition.summary(contentMap));
901 send(jinglePacket);
902 }
903
904 private boolean checkForIceRestart(
905 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
906 final RtpContentMap existing = getRemoteContentMap();
907 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
908 final IceUdpTransportInfo.Credentials newCredentials;
909 try {
910 existingCredentials = existing.getCredentials();
911 newCredentials = rtpContentMap.getDistinctCredentials();
912 } catch (final IllegalStateException e) {
913 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
914 return false;
915 }
916 if (existingCredentials.contains(newCredentials)) {
917 return false;
918 }
919 // TODO an alternative approach is to check if we already got an iq result to our
920 // ICE-restart
921 // and if that's the case we are seeing an answer.
922 // This might be more spec compliant but also more error prone potentially
923 final boolean isSignalStateStable = this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
924 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
925 final boolean isOffer = rtpContentMap.emptyCandidates();
926 final RtpContentMap restartContentMap;
927 try {
928 if (isOffer) {
929 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
930 restartContentMap =
931 existing.modifiedCredentials(
932 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
933 } else {
934 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
935 Log.d(
936 Config.LOGTAG,
937 "received confirmation of ICE restart"
938 + newCredentials
939 + " peer_setup="
940 + setup);
941 // DTLS setup attribute needs to be rewritten to reflect current peer state
942 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
943 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
944 }
945 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
946 return isOffer;
947 } else {
948 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
949 respondWithTieBreak(jinglePacket);
950 return true;
951 }
952 } catch (final Exception exception) {
953 respondOk(jinglePacket);
954 final Throwable rootCause = Throwables.getRootCause(exception);
955 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
956 // If this happens a termination is already in progress
957 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
958 return true;
959 }
960 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
961 webRTCWrapper.close();
962 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
963 return true;
964 }
965 }
966
967 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
968 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
969 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
970 throw new IllegalStateException("Invalid peer setup");
971 }
972 return peerSetup;
973 }
974
975 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
976 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
977 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
978 }
979 this.peerDtlsSetup = setup;
980 }
981
982 private boolean applyIceRestart(
983 final JinglePacket jinglePacket,
984 final RtpContentMap restartContentMap,
985 final boolean isOffer)
986 throws ExecutionException, InterruptedException {
987 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
988 final org.webrtc.SessionDescription.Type type =
989 isOffer
990 ? org.webrtc.SessionDescription.Type.OFFER
991 : org.webrtc.SessionDescription.Type.ANSWER;
992 org.webrtc.SessionDescription sdp =
993 new org.webrtc.SessionDescription(type, sessionDescription.toString());
994 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
995 if (isInitiator()) {
996 // We ignore the offer and respond with tie-break. This will clause the responder
997 // not to apply the content map
998 return false;
999 }
1000 }
1001 webRTCWrapper.setRemoteDescription(sdp).get();
1002 setRemoteContentMap(restartContentMap);
1003 if (isOffer) {
1004 final SessionDescription localSessionDescription = setLocalSessionDescription();
1005 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1006 // We need to respond OK before sending any candidates
1007 respondOk(jinglePacket);
1008 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1009 } else {
1010 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1011 }
1012 return true;
1013 }
1014
1015 private void processCandidates(
1016 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
1017 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
1018 processCandidate(content);
1019 }
1020 }
1021
1022 private void processCandidate(
1023 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
1024 final RtpContentMap rtpContentMap = getRemoteContentMap();
1025 final List<String> indices = toIdentificationTags(rtpContentMap);
1026 final String sdpMid = content.getKey(); // aka content name
1027 final IceUdpTransportInfo transport = content.getValue().transport;
1028 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1029
1030 // TODO check that credentials remained the same
1031
1032 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1033 final String sdp;
1034 try {
1035 sdp = candidate.toSdpAttribute(credentials.ufrag);
1036 } catch (final IllegalArgumentException e) {
1037 Log.d(
1038 Config.LOGTAG,
1039 id.account.getJid().asBareJid()
1040 + ": ignoring invalid ICE candidate "
1041 + e.getMessage());
1042 continue;
1043 }
1044 final int mLineIndex = indices.indexOf(sdpMid);
1045 if (mLineIndex < 0) {
1046 Log.w(
1047 Config.LOGTAG,
1048 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1049 }
1050 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1051 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1052 this.webRTCWrapper.addIceCandidate(iceCandidate);
1053 }
1054 }
1055
1056 private RtpContentMap getRemoteContentMap() {
1057 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1058 }
1059
1060 private RtpContentMap getLocalContentMap() {
1061 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1062 }
1063
1064 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1065 final Group originalGroup = rtpContentMap.group;
1066 final List<String> identificationTags =
1067 originalGroup == null
1068 ? rtpContentMap.getNames()
1069 : originalGroup.getIdentificationTags();
1070 if (identificationTags.size() == 0) {
1071 Log.w(
1072 Config.LOGTAG,
1073 id.account.getJid().asBareJid()
1074 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1075 }
1076 return identificationTags;
1077 }
1078
1079 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1080 final JinglePacket jinglePacket, final boolean expectVerification) {
1081 final RtpContentMap receivedContentMap;
1082 try {
1083 receivedContentMap = RtpContentMap.of(jinglePacket);
1084 } catch (final Exception e) {
1085 return Futures.immediateFailedFuture(e);
1086 }
1087 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1088 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1089 id.account
1090 .getAxolotlService()
1091 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1092 return Futures.transform(
1093 future,
1094 omemoVerifiedPayload -> {
1095 // TODO test if an exception here triggers a correct abort
1096 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1097 Log.d(
1098 Config.LOGTAG,
1099 id.account.getJid().asBareJid()
1100 + ": received verifiable DTLS fingerprint via "
1101 + omemoVerification);
1102 return omemoVerifiedPayload.getPayload();
1103 },
1104 MoreExecutors.directExecutor());
1105 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1106 return Futures.immediateFailedFuture(
1107 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1108 } else {
1109 return Futures.immediateFuture(receivedContentMap);
1110 }
1111 }
1112
1113 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1114 if (isInitiator()) {
1115 Log.d(
1116 Config.LOGTAG,
1117 String.format(
1118 "%s: received session-initiate even though we were initiating",
1119 id.account.getJid().asBareJid()));
1120 if (isTerminated()) {
1121 Log.d(
1122 Config.LOGTAG,
1123 String.format(
1124 "%s: got a reason to terminate with out-of-order. but already in state %s",
1125 id.account.getJid().asBareJid(), getState()));
1126 respondWithOutOfOrder(jinglePacket);
1127 } else {
1128 terminateWithOutOfOrder(jinglePacket);
1129 }
1130 return;
1131 }
1132 final Presence presence = id.getContact().getPresences().get(id.getWith().getResource());
1133 final ServiceDiscoveryResult disco = presence == null ? null : presence.getServiceDiscoveryResult();
1134 if (disco != null && disco.getFeatures().contains("urn:ietf:rfc:3264")) webRTCWrapper.setRFC3264(true);
1135 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1136 Futures.addCallback(
1137 future,
1138 new FutureCallback<RtpContentMap>() {
1139 @Override
1140 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1141 receiveSessionInitiate(jinglePacket, rtpContentMap);
1142 }
1143
1144 @Override
1145 public void onFailure(@NonNull final Throwable throwable) {
1146 respondOk(jinglePacket);
1147 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1148 }
1149 },
1150 MoreExecutors.directExecutor());
1151 }
1152
1153 private void receiveSessionInitiate(
1154 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1155 try {
1156 contentMap.requireContentDescriptions();
1157 contentMap.requireDTLSFingerprint(true);
1158 } catch (final RuntimeException e) {
1159 Log.d(
1160 Config.LOGTAG,
1161 id.account.getJid().asBareJid() + ": improperly formatted contents",
1162 Throwables.getRootCause(e));
1163 respondOk(jinglePacket);
1164 sendSessionTerminate(Reason.of(e), e.getMessage());
1165 return;
1166 }
1167 Log.d(
1168 Config.LOGTAG,
1169 "processing session-init with " + contentMap.contents.size() + " contents");
1170 final State target;
1171 if (this.state == State.PROCEED) {
1172 Preconditions.checkState(
1173 proposedMedia != null && proposedMedia.size() > 0,
1174 "proposed media must be set when processing pre-approved session-initiate");
1175 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1176 sendSessionTerminate(
1177 Reason.SECURITY_ERROR,
1178 String.format(
1179 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1180 this.proposedMedia, contentMap.getMedia()));
1181 return;
1182 }
1183 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1184 } else {
1185 target = State.SESSION_INITIALIZED;
1186 }
1187 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1188 respondOk(jinglePacket);
1189 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1190 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1191 Log.d(
1192 Config.LOGTAG,
1193 id.account.getJid().asBareJid()
1194 + ": automatically accepting session-initiate");
1195 sendSessionAccept();
1196 } else {
1197 Log.d(
1198 Config.LOGTAG,
1199 id.account.getJid().asBareJid()
1200 + ": received not pre-approved session-initiate. start ringing");
1201 startRinging();
1202 }
1203 } else {
1204 Log.d(
1205 Config.LOGTAG,
1206 String.format(
1207 "%s: received session-initiate while in state %s",
1208 id.account.getJid().asBareJid(), state));
1209 terminateWithOutOfOrder(jinglePacket);
1210 }
1211 }
1212
1213 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1214 if (!isInitiator()) {
1215 Log.d(
1216 Config.LOGTAG,
1217 String.format(
1218 "%s: received session-accept even though we were responding",
1219 id.account.getJid().asBareJid()));
1220 terminateWithOutOfOrder(jinglePacket);
1221 return;
1222 }
1223 final ListenableFuture<RtpContentMap> future =
1224 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1225 Futures.addCallback(
1226 future,
1227 new FutureCallback<RtpContentMap>() {
1228 @Override
1229 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1230 receiveSessionAccept(jinglePacket, rtpContentMap);
1231 }
1232
1233 @Override
1234 public void onFailure(@NonNull final Throwable throwable) {
1235 respondOk(jinglePacket);
1236 Log.d(
1237 Config.LOGTAG,
1238 id.account.getJid().asBareJid()
1239 + ": improperly formatted contents in session-accept",
1240 throwable);
1241 webRTCWrapper.close();
1242 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1243 }
1244 },
1245 MoreExecutors.directExecutor());
1246 }
1247
1248 private void receiveSessionAccept(
1249 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1250 try {
1251 contentMap.requireContentDescriptions();
1252 contentMap.requireDTLSFingerprint();
1253 } catch (final RuntimeException e) {
1254 respondOk(jinglePacket);
1255 Log.d(
1256 Config.LOGTAG,
1257 id.account.getJid().asBareJid()
1258 + ": improperly formatted contents in session-accept",
1259 e);
1260 webRTCWrapper.close();
1261 sendSessionTerminate(Reason.of(e), e.getMessage());
1262 return;
1263 }
1264 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1265 if (!initiatorMedia.equals(contentMap.getMedia())) {
1266 sendSessionTerminate(
1267 Reason.SECURITY_ERROR,
1268 String.format(
1269 "Your session-included included media %s but our session-initiate was %s",
1270 this.proposedMedia, contentMap.getMedia()));
1271 return;
1272 }
1273 Log.d(
1274 Config.LOGTAG,
1275 "processing session-accept with " + contentMap.contents.size() + " contents");
1276 if (transition(State.SESSION_ACCEPTED)) {
1277 respondOk(jinglePacket);
1278 receiveSessionAccept(contentMap);
1279 } else {
1280 Log.d(
1281 Config.LOGTAG,
1282 String.format(
1283 "%s: received session-accept while in state %s",
1284 id.account.getJid().asBareJid(), state));
1285 respondOk(jinglePacket);
1286 }
1287 }
1288
1289 private void receiveSessionAccept(final RtpContentMap contentMap) {
1290 this.responderRtpContentMap = contentMap;
1291 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1292 final SessionDescription sessionDescription;
1293 try {
1294 sessionDescription = SessionDescription.of(contentMap, false);
1295 } catch (final IllegalArgumentException | NullPointerException e) {
1296 Log.d(
1297 Config.LOGTAG,
1298 id.account.getJid().asBareJid()
1299 + ": unable convert offer from session-accept to SDP",
1300 e);
1301 webRTCWrapper.close();
1302 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1303 return;
1304 }
1305 final org.webrtc.SessionDescription answer =
1306 new org.webrtc.SessionDescription(
1307 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1308 try {
1309 this.webRTCWrapper.setRemoteDescription(answer).get();
1310 } catch (final Exception e) {
1311 Log.d(
1312 Config.LOGTAG,
1313 id.account.getJid().asBareJid()
1314 + ": unable to set remote description after receiving session-accept",
1315 Throwables.getRootCause(e));
1316 webRTCWrapper.close();
1317 sendSessionTerminate(
1318 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1319 return;
1320 }
1321 processCandidates(contentMap.contents.entrySet());
1322 }
1323
1324 private void sendSessionAccept() {
1325 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1326 if (rtpContentMap == null) {
1327 throw new IllegalStateException("initiator RTP Content Map has not been set");
1328 }
1329 final SessionDescription offer;
1330 try {
1331 offer = SessionDescription.of(rtpContentMap, true);
1332 } catch (final IllegalArgumentException | NullPointerException e) {
1333 Log.d(
1334 Config.LOGTAG,
1335 id.account.getJid().asBareJid()
1336 + ": unable convert offer from session-initiate to SDP",
1337 e);
1338 webRTCWrapper.close();
1339 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1340 return;
1341 }
1342 sendSessionAccept(rtpContentMap.getMedia(), offer);
1343 }
1344
1345 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1346 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1347 }
1348
1349 private synchronized void sendSessionAccept(
1350 final Set<Media> media,
1351 final SessionDescription offer,
1352 final List<PeerConnection.IceServer> iceServers) {
1353 if (isTerminated()) {
1354 Log.w(
1355 Config.LOGTAG,
1356 id.account.getJid().asBareJid()
1357 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1358 return;
1359 }
1360 try {
1361 setupWebRTC(media, iceServers);
1362 } catch (final WebRTCWrapper.InitializationException e) {
1363 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1364 webRTCWrapper.close();
1365 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1366 return;
1367 }
1368 final org.webrtc.SessionDescription sdp =
1369 new org.webrtc.SessionDescription(
1370 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1371 try {
1372 this.webRTCWrapper.setRemoteDescription(sdp).get();
1373 addIceCandidatesFromBlackLog();
1374 org.webrtc.SessionDescription webRTCSessionDescription =
1375 this.webRTCWrapper.setLocalDescription().get();
1376 prepareSessionAccept(webRTCSessionDescription);
1377 } catch (final Exception e) {
1378 failureToAcceptSession(e);
1379 }
1380 }
1381
1382 private void failureToAcceptSession(final Throwable throwable) {
1383 if (isTerminated()) {
1384 return;
1385 }
1386 final Throwable rootCause = Throwables.getRootCause(throwable);
1387 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1388 webRTCWrapper.close();
1389 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1390 }
1391
1392 private void addIceCandidatesFromBlackLog() {
1393 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1394 while ((foo = this.pendingIceCandidates.poll()) != null) {
1395 processCandidate(foo);
1396 Log.d(
1397 Config.LOGTAG,
1398 id.account.getJid().asBareJid() + ": added candidate from back log");
1399 }
1400 }
1401
1402 private void prepareSessionAccept(
1403 final org.webrtc.SessionDescription initialWebRTCSessionDescription) {
1404 Futures.addCallback(
1405 webRTCWrapper.getRFC3264() ? Futures.withTimeout(iceGatheringComplete, 2, TimeUnit.SECONDS, JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE) : Futures.immediateFuture(null),
1406 new FutureCallback<Collection<IceCandidate>>() {
1407 @Override
1408 public void onSuccess(final Collection<IceCandidate> iceCandidates) {
1409 org.webrtc.SessionDescription webRTCSessionDescription =
1410 JingleRtpConnection.this.webRTCWrapper.getLocalDescription();
1411 final SessionDescription sessionDescription =
1412 SessionDescription.parse(webRTCSessionDescription.description);
1413 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1414 JingleRtpConnection.this.responderRtpContentMap = respondingRtpContentMap;
1415 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1416 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1417 prepareOutgoingContentMap(respondingRtpContentMap);
1418 Futures.addCallback(
1419 outgoingContentMapFuture,
1420 new FutureCallback<RtpContentMap>() {
1421 @Override
1422 public void onSuccess(final RtpContentMap outgoingContentMap) {
1423 sendSessionAccept(outgoingContentMap);
1424 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1425 }
1426
1427 @Override
1428 public void onFailure(@NonNull Throwable throwable) {
1429 failureToAcceptSession(throwable);
1430 }
1431 },
1432 MoreExecutors.directExecutor());
1433 }
1434
1435 @Override
1436 public void onFailure(@NonNull final Throwable throwable) {
1437 Log.e(Config.LOGTAG, "ICE gathering didn't finish clean: " + throwable);
1438 onSuccess(null);
1439 }
1440 },
1441 MoreExecutors.directExecutor()
1442 );
1443 }
1444
1445 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1446 if (isTerminated()) {
1447 Log.w(
1448 Config.LOGTAG,
1449 id.account.getJid().asBareJid()
1450 + ": preparing session accept was too slow. already terminated. nothing to do.");
1451 return;
1452 }
1453 transitionOrThrow(State.SESSION_ACCEPTED);
1454 final JinglePacket sessionAccept =
1455 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1456 send(sessionAccept);
1457 }
1458
1459 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1460 final RtpContentMap rtpContentMap) {
1461 if (this.omemoVerification.hasDeviceId()) {
1462 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1463 verifiedPayloadFuture =
1464 id.account
1465 .getAxolotlService()
1466 .encrypt(
1467 rtpContentMap,
1468 id.with,
1469 omemoVerification.getDeviceId());
1470 return Futures.transform(
1471 verifiedPayloadFuture,
1472 verifiedPayload -> {
1473 omemoVerification.setOrEnsureEqual(verifiedPayload);
1474 return verifiedPayload.getPayload();
1475 },
1476 MoreExecutors.directExecutor());
1477 } else {
1478 return Futures.immediateFuture(rtpContentMap);
1479 }
1480 }
1481
1482 synchronized void deliveryMessage(
1483 final Jid from,
1484 final Element message,
1485 final String serverMessageId,
1486 final long timestamp) {
1487 Log.d(
1488 Config.LOGTAG,
1489 id.account.getJid().asBareJid()
1490 + ": delivered message to JingleRtpConnection "
1491 + message);
1492 switch (message.getName()) {
1493 case "propose":
1494 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1495 break;
1496 case "proceed":
1497 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1498 break;
1499 case "retract":
1500 receiveRetract(from, serverMessageId, timestamp);
1501 break;
1502 case "reject":
1503 receiveReject(from, serverMessageId, timestamp);
1504 break;
1505 case "accept":
1506 receiveAccept(from, serverMessageId, timestamp);
1507 break;
1508 default:
1509 break;
1510 }
1511 }
1512
1513 void deliverFailedProceed(final String message) {
1514 Log.d(
1515 Config.LOGTAG,
1516 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1517 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1518 webRTCWrapper.close();
1519 Log.d(
1520 Config.LOGTAG,
1521 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1522 this.finish();
1523 }
1524 }
1525
1526 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1527 final boolean originatedFromMyself =
1528 from.asBareJid().equals(id.account.getJid().asBareJid());
1529 if (originatedFromMyself) {
1530 if (transition(State.ACCEPTED)) {
1531 acceptedOnOtherDevice(serverMsgId, timestamp);
1532 } else {
1533 Log.d(
1534 Config.LOGTAG,
1535 id.account.getJid().asBareJid()
1536 + ": unable to transition to accept because already in state="
1537 + this.state);
1538 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1539 }
1540 } else {
1541 Log.d(
1542 Config.LOGTAG,
1543 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1544 }
1545 }
1546
1547 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1548 if (serverMsgId != null) {
1549 this.message.setServerMsgId(serverMsgId);
1550 }
1551 this.message.setTime(timestamp);
1552 this.message.setCarbon(true); // indicate that call was accepted on other device
1553 this.writeLogMessageSuccess(0);
1554 this.xmppConnectionService
1555 .getNotificationService()
1556 .cancelIncomingCallNotification();
1557 this.finish();
1558 }
1559
1560 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1561 final boolean originatedFromMyself =
1562 from.asBareJid().equals(id.account.getJid().asBareJid());
1563 // reject from another one of my clients
1564 if (originatedFromMyself) {
1565 receiveRejectFromMyself(serverMsgId, timestamp);
1566 } else if (isInitiator()) {
1567 if (from.equals(id.with)) {
1568 receiveRejectFromResponder();
1569 } else {
1570 Log.d(
1571 Config.LOGTAG,
1572 id.account.getJid()
1573 + ": ignoring reject from "
1574 + from
1575 + " for session with "
1576 + id.with);
1577 }
1578 } else {
1579 Log.d(
1580 Config.LOGTAG,
1581 id.account.getJid()
1582 + ": ignoring reject from "
1583 + from
1584 + " for session with "
1585 + id.with);
1586 }
1587 }
1588
1589 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1590 if (transition(State.REJECTED)) {
1591 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1592 this.finish();
1593 if (serverMsgId != null) {
1594 this.message.setServerMsgId(serverMsgId);
1595 }
1596 this.message.setTime(timestamp);
1597 this.message.setCarbon(true); // indicate that call was rejected on other device
1598 writeLogMessageMissed();
1599 } else {
1600 Log.d(
1601 Config.LOGTAG,
1602 "not able to transition into REJECTED because already in " + this.state);
1603 }
1604 }
1605
1606 private void receiveRejectFromResponder() {
1607 if (isInState(State.PROCEED)) {
1608 Log.d(
1609 Config.LOGTAG,
1610 id.account.getJid()
1611 + ": received reject while still in proceed. callee reconsidered");
1612 closeTransitionLogFinish(State.REJECTED_RACED);
1613 return;
1614 }
1615 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1616 Log.d(
1617 Config.LOGTAG,
1618 id.account.getJid()
1619 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1620 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1621 return;
1622 }
1623 Log.d(
1624 Config.LOGTAG,
1625 id.account.getJid()
1626 + ": ignoring reject from responder because already in state "
1627 + this.state);
1628 }
1629
1630 private void receivePropose(
1631 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1632 final boolean originatedFromMyself =
1633 from.asBareJid().equals(id.account.getJid().asBareJid());
1634 if (originatedFromMyself) {
1635 Log.d(
1636 Config.LOGTAG,
1637 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1638 } else if (transition(
1639 State.PROPOSED,
1640 () -> {
1641 final Collection<RtpDescription> descriptions =
1642 Collections2.transform(
1643 Collections2.filter(
1644 propose.getDescriptions(),
1645 d -> d instanceof RtpDescription),
1646 input -> (RtpDescription) input);
1647 final Collection<Media> media =
1648 Collections2.transform(descriptions, RtpDescription::getMedia);
1649 Preconditions.checkState(
1650 !media.contains(Media.UNKNOWN),
1651 "RTP descriptions contain unknown media");
1652 Log.d(
1653 Config.LOGTAG,
1654 id.account.getJid().asBareJid()
1655 + ": received session proposal from "
1656 + from
1657 + " for "
1658 + media);
1659 this.proposedMedia = Sets.newHashSet(media);
1660 })) {
1661 if (serverMsgId != null) {
1662 this.message.setServerMsgId(serverMsgId);
1663 }
1664 this.message.setTime(timestamp);
1665 startRinging();
1666 } else {
1667 Log.d(
1668 Config.LOGTAG,
1669 id.account.getJid()
1670 + ": ignoring session proposal because already in "
1671 + state);
1672 }
1673 }
1674
1675 private void startRinging() {
1676 Log.d(
1677 Config.LOGTAG,
1678 id.account.getJid().asBareJid()
1679 + ": received call from "
1680 + id.with
1681 + ". start ringing");
1682 ringingTimeoutFuture =
1683 jingleConnectionManager.schedule(
1684 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1685 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1686 }
1687
1688 private synchronized void ringingTimeout() {
1689 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1690 switch (this.state) {
1691 case PROPOSED:
1692 message.markUnread();
1693 rejectCallFromProposed();
1694 break;
1695 case SESSION_INITIALIZED:
1696 message.markUnread();
1697 rejectCallFromSessionInitiate();
1698 break;
1699 }
1700 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1701 }
1702
1703 private void cancelRingingTimeout() {
1704 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1705 if (future != null && !future.isCancelled()) {
1706 future.cancel(false);
1707 }
1708 }
1709
1710 private void receiveProceed(
1711 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1712 final Set<Media> media =
1713 Preconditions.checkNotNull(
1714 this.proposedMedia, "Proposed media has to be set before handling proceed");
1715 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1716 if (from.equals(id.with)) {
1717 if (isInitiator()) {
1718 if (transition(State.PROCEED)) {
1719 if (serverMsgId != null) {
1720 this.message.setServerMsgId(serverMsgId);
1721 }
1722 this.message.setTime(timestamp);
1723 final Integer remoteDeviceId = proceed.getDeviceId();
1724 if (isOmemoEnabled()) {
1725 this.omemoVerification.setDeviceId(remoteDeviceId);
1726 } else {
1727 if (remoteDeviceId != null) {
1728 Log.d(
1729 Config.LOGTAG,
1730 id.account.getJid().asBareJid()
1731 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1732 }
1733 this.omemoVerification.setDeviceId(null);
1734 }
1735 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1736 } else {
1737 Log.d(
1738 Config.LOGTAG,
1739 String.format(
1740 "%s: ignoring proceed because already in %s",
1741 id.account.getJid().asBareJid(), this.state));
1742 }
1743 } else {
1744 Log.d(
1745 Config.LOGTAG,
1746 String.format(
1747 "%s: ignoring proceed because we were not initializing",
1748 id.account.getJid().asBareJid()));
1749 }
1750 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1751 if (transition(State.ACCEPTED)) {
1752 Log.d(
1753 Config.LOGTAG,
1754 id.account.getJid().asBareJid()
1755 + ": moved session with "
1756 + id.with
1757 + " into state accepted after received carbon copied proceed");
1758 acceptedOnOtherDevice(serverMsgId, timestamp);
1759 }
1760 } else {
1761 Log.d(
1762 Config.LOGTAG,
1763 String.format(
1764 "%s: ignoring proceed from %s. was expected from %s",
1765 id.account.getJid().asBareJid(), from, id.with));
1766 }
1767 }
1768
1769 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1770 if (from.equals(id.with)) {
1771 final State target =
1772 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1773 if (transition(target)) {
1774 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1775 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1776 Log.d(
1777 Config.LOGTAG,
1778 id.account.getJid().asBareJid()
1779 + ": session with "
1780 + id.with
1781 + " has been retracted (serverMsgId="
1782 + serverMsgId
1783 + ")");
1784 if (serverMsgId != null) {
1785 this.message.setServerMsgId(serverMsgId);
1786 }
1787 this.message.setTime(timestamp);
1788 if (target == State.RETRACTED) {
1789 this.message.markUnread();
1790 }
1791 writeLogMessageMissed();
1792 finish();
1793 } else {
1794 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1795 }
1796 } else {
1797 // TODO parse retract from self
1798 Log.d(
1799 Config.LOGTAG,
1800 id.account.getJid().asBareJid()
1801 + ": received retract from "
1802 + from
1803 + ". expected retract from"
1804 + id.with
1805 + ". ignoring");
1806 }
1807 }
1808
1809 public void sendSessionInitiate() {
1810 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1811 }
1812
1813 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1814 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1815 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1816 }
1817
1818 private synchronized void sendSessionInitiate(
1819 final Set<Media> media,
1820 final State targetState,
1821 final List<PeerConnection.IceServer> iceServers) {
1822 if (isTerminated()) {
1823 Log.w(
1824 Config.LOGTAG,
1825 id.account.getJid().asBareJid()
1826 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1827 return;
1828 }
1829 try {
1830 setupWebRTC(media, iceServers);
1831 } catch (final WebRTCWrapper.InitializationException e) {
1832 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1833 webRTCWrapper.close();
1834 sendRetract(Reason.ofThrowable(e));
1835 return;
1836 }
1837 try {
1838 org.webrtc.SessionDescription webRTCSessionDescription =
1839 this.webRTCWrapper.setLocalDescription().get();
1840 prepareSessionInitiate(webRTCSessionDescription, targetState);
1841 } catch (final Exception e) {
1842 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1843 // exceptions
1844 failureToInitiateSession(e, targetState);
1845 }
1846 }
1847
1848 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1849 if (isTerminated()) {
1850 return;
1851 }
1852 Log.d(
1853 Config.LOGTAG,
1854 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1855 Throwables.getRootCause(throwable));
1856 webRTCWrapper.close();
1857 final Reason reason = Reason.ofThrowable(throwable);
1858 if (isInState(targetState)) {
1859 sendSessionTerminate(reason, throwable.getMessage());
1860 } else {
1861 sendRetract(reason);
1862 }
1863 }
1864
1865 private void sendRetract(final Reason reason) {
1866 // TODO embed reason into retract
1867 sendJingleMessage("retract", id.with.asBareJid());
1868 transitionOrThrow(reasonToState(reason));
1869 this.finish();
1870 }
1871
1872 private void prepareSessionInitiate(
1873 final org.webrtc.SessionDescription initialWebRTCSessionDescription, final State targetState) {
1874 Futures.addCallback(
1875 webRTCWrapper.getRFC3264() ? Futures.withTimeout(iceGatheringComplete, 2, TimeUnit.SECONDS, JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE) : Futures.immediateFuture(null),
1876 new FutureCallback<Collection<IceCandidate>>() {
1877 @Override
1878 public void onSuccess(final Collection<IceCandidate> iceCandidates) {
1879 org.webrtc.SessionDescription webRTCSessionDescription =
1880 JingleRtpConnection.this.webRTCWrapper.getLocalDescription();
1881 final SessionDescription sessionDescription =
1882 SessionDescription.parse(webRTCSessionDescription.description);
1883 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1884 JingleRtpConnection.this.initiatorRtpContentMap = rtpContentMap;
1885 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1886 encryptSessionInitiate(rtpContentMap);
1887 Futures.addCallback(
1888 outgoingContentMapFuture,
1889 new FutureCallback<RtpContentMap>() {
1890 @Override
1891 public void onSuccess(final RtpContentMap outgoingContentMap) {
1892 sendSessionInitiate(outgoingContentMap, targetState);
1893 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1894 }
1895
1896 @Override
1897 public void onFailure(@NonNull final Throwable throwable) {
1898 failureToInitiateSession(throwable, targetState);
1899 }
1900 },
1901 MoreExecutors.directExecutor()
1902 );
1903 }
1904
1905 @Override
1906 public void onFailure(@NonNull final Throwable throwable) {
1907 Log.e(Config.LOGTAG, "ICE gathering didn't finish clean: " + throwable);
1908 onSuccess(null);
1909 }
1910 },
1911 MoreExecutors.directExecutor()
1912 );
1913 }
1914
1915 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1916 if (isTerminated()) {
1917 Log.w(
1918 Config.LOGTAG,
1919 id.account.getJid().asBareJid()
1920 + ": preparing session was too slow. already terminated. nothing to do.");
1921 return;
1922 }
1923 this.transitionOrThrow(targetState);
1924 final JinglePacket sessionInitiate =
1925 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1926 send(sessionInitiate);
1927 }
1928
1929 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1930 final RtpContentMap rtpContentMap) {
1931 if (this.omemoVerification.hasDeviceId()) {
1932 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1933 verifiedPayloadFuture =
1934 id.account
1935 .getAxolotlService()
1936 .encrypt(
1937 rtpContentMap,
1938 id.with,
1939 omemoVerification.getDeviceId());
1940 final ListenableFuture<RtpContentMap> future =
1941 Futures.transform(
1942 verifiedPayloadFuture,
1943 verifiedPayload -> {
1944 omemoVerification.setSessionFingerprint(
1945 verifiedPayload.getFingerprint());
1946 return verifiedPayload.getPayload();
1947 },
1948 MoreExecutors.directExecutor());
1949 if (Config.REQUIRE_RTP_VERIFICATION) {
1950 return future;
1951 }
1952 return Futures.catching(
1953 future,
1954 CryptoFailedException.class,
1955 e -> {
1956 Log.w(
1957 Config.LOGTAG,
1958 id.account.getJid().asBareJid()
1959 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1960 e);
1961 return rtpContentMap;
1962 },
1963 MoreExecutors.directExecutor());
1964 } else {
1965 return Futures.immediateFuture(rtpContentMap);
1966 }
1967 }
1968
1969 private void sendSessionTerminate(final Reason reason) {
1970 sendSessionTerminate(reason, null);
1971 }
1972
1973 private void sendSessionTerminate(final Reason reason, final String text) {
1974 final State previous = this.state;
1975 final State target = reasonToState(reason);
1976 transitionOrThrow(target);
1977 if (previous != State.NULL) {
1978 writeLogMessage(target);
1979 }
1980 final JinglePacket jinglePacket =
1981 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1982 jinglePacket.setReason(reason, text);
1983 Log.d(Config.LOGTAG, jinglePacket.toString());
1984 send(jinglePacket);
1985 finish();
1986 }
1987
1988 private void sendTransportInfo(
1989 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1990 final RtpContentMap transportInfo;
1991 try {
1992 final RtpContentMap rtpContentMap =
1993 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1994 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1995 } catch (final Exception e) {
1996 Log.d(
1997 Config.LOGTAG,
1998 id.account.getJid().asBareJid()
1999 + ": unable to prepare transport-info from candidate for content="
2000 + contentName);
2001 return;
2002 }
2003 final JinglePacket jinglePacket =
2004 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2005 send(jinglePacket);
2006 }
2007
2008 private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
2009 // TODO send all candidates in one transport-info
2010 }
2011
2012
2013 private void send(final JinglePacket jinglePacket) {
2014 jinglePacket.setTo(id.with);
2015 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
2016 }
2017
2018 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
2019 if (response.getType() == IqPacket.TYPE.ERROR) {
2020 handleIqErrorResponse(response);
2021 return;
2022 }
2023 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2024 handleIqTimeoutResponse(response);
2025 }
2026 }
2027
2028 private void handleIqErrorResponse(final IqPacket response) {
2029 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
2030 final String errorCondition = response.getErrorCondition();
2031 Log.d(
2032 Config.LOGTAG,
2033 id.account.getJid().asBareJid()
2034 + ": received IQ-error from "
2035 + response.getFrom()
2036 + " in RTP session. "
2037 + errorCondition);
2038 if (isTerminated()) {
2039 Log.i(
2040 Config.LOGTAG,
2041 id.account.getJid().asBareJid()
2042 + ": ignoring error because session was already terminated");
2043 return;
2044 }
2045 this.webRTCWrapper.close();
2046 final State target;
2047 if (Arrays.asList(
2048 "service-unavailable",
2049 "recipient-unavailable",
2050 "remote-server-not-found",
2051 "remote-server-timeout")
2052 .contains(errorCondition)) {
2053 target = State.TERMINATED_CONNECTIVITY_ERROR;
2054 } else {
2055 target = State.TERMINATED_APPLICATION_FAILURE;
2056 }
2057 transitionOrThrow(target);
2058 this.finish();
2059 }
2060
2061 private void handleIqTimeoutResponse(final IqPacket response) {
2062 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
2063 Log.d(
2064 Config.LOGTAG,
2065 id.account.getJid().asBareJid()
2066 + ": received IQ timeout in RTP session with "
2067 + id.with
2068 + ". terminating with connectivity error");
2069 if (isTerminated()) {
2070 Log.i(
2071 Config.LOGTAG,
2072 id.account.getJid().asBareJid()
2073 + ": ignoring error because session was already terminated");
2074 return;
2075 }
2076 this.webRTCWrapper.close();
2077 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
2078 this.finish();
2079 }
2080
2081 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
2082 Log.d(
2083 Config.LOGTAG,
2084 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
2085 this.webRTCWrapper.close();
2086 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
2087 respondWithOutOfOrder(jinglePacket);
2088 this.finish();
2089 }
2090
2091 private void respondWithTieBreak(final JinglePacket jinglePacket) {
2092 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
2093 }
2094
2095 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
2096 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
2097 }
2098
2099 private void respondWithItemNotFound(final JinglePacket jinglePacket) {
2100 respondWithJingleError(jinglePacket, null, "item-not-found", "cancel");
2101 }
2102
2103 void respondWithJingleError(
2104 final IqPacket original,
2105 String jingleCondition,
2106 String condition,
2107 String conditionType) {
2108 jingleConnectionManager.respondWithJingleError(
2109 id.account, original, jingleCondition, condition, conditionType);
2110 }
2111
2112 private void respondOk(final JinglePacket jinglePacket) {
2113 xmppConnectionService.sendIqPacket(
2114 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
2115 }
2116
2117 public RtpEndUserState getEndUserState() {
2118 switch (this.state) {
2119 case NULL:
2120 case PROPOSED:
2121 case SESSION_INITIALIZED:
2122 if (isInitiator()) {
2123 return RtpEndUserState.RINGING;
2124 } else {
2125 return RtpEndUserState.INCOMING_CALL;
2126 }
2127 case PROCEED:
2128 if (isInitiator()) {
2129 return RtpEndUserState.RINGING;
2130 } else {
2131 return RtpEndUserState.ACCEPTING_CALL;
2132 }
2133 case SESSION_INITIALIZED_PRE_APPROVED:
2134 if (isInitiator()) {
2135 return RtpEndUserState.RINGING;
2136 } else {
2137 return RtpEndUserState.CONNECTING;
2138 }
2139 case SESSION_ACCEPTED:
2140 final ContentAddition ca = getPendingContentAddition();
2141 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2142 return RtpEndUserState.INCOMING_CONTENT_ADD;
2143 }
2144 return getPeerConnectionStateAsEndUserState();
2145 case REJECTED:
2146 case REJECTED_RACED:
2147 case TERMINATED_DECLINED_OR_BUSY:
2148 if (isInitiator()) {
2149 return RtpEndUserState.DECLINED_OR_BUSY;
2150 } else {
2151 return RtpEndUserState.ENDED;
2152 }
2153 case TERMINATED_SUCCESS:
2154 case ACCEPTED:
2155 case RETRACTED:
2156 case TERMINATED_CANCEL_OR_TIMEOUT:
2157 return RtpEndUserState.ENDED;
2158 case RETRACTED_RACED:
2159 if (isInitiator()) {
2160 return RtpEndUserState.ENDED;
2161 } else {
2162 return RtpEndUserState.RETRACTED;
2163 }
2164 case TERMINATED_CONNECTIVITY_ERROR:
2165 return zeroDuration()
2166 ? RtpEndUserState.CONNECTIVITY_ERROR
2167 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2168 case TERMINATED_APPLICATION_FAILURE:
2169 return RtpEndUserState.APPLICATION_ERROR;
2170 case TERMINATED_SECURITY_ERROR:
2171 return RtpEndUserState.SECURITY_ERROR;
2172 }
2173 throw new IllegalStateException(
2174 String.format("%s has no equivalent EndUserState", this.state));
2175 }
2176
2177 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2178 final PeerConnection.PeerConnectionState state;
2179 try {
2180 state = webRTCWrapper.getState();
2181 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2182 // We usually close the WebRTCWrapper *before* transitioning so we might still
2183 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2184 return RtpEndUserState.ENDING_CALL;
2185 }
2186 switch (state) {
2187 case CONNECTED:
2188 return RtpEndUserState.CONNECTED;
2189 case NEW:
2190 case CONNECTING:
2191 return RtpEndUserState.CONNECTING;
2192 case CLOSED:
2193 return RtpEndUserState.ENDING_CALL;
2194 default:
2195 return zeroDuration()
2196 ? RtpEndUserState.CONNECTIVITY_ERROR
2197 : RtpEndUserState.RECONNECTING;
2198 }
2199 }
2200
2201 public ContentAddition getPendingContentAddition() {
2202 final RtpContentMap in = this.incomingContentAdd;
2203 final RtpContentMap out = this.outgoingContentAdd;
2204 if (out != null) {
2205 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2206 } else if (in != null) {
2207 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2208 } else {
2209 return null;
2210 }
2211 }
2212
2213 public Set<Media> getMedia() {
2214 final State current = getState();
2215 if (current == State.NULL) {
2216 if (isInitiator()) {
2217 return Preconditions.checkNotNull(
2218 this.proposedMedia, "RTP connection has not been initialized properly");
2219 }
2220 throw new IllegalStateException("RTP connection has not been initialized yet");
2221 }
2222 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2223 return Preconditions.checkNotNull(
2224 this.proposedMedia, "RTP connection has not been initialized properly");
2225 }
2226 final RtpContentMap localContentMap = getLocalContentMap();
2227 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2228 if (localContentMap != null) {
2229 return localContentMap.getMedia();
2230 } else if (initiatorContentMap != null) {
2231 return initiatorContentMap.getMedia();
2232 } else if (isTerminated()) {
2233 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2234 } else {
2235 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2236 }
2237 }
2238
2239 public boolean isVerified() {
2240 final String fingerprint = this.omemoVerification.getFingerprint();
2241 if (fingerprint == null) {
2242 return false;
2243 }
2244 final FingerprintStatus status =
2245 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2246 return status != null && status.isVerified();
2247 }
2248
2249 public boolean addMedia(final Media media) {
2250 final Set<Media> currentMedia = getMedia();
2251 if (currentMedia.contains(media)) {
2252 throw new IllegalStateException(String.format("%s has already been proposed", media));
2253 }
2254 // TODO add state protection - can only add while ACCEPTED or so
2255 Log.d(Config.LOGTAG,"adding media: "+media);
2256 return webRTCWrapper.addTrack(media);
2257 }
2258
2259 public synchronized void acceptCall() {
2260 switch (this.state) {
2261 case PROPOSED:
2262 cancelRingingTimeout();
2263 acceptCallFromProposed();
2264 break;
2265 case SESSION_INITIALIZED:
2266 cancelRingingTimeout();
2267 acceptCallFromSessionInitialized();
2268 break;
2269 case ACCEPTED:
2270 Log.w(
2271 Config.LOGTAG,
2272 id.account.getJid().asBareJid()
2273 + ": the call has already been accepted with another client. UI was just lagging behind");
2274 break;
2275 case PROCEED:
2276 case SESSION_ACCEPTED:
2277 Log.w(
2278 Config.LOGTAG,
2279 id.account.getJid().asBareJid()
2280 + ": the call has already been accepted. user probably double tapped the UI");
2281 break;
2282 default:
2283 throw new IllegalStateException("Can not accept call from " + this.state);
2284 }
2285 }
2286
2287 public void notifyPhoneCall() {
2288 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2289 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2290 rejectCall();
2291 } else {
2292 endCall();
2293 }
2294 }
2295
2296 public synchronized void rejectCall() {
2297 if (isTerminated()) {
2298 Log.w(
2299 Config.LOGTAG,
2300 id.account.getJid().asBareJid()
2301 + ": received rejectCall() when session has already been terminated. nothing to do");
2302 return;
2303 }
2304 switch (this.state) {
2305 case PROPOSED:
2306 rejectCallFromProposed();
2307 break;
2308 case SESSION_INITIALIZED:
2309 rejectCallFromSessionInitiate();
2310 break;
2311 default:
2312 throw new IllegalStateException("Can not reject call from " + this.state);
2313 }
2314 }
2315
2316 public synchronized void endCall() {
2317 if (isTerminated()) {
2318 Log.w(
2319 Config.LOGTAG,
2320 id.account.getJid().asBareJid()
2321 + ": received endCall() when session has already been terminated. nothing to do");
2322 return;
2323 }
2324 if (isInState(State.PROPOSED) && !isInitiator()) {
2325 rejectCallFromProposed();
2326 return;
2327 }
2328 if (isInState(State.PROCEED)) {
2329 if (isInitiator()) {
2330 retractFromProceed();
2331 } else {
2332 rejectCallFromProceed();
2333 }
2334 return;
2335 }
2336 if (isInitiator()
2337 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2338 this.webRTCWrapper.close();
2339 sendSessionTerminate(Reason.CANCEL);
2340 return;
2341 }
2342 if (isInState(State.SESSION_INITIALIZED)) {
2343 rejectCallFromSessionInitiate();
2344 return;
2345 }
2346 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2347 this.webRTCWrapper.close();
2348 sendSessionTerminate(Reason.SUCCESS);
2349 return;
2350 }
2351 if (isInState(
2352 State.TERMINATED_APPLICATION_FAILURE,
2353 State.TERMINATED_CONNECTIVITY_ERROR,
2354 State.TERMINATED_DECLINED_OR_BUSY)) {
2355 Log.d(
2356 Config.LOGTAG,
2357 "ignoring request to end call because already in state " + this.state);
2358 return;
2359 }
2360 throw new IllegalStateException(
2361 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2362 }
2363
2364 private void retractFromProceed() {
2365 Log.d(Config.LOGTAG, "retract from proceed");
2366 this.sendJingleMessage("retract");
2367 closeTransitionLogFinish(State.RETRACTED_RACED);
2368 }
2369
2370 private void closeTransitionLogFinish(final State state) {
2371 this.webRTCWrapper.close();
2372 transitionOrThrow(state);
2373 writeLogMessage(state);
2374 finish();
2375 }
2376
2377 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2378 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2379 final Presence presence = id.getContact().getPresences().get(id.getWith().getResource());
2380 final ServiceDiscoveryResult disco = presence == null ? null : presence.getServiceDiscoveryResult();
2381 if (disco != null && disco.getFeatures().contains("urn:ietf:rfc:3264")) webRTCWrapper.setRFC3264(true);
2382 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2383 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2384 }
2385
2386 private void acceptCallFromProposed() {
2387 transitionOrThrow(State.PROCEED);
2388 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2389 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2390 this.sendJingleMessage("proceed");
2391 }
2392
2393 private void rejectCallFromProposed() {
2394 transitionOrThrow(State.REJECTED);
2395 writeLogMessageMissed();
2396 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2397 this.sendJingleMessage("reject");
2398 finish();
2399 }
2400
2401 private void rejectCallFromProceed() {
2402 this.sendJingleMessage("reject");
2403 closeTransitionLogFinish(State.REJECTED_RACED);
2404 }
2405
2406 private void rejectCallFromSessionInitiate() {
2407 webRTCWrapper.close();
2408 sendSessionTerminate(Reason.DECLINE);
2409 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2410 }
2411
2412 private void sendJingleMessage(final String action) {
2413 sendJingleMessage(action, id.with);
2414 }
2415
2416 private void sendJingleMessage(final String action, final Jid to) {
2417 final MessagePacket messagePacket = new MessagePacket();
2418 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2419 messagePacket.setTo(to);
2420 final Element intent =
2421 messagePacket
2422 .addChild(action, Namespace.JINGLE_MESSAGE)
2423 .setAttribute("id", id.sessionId);
2424 if ("proceed".equals(action)) {
2425 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2426 if (isOmemoEnabled()) {
2427 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2428 final Element device =
2429 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2430 device.setAttribute("id", deviceId);
2431 }
2432 }
2433 messagePacket.addChild("store", "urn:xmpp:hints");
2434 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2435 }
2436
2437 private boolean isOmemoEnabled() {
2438 final Conversational conversational = message.getConversation();
2439 if (conversational instanceof Conversation) {
2440 return ((Conversation) conversational).getNextEncryption()
2441 == Message.ENCRYPTION_AXOLOTL;
2442 }
2443 return false;
2444 }
2445
2446 private void acceptCallFromSessionInitialized() {
2447 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2448 sendSessionAccept();
2449 }
2450
2451 private synchronized boolean isInState(State... state) {
2452 return Arrays.asList(state).contains(this.state);
2453 }
2454
2455 private boolean transition(final State target) {
2456 return transition(target, null);
2457 }
2458
2459 private synchronized boolean transition(final State target, final Runnable runnable) {
2460 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2461 if (validTransitions != null && validTransitions.contains(target)) {
2462 this.state = target;
2463 if (runnable != null) {
2464 runnable.run();
2465 }
2466 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2467 updateEndUserState();
2468 updateOngoingCallNotification();
2469 return true;
2470 } else {
2471 return false;
2472 }
2473 }
2474
2475 void transitionOrThrow(final State target) {
2476 if (!transition(target)) {
2477 throw new IllegalStateException(
2478 String.format("Unable to transition from %s to %s", this.state, target));
2479 }
2480 }
2481
2482 @Override
2483 public void onIceCandidate(final IceCandidate iceCandidate) {
2484 final RtpContentMap rtpContentMap =
2485 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2486 final IceUdpTransportInfo.Credentials credentials;
2487 try {
2488 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2489 } catch (final IllegalArgumentException e) {
2490 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2491 return;
2492 }
2493 final String uFrag = credentials.ufrag;
2494 final IceUdpTransportInfo.Candidate candidate =
2495 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2496 if (candidate == null) {
2497 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2498 return;
2499 }
2500 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2501 sendTransportInfo(iceCandidate.sdpMid, candidate);
2502 }
2503
2504 @Override
2505 public void onIceGatheringComplete(Collection<IceCandidate> iceCandidates) {
2506 iceGatheringComplete.set(iceCandidates);
2507 }
2508
2509 @Override
2510 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2511 Log.d(
2512 Config.LOGTAG,
2513 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2514 this.stateHistory.add(newState);
2515 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2516 this.sessionDuration.start();
2517 updateOngoingCallNotification();
2518 } else if (this.sessionDuration.isRunning()) {
2519 this.sessionDuration.stop();
2520 updateOngoingCallNotification();
2521 }
2522
2523 final boolean neverConnected =
2524 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2525
2526 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2527 if (neverConnected) {
2528 if (isTerminated()) {
2529 Log.d(
2530 Config.LOGTAG,
2531 id.account.getJid().asBareJid()
2532 + ": not sending session-terminate after connectivity error because session is already in state "
2533 + this.state);
2534 return;
2535 }
2536 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2537 return;
2538 } else {
2539 this.restartIce();
2540 }
2541 }
2542 updateEndUserState();
2543 }
2544
2545 private void restartIce() {
2546 this.stateHistory.clear();
2547 this.webRTCWrapper.restartIceAsync();
2548 }
2549
2550 @Override
2551 public void onRenegotiationNeeded() {
2552 this.webRTCWrapper.execute(this::renegotiate);
2553 }
2554
2555 private void renegotiate() {
2556 final SessionDescription sessionDescription;
2557 try {
2558 sessionDescription = setLocalSessionDescription();
2559 } catch (final Exception e) {
2560 final Throwable cause = Throwables.getRootCause(e);
2561 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2562 webRTCWrapper.close();
2563 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2564 return;
2565 }
2566 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2567 final RtpContentMap currentContentMap = getLocalContentMap();
2568 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2569 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2570
2571 Log.d(
2572 Config.LOGTAG,
2573 id.getAccount().getJid().asBareJid()
2574 + ": renegotiate. iceRestart="
2575 + iceRestart
2576 + " content id diff="
2577 + diff);
2578
2579 if (diff.hasModifications() && iceRestart) {
2580 webRTCWrapper.close();
2581 sendSessionTerminate(
2582 Reason.FAILED_APPLICATION,
2583 "WebRTC unexpectedly tried to modify content and transport at once");
2584 return;
2585 }
2586
2587 if (iceRestart) {
2588 initiateIceRestart(rtpContentMap);
2589 return;
2590 } else if (diff.isEmpty()) {
2591 Log.d(
2592 Config.LOGTAG,
2593 "renegotiation. nothing to do. SignalingState="
2594 + this.webRTCWrapper.getSignalingState());
2595 }
2596
2597 if (diff.added.size() > 0) {
2598 modifyLocalContentMap(rtpContentMap);
2599 sendContentAdd(rtpContentMap, diff.added);
2600 }
2601 }
2602
2603 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2604 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2605 final JinglePacket jinglePacket =
2606 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2607 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2608 jinglePacket.setTo(id.with);
2609 xmppConnectionService.sendIqPacket(
2610 id.account,
2611 jinglePacket,
2612 (account, response) -> {
2613 if (response.getType() == IqPacket.TYPE.RESULT) {
2614 Log.d(Config.LOGTAG, "received success to our ice restart");
2615 setLocalContentMap(rtpContentMap);
2616 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2617 return;
2618 }
2619 if (response.getType() == IqPacket.TYPE.ERROR) {
2620 if (isTieBreak(response)) {
2621 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2622 return;
2623 }
2624 handleIqErrorResponse(response);
2625 }
2626 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2627 handleIqTimeoutResponse(response);
2628 }
2629 });
2630 }
2631
2632 private boolean isTieBreak(final IqPacket response) {
2633 final Element error = response.findChild("error");
2634 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2635 }
2636
2637 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2638 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2639 this.outgoingContentAdd = contentAdd;
2640 final JinglePacket jinglePacket =
2641 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2642 jinglePacket.setTo(id.with);
2643 xmppConnectionService.sendIqPacket(
2644 id.account,
2645 jinglePacket,
2646 (connection, response) -> {
2647 if (response.getType() == IqPacket.TYPE.RESULT) {
2648 Log.d(
2649 Config.LOGTAG,
2650 id.getAccount().getJid().asBareJid()
2651 + ": received ACK to our content-add");
2652 return;
2653 }
2654 if (response.getType() == IqPacket.TYPE.ERROR) {
2655 if (isTieBreak(response)) {
2656 this.outgoingContentAdd = null;
2657 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2658 return;
2659 }
2660 handleIqErrorResponse(response);
2661 }
2662 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2663 handleIqTimeoutResponse(response);
2664 }
2665 });
2666 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2667 }
2668
2669 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2670 if (isInitiator()) {
2671 this.initiatorRtpContentMap = rtpContentMap;
2672 } else {
2673 this.responderRtpContentMap = rtpContentMap;
2674 }
2675 }
2676
2677 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2678 if (isInitiator()) {
2679 this.responderRtpContentMap = rtpContentMap;
2680 } else {
2681 this.initiatorRtpContentMap = rtpContentMap;
2682 }
2683 }
2684
2685 // this method is to be used for content map modifications that modify media
2686 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2687 final RtpContentMap activeContents = rtpContentMap.activeContents();
2688 setLocalContentMap(activeContents);
2689 this.webRTCWrapper.switchSpeakerPhonePreference(
2690 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2691 updateEndUserState();
2692 }
2693
2694 private SessionDescription setLocalSessionDescription()
2695 throws ExecutionException, InterruptedException {
2696 final org.webrtc.SessionDescription sessionDescription =
2697 this.webRTCWrapper.setLocalDescription().get();
2698 return SessionDescription.parse(sessionDescription.description);
2699 }
2700
2701 private void closeWebRTCSessionAfterFailedConnection() {
2702 this.webRTCWrapper.close();
2703 synchronized (this) {
2704 if (isTerminated()) {
2705 Log.d(
2706 Config.LOGTAG,
2707 id.account.getJid().asBareJid()
2708 + ": no need to send session-terminate after failed connection. Other party already did");
2709 return;
2710 }
2711 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2712 }
2713 }
2714
2715 public boolean zeroDuration() {
2716 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2717 }
2718
2719 public long getCallDuration() {
2720 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2721 }
2722
2723 public AppRTCAudioManager getAudioManager() {
2724 return webRTCWrapper.getAudioManager();
2725 }
2726
2727 public boolean isMicrophoneEnabled() {
2728 return webRTCWrapper.isMicrophoneEnabled();
2729 }
2730
2731 public boolean setMicrophoneEnabled(final boolean enabled) {
2732 return webRTCWrapper.setMicrophoneEnabled(enabled);
2733 }
2734
2735 public boolean isVideoEnabled() {
2736 return webRTCWrapper.isVideoEnabled();
2737 }
2738
2739 public void setVideoEnabled(final boolean enabled) {
2740 webRTCWrapper.setVideoEnabled(enabled);
2741 }
2742
2743 public boolean isCameraSwitchable() {
2744 return webRTCWrapper.isCameraSwitchable();
2745 }
2746
2747 public boolean isFrontCamera() {
2748 return webRTCWrapper.isFrontCamera();
2749 }
2750
2751 public ListenableFuture<Boolean> switchCamera() {
2752 return webRTCWrapper.switchCamera();
2753 }
2754
2755 @Override
2756 public void onAudioDeviceChanged(
2757 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2758 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2759 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2760 selectedAudioDevice, availableAudioDevices);
2761 }
2762
2763 private void updateEndUserState() {
2764 final RtpEndUserState endUserState = getEndUserState();
2765 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2766 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2767 id.account, id.with, id.sessionId, endUserState);
2768 }
2769
2770 private void updateOngoingCallNotification() {
2771 final State state = this.state;
2772 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2773 final boolean reconnecting;
2774 if (state == State.SESSION_ACCEPTED) {
2775 reconnecting =
2776 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2777 } else {
2778 reconnecting = false;
2779 }
2780 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2781 } else {
2782 xmppConnectionService.removeOngoingCall();
2783 }
2784 }
2785
2786 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2787 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2788 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2789 request.setTo(id.account.getDomain());
2790 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2791 xmppConnectionService.sendIqPacket(
2792 id.account,
2793 request,
2794 (account, response) -> {
2795 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2796 new ImmutableList.Builder<>();
2797 if (response.getType() == IqPacket.TYPE.RESULT) {
2798 final Element services =
2799 response.findChild(
2800 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2801 final List<Element> children =
2802 services == null
2803 ? Collections.emptyList()
2804 : services.getChildren();
2805 for (final Element child : children) {
2806 if ("service".equals(child.getName())) {
2807 final String type = child.getAttribute("type");
2808 final String host = child.getAttribute("host");
2809 final String sport = child.getAttribute("port");
2810 final Integer port =
2811 sport == null ? null : Ints.tryParse(sport);
2812 final String transport = child.getAttribute("transport");
2813 final String username = child.getAttribute("username");
2814 final String password = child.getAttribute("password");
2815 if (Strings.isNullOrEmpty(host) || port == null) {
2816 continue;
2817 }
2818 if (port < 0 || port > 65535) {
2819 continue;
2820 }
2821
2822
2823
2824
2825 if (Arrays.asList("stun", "stuns", "turn", "turns")
2826 .contains(type)
2827 && Arrays.asList("udp", "tcp").contains(transport)) {
2828 if (Arrays.asList("stuns", "turns").contains(type)
2829 && "udp".equals(transport)) {
2830 Log.d(
2831 Config.LOGTAG,
2832 id.account.getJid().asBareJid()
2833 + ": skipping invalid combination of udp/tls in external services");
2834 continue;
2835 }
2836
2837 // STUN URLs do not support a query section since M110
2838 final String uri;
2839 if (Arrays.asList("stun","stuns").contains(type)) {
2840 uri = String.format("%s:%s%s", type, IP.wrapIPv6(host),port);
2841 } else {
2842 uri = String.format(
2843 "%s:%s:%s?transport=%s",
2844 type,
2845 IP.wrapIPv6(host),
2846 port,
2847 transport);
2848 }
2849
2850 final PeerConnection.IceServer.Builder iceServerBuilder =
2851 PeerConnection.IceServer.builder(uri);
2852 iceServerBuilder.setTlsCertPolicy(
2853 PeerConnection.TlsCertPolicy
2854 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2855 if (username != null && password != null) {
2856 iceServerBuilder.setUsername(username);
2857 iceServerBuilder.setPassword(password);
2858 } else if (Arrays.asList("turn", "turns").contains(type)) {
2859 // The WebRTC spec requires throwing an
2860 // InvalidAccessError when username (from libwebrtc
2861 // source coder)
2862 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2863 Log.d(
2864 Config.LOGTAG,
2865 id.account.getJid().asBareJid()
2866 + ": skipping "
2867 + type
2868 + "/"
2869 + transport
2870 + " without username and password");
2871 continue;
2872 }
2873 final PeerConnection.IceServer iceServer =
2874 iceServerBuilder.createIceServer();
2875 Log.d(
2876 Config.LOGTAG,
2877 id.account.getJid().asBareJid()
2878 + ": discovered ICE Server: "
2879 + iceServer);
2880 listBuilder.add(iceServer);
2881 }
2882 }
2883 }
2884 }
2885 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2886 if (iceServers.size() == 0) {
2887 Log.w(
2888 Config.LOGTAG,
2889 id.account.getJid().asBareJid()
2890 + ": no ICE server found "
2891 + response);
2892 }
2893 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2894 });
2895 } else {
2896 Log.w(
2897 Config.LOGTAG,
2898 id.account.getJid().asBareJid() + ": has no external service discovery");
2899 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2900 }
2901 }
2902
2903 private void finish() {
2904 if (isTerminated()) {
2905 this.cancelRingingTimeout();
2906 this.webRTCWrapper.verifyClosed();
2907 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2908 this.jingleConnectionManager.finishConnectionOrThrow(this);
2909 try {
2910 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2911 log.getParentFile().mkdirs();
2912 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2913 } catch (final IOException e) { }
2914 } else {
2915 throw new IllegalStateException(
2916 String.format("Unable to call finish from %s", this.state));
2917 }
2918 }
2919
2920 private void writeLogMessage(final State state) {
2921 final long duration = getCallDuration();
2922 if (state == State.TERMINATED_SUCCESS
2923 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2924 writeLogMessageSuccess(duration);
2925 } else {
2926 writeLogMessageMissed();
2927 }
2928 }
2929
2930 private void writeLogMessageSuccess(final long duration) {
2931 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2932 this.writeMessage();
2933 }
2934
2935 private void writeLogMessageMissed() {
2936 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2937 this.writeMessage();
2938 }
2939
2940 private void writeMessage() {
2941 final Conversational conversational = message.getConversation();
2942 if (conversational instanceof Conversation) {
2943 ((Conversation) conversational).add(this.message);
2944 xmppConnectionService.createMessageAsync(message);
2945 xmppConnectionService.updateConversationUi();
2946 } else {
2947 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2948 }
2949 }
2950
2951 public State getState() {
2952 return this.state;
2953 }
2954
2955 boolean isTerminated() {
2956 return TERMINATED.contains(this.state);
2957 }
2958
2959 public Optional<VideoTrack> getLocalVideoTrack() {
2960 return webRTCWrapper.getLocalVideoTrack();
2961 }
2962
2963 public Optional<VideoTrack> getRemoteVideoTrack() {
2964 return webRTCWrapper.getRemoteVideoTrack();
2965 }
2966
2967 public EglBase.Context getEglBaseContext() {
2968 return webRTCWrapper.getEglBaseContext();
2969 }
2970
2971 void setProposedMedia(final Set<Media> media) {
2972 this.proposedMedia = media;
2973 }
2974
2975 public void fireStateUpdate() {
2976 final RtpEndUserState endUserState = getEndUserState();
2977 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2978 id.account, id.with, id.sessionId, endUserState);
2979 }
2980
2981 public boolean isSwitchToVideoAvailable() {
2982 final boolean prerequisite =
2983 Media.audioOnly(getMedia())
2984 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2985 .contains(getEndUserState());
2986 return prerequisite && remoteHasVideoFeature();
2987 }
2988
2989 private boolean remoteHasVideoFeature() {
2990 final Contact contact = id.getContact();
2991 final Presence presence =
2992 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2993 final ServiceDiscoveryResult serviceDiscoveryResult =
2994 presence == null ? null : presence.getServiceDiscoveryResult();
2995 final List<String> features =
2996 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
2997 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
2998 }
2999
3000 private interface OnIceServersDiscovered {
3001 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
3002 }
3003}