1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4import android.os.Environment;
5
6import androidx.annotation.NonNull;
7import androidx.annotation.Nullable;
8
9import com.google.common.base.Joiner;
10import com.google.common.base.Optional;
11import com.google.common.base.Preconditions;
12import com.google.common.base.Stopwatch;
13import com.google.common.base.Strings;
14import com.google.common.base.Throwables;
15import com.google.common.collect.Collections2;
16import com.google.common.collect.ImmutableList;
17import com.google.common.collect.ImmutableMap;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Multimap;
23import com.google.common.collect.Sets;
24import com.google.common.primitives.Ints;
25import com.google.common.util.concurrent.FutureCallback;
26import com.google.common.util.concurrent.Futures;
27import com.google.common.util.concurrent.ListenableFuture;
28import com.google.common.util.concurrent.MoreExecutors;
29import com.google.common.util.concurrent.SettableFuture;
30
31import org.webrtc.DtmfSender;
32import org.webrtc.EglBase;
33import org.webrtc.IceCandidate;
34import org.webrtc.PeerConnection;
35import org.webrtc.VideoTrack;
36
37import java.io.File;
38import java.io.IOException;
39import java.util.Arrays;
40import java.util.Collection;
41import java.util.Collections;
42import java.util.LinkedList;
43import java.util.List;
44import java.util.Map;
45import java.util.Queue;
46import java.util.Set;
47import java.util.concurrent.ExecutionException;
48import java.util.concurrent.ScheduledFuture;
49import java.util.concurrent.TimeUnit;
50
51import eu.siacs.conversations.BuildConfig;
52import eu.siacs.conversations.Config;
53import eu.siacs.conversations.crypto.axolotl.AxolotlService;
54import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
55import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
56import eu.siacs.conversations.entities.Account;
57import eu.siacs.conversations.entities.Contact;
58import eu.siacs.conversations.entities.Conversation;
59import eu.siacs.conversations.entities.Conversational;
60import eu.siacs.conversations.entities.Message;
61import eu.siacs.conversations.entities.Presence;
62import eu.siacs.conversations.entities.RtpSessionStatus;
63import eu.siacs.conversations.entities.ServiceDiscoveryResult;
64import eu.siacs.conversations.services.AppRTCAudioManager;
65import eu.siacs.conversations.utils.IP;
66import eu.siacs.conversations.xml.Element;
67import eu.siacs.conversations.xml.Namespace;
68import eu.siacs.conversations.xmpp.Jid;
69import eu.siacs.conversations.xmpp.OnIqPacketReceived;
70import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
71import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
72import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
73import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
74import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
75import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
76import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
77import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
78import eu.siacs.conversations.xmpp.stanzas.IqPacket;
79import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
80
81public class JingleRtpConnection extends AbstractJingleConnection
82 implements WebRTCWrapper.EventCallback {
83
84 public static final List<State> STATES_SHOWING_ONGOING_CALL =
85 Arrays.asList(
86 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
87 private static final long BUSY_TIME_OUT = 30;
88 private static final List<State> TERMINATED =
89 Arrays.asList(
90 State.ACCEPTED,
91 State.REJECTED,
92 State.REJECTED_RACED,
93 State.RETRACTED,
94 State.RETRACTED_RACED,
95 State.TERMINATED_SUCCESS,
96 State.TERMINATED_DECLINED_OR_BUSY,
97 State.TERMINATED_CONNECTIVITY_ERROR,
98 State.TERMINATED_CANCEL_OR_TIMEOUT,
99 State.TERMINATED_APPLICATION_FAILURE,
100 State.TERMINATED_SECURITY_ERROR);
101
102 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
103
104 static {
105 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
106 new ImmutableMap.Builder<>();
107 transitionBuilder.put(
108 State.NULL,
109 ImmutableList.of(
110 State.PROPOSED,
111 State.SESSION_INITIALIZED,
112 State.TERMINATED_APPLICATION_FAILURE,
113 State.TERMINATED_SECURITY_ERROR));
114 transitionBuilder.put(
115 State.PROPOSED,
116 ImmutableList.of(
117 State.ACCEPTED,
118 State.PROCEED,
119 State.REJECTED,
120 State.RETRACTED,
121 State.TERMINATED_APPLICATION_FAILURE,
122 State.TERMINATED_SECURITY_ERROR,
123 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
124 // rebinds
125 ));
126 transitionBuilder.put(
127 State.PROCEED,
128 ImmutableList.of(
129 State.REJECTED_RACED,
130 State.RETRACTED_RACED,
131 State.SESSION_INITIALIZED_PRE_APPROVED,
132 State.TERMINATED_SUCCESS,
133 State.TERMINATED_APPLICATION_FAILURE,
134 State.TERMINATED_SECURITY_ERROR,
135 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
136 // bounces of the proceed message
137 ));
138 transitionBuilder.put(
139 State.SESSION_INITIALIZED,
140 ImmutableList.of(
141 State.SESSION_ACCEPTED,
142 State.TERMINATED_SUCCESS,
143 State.TERMINATED_DECLINED_OR_BUSY,
144 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
145 // and IQ timeouts
146 State.TERMINATED_CANCEL_OR_TIMEOUT,
147 State.TERMINATED_APPLICATION_FAILURE,
148 State.TERMINATED_SECURITY_ERROR));
149 transitionBuilder.put(
150 State.SESSION_INITIALIZED_PRE_APPROVED,
151 ImmutableList.of(
152 State.SESSION_ACCEPTED,
153 State.TERMINATED_SUCCESS,
154 State.TERMINATED_DECLINED_OR_BUSY,
155 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
156 // and IQ timeouts
157 State.TERMINATED_CANCEL_OR_TIMEOUT,
158 State.TERMINATED_APPLICATION_FAILURE,
159 State.TERMINATED_SECURITY_ERROR));
160 transitionBuilder.put(
161 State.SESSION_ACCEPTED,
162 ImmutableList.of(
163 State.TERMINATED_SUCCESS,
164 State.TERMINATED_DECLINED_OR_BUSY,
165 State.TERMINATED_CONNECTIVITY_ERROR,
166 State.TERMINATED_CANCEL_OR_TIMEOUT,
167 State.TERMINATED_APPLICATION_FAILURE,
168 State.TERMINATED_SECURITY_ERROR));
169 VALID_TRANSITIONS = transitionBuilder.build();
170 }
171
172 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
173 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
174 pendingIceCandidates = new LinkedList<>();
175 private final OmemoVerification omemoVerification = new OmemoVerification();
176 private final Message message;
177 private State state = State.NULL;
178 private Set<Media> proposedMedia;
179 private RtpContentMap initiatorRtpContentMap;
180 private RtpContentMap responderRtpContentMap;
181 private RtpContentMap incomingContentAdd;
182 private RtpContentMap outgoingContentAdd;
183 private IceUdpTransportInfo.Setup peerDtlsSetup;
184 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
185 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
186 private ScheduledFuture<?> ringingTimeoutFuture;
187 private final long created = System.currentTimeMillis() / 1000L;
188 private final SettableFuture<Collection<IceCandidate>> iceGatheringComplete = SettableFuture.create();
189
190 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
191 super(jingleConnectionManager, id, initiator);
192 final Conversation conversation =
193 jingleConnectionManager
194 .getXmppConnectionService()
195 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
196 this.message =
197 new Message(
198 conversation,
199 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
200 Message.TYPE_RTP_SESSION,
201 id.sessionId);
202 }
203
204 private static State reasonToState(Reason reason) {
205 switch (reason) {
206 case SUCCESS:
207 return State.TERMINATED_SUCCESS;
208 case DECLINE:
209 case BUSY:
210 return State.TERMINATED_DECLINED_OR_BUSY;
211 case CANCEL:
212 case TIMEOUT:
213 return State.TERMINATED_CANCEL_OR_TIMEOUT;
214 case SECURITY_ERROR:
215 return State.TERMINATED_SECURITY_ERROR;
216 case FAILED_APPLICATION:
217 case UNSUPPORTED_TRANSPORTS:
218 case UNSUPPORTED_APPLICATIONS:
219 return State.TERMINATED_APPLICATION_FAILURE;
220 default:
221 return State.TERMINATED_CONNECTIVITY_ERROR;
222 }
223 }
224
225 @Override
226 synchronized void deliverPacket(final JinglePacket jinglePacket) {
227 switch (jinglePacket.getAction()) {
228 case SESSION_INITIATE:
229 receiveSessionInitiate(jinglePacket);
230 break;
231 case TRANSPORT_INFO:
232 receiveTransportInfo(jinglePacket);
233 break;
234 case SESSION_ACCEPT:
235 receiveSessionAccept(jinglePacket);
236 break;
237 case SESSION_TERMINATE:
238 receiveSessionTerminate(jinglePacket);
239 break;
240 case CONTENT_ADD:
241 receiveContentAdd(jinglePacket);
242 break;
243 case CONTENT_ACCEPT:
244 receiveContentAccept(jinglePacket);
245 break;
246 case CONTENT_REJECT:
247 receiveContentReject(jinglePacket);
248 break;
249 case CONTENT_REMOVE:
250 receiveContentRemove(jinglePacket);
251 break;
252 case CONTENT_MODIFY:
253 receiveContentModify(jinglePacket);
254 break;
255 default:
256 respondOk(jinglePacket);
257 Log.d(
258 Config.LOGTAG,
259 String.format(
260 "%s: received unhandled jingle action %s",
261 id.account.getJid().asBareJid(), jinglePacket.getAction()));
262 break;
263 }
264 }
265
266 @Override
267 synchronized void notifyRebound() {
268 if (isTerminated()) {
269 return;
270 }
271 webRTCWrapper.close();
272 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
273 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
274 }
275 if (isInState(
276 State.SESSION_INITIALIZED,
277 State.SESSION_INITIALIZED_PRE_APPROVED,
278 State.SESSION_ACCEPTED)) {
279 // we might have already changed resources (full jid) at this point; so this might not
280 // even reach the other party
281 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
282 } else {
283 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
284 finish();
285 }
286 }
287
288 public boolean applyDtmfTone(String tone) {
289 return webRTCWrapper.applyDtmfTone(tone);
290 }
291
292 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
293 respondOk(jinglePacket);
294 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
295 final State previous = this.state;
296 Log.d(
297 Config.LOGTAG,
298 id.account.getJid().asBareJid()
299 + ": received session terminate reason="
300 + wrapper.reason
301 + "("
302 + Strings.nullToEmpty(wrapper.text)
303 + ") while in state "
304 + previous);
305 if (TERMINATED.contains(previous)) {
306 Log.d(
307 Config.LOGTAG,
308 id.account.getJid().asBareJid()
309 + ": ignoring session terminate because already in "
310 + previous);
311 return;
312 }
313 webRTCWrapper.close();
314 final State target = reasonToState(wrapper.reason);
315 transitionOrThrow(target);
316 writeLogMessage(target);
317 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
318 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
319 }
320 finish();
321 }
322
323 private void receiveTransportInfo(final JinglePacket jinglePacket) {
324 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
325 // INITIALIZED only after transport-info has been received
326 if (isInState(
327 State.NULL,
328 State.PROCEED,
329 State.SESSION_INITIALIZED,
330 State.SESSION_INITIALIZED_PRE_APPROVED,
331 State.SESSION_ACCEPTED)) {
332 final RtpContentMap contentMap;
333 try {
334 contentMap = RtpContentMap.of(jinglePacket);
335 } catch (final IllegalArgumentException | NullPointerException e) {
336 Log.d(
337 Config.LOGTAG,
338 id.account.getJid().asBareJid()
339 + ": improperly formatted contents; ignoring",
340 e);
341 respondOk(jinglePacket);
342 return;
343 }
344 receiveTransportInfo(jinglePacket, contentMap);
345 } else {
346 if (isTerminated()) {
347 respondOk(jinglePacket);
348 Log.d(
349 Config.LOGTAG,
350 id.account.getJid().asBareJid()
351 + ": ignoring out-of-order transport info; we where already terminated");
352 } else {
353 Log.d(
354 Config.LOGTAG,
355 id.account.getJid().asBareJid()
356 + ": received transport info while in state="
357 + this.state);
358 terminateWithOutOfOrder(jinglePacket);
359 }
360 }
361 }
362
363 private void receiveTransportInfo(
364 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
365 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
366 contentMap.contents.entrySet();
367 final RtpContentMap remote = getRemoteContentMap();
368 final Set<String> remoteContentIds = remote == null ? Collections.emptySet() : remote.contents.keySet();
369 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
370 Log.d(Config.LOGTAG,"received transport-info for unknown contents "+contentMap.contents.keySet()+" (known: "+remoteContentIds+")");
371 respondOk(jinglePacket);
372 pendingIceCandidates.addAll(candidates);
373 return;
374 }
375 if (this.state != State.SESSION_ACCEPTED) {
376 Log.d(Config.LOGTAG,"received transport-info prematurely. adding to backlog");
377 respondOk(jinglePacket);
378 pendingIceCandidates.addAll(candidates);
379 return;
380 }
381 // zero candidates + modified credentials are an ICE restart offer
382 if (checkForIceRestart(jinglePacket, contentMap)) {
383 return;
384 }
385 respondOk(jinglePacket);
386 try {
387 processCandidates(candidates);
388 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
389 Log.w(
390 Config.LOGTAG,
391 id.account.getJid().asBareJid()
392 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
393 }
394 }
395
396 private void receiveContentAdd(final JinglePacket jinglePacket) {
397 final RtpContentMap modification;
398 try {
399 modification = RtpContentMap.of(jinglePacket);
400 modification.requireContentDescriptions();
401 } catch (final RuntimeException e) {
402 Log.d(
403 Config.LOGTAG,
404 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
405 Throwables.getRootCause(e));
406 respondOk(jinglePacket);
407 webRTCWrapper.close();
408 sendSessionTerminate(Reason.of(e), e.getMessage());
409 return;
410 }
411 if (isInState(State.SESSION_ACCEPTED)) {
412 receiveContentAdd(jinglePacket, modification);
413 } else {
414 terminateWithOutOfOrder(jinglePacket);
415 }
416 }
417
418 private void receiveContentAdd(
419 final JinglePacket jinglePacket, final RtpContentMap modification) {
420 final RtpContentMap remote = getRemoteContentMap();
421 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
422 respondOk(jinglePacket);
423 this.webRTCWrapper.close();
424 sendSessionTerminate(
425 Reason.FAILED_APPLICATION,
426 String.format(
427 "contents with names %s already exists",
428 Joiner.on(", ").join(modification.getNames())));
429 return;
430 }
431 final ContentAddition contentAddition =
432 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
433
434 final RtpContentMap outgoing = this.outgoingContentAdd;
435 final Set<ContentAddition.Summary> outgoingContentAddSummary =
436 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
437
438 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
439 if (isInitiator()) {
440 Log.d(
441 Config.LOGTAG,
442 id.getAccount().getJid().asBareJid()
443 + ": respond with tie break to matching content-add offer");
444 respondWithTieBreak(jinglePacket);
445 } else {
446 Log.d(
447 Config.LOGTAG,
448 id.getAccount().getJid().asBareJid()
449 + ": automatically accept matching content-add offer");
450 acceptContentAdd(contentAddition.summary, modification);
451 }
452 return;
453 }
454
455 // once we can display multiple video tracks we can be more loose with this condition
456 // theoretically it should also be fine to automatically accept audio only contents
457 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
458 Log.d(
459 Config.LOGTAG,
460 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
461 this.incomingContentAdd = modification;
462 respondOk(jinglePacket);
463 updateEndUserState();
464 } else {
465 respondOk(jinglePacket);
466 // TODO do we want to add a reason?
467 rejectContentAdd(modification);
468 }
469 }
470
471 private void receiveContentAccept(final JinglePacket jinglePacket) {
472 final RtpContentMap receivedContentAccept;
473 try {
474 receivedContentAccept = RtpContentMap.of(jinglePacket);
475 receivedContentAccept.requireContentDescriptions();
476 } catch (final RuntimeException e) {
477 Log.d(
478 Config.LOGTAG,
479 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
480 Throwables.getRootCause(e));
481 respondOk(jinglePacket);
482 webRTCWrapper.close();
483 sendSessionTerminate(Reason.of(e), e.getMessage());
484 return;
485 }
486
487 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
488 if (outgoingContentAdd == null) {
489 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
490 terminateWithOutOfOrder(jinglePacket);
491 return;
492 }
493 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
494 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
495 this.outgoingContentAdd = null;
496 respondOk(jinglePacket);
497 receiveContentAccept(receivedContentAccept);
498 } else {
499 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
500 terminateWithOutOfOrder(jinglePacket);
501 }
502 }
503
504 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
505 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
506 final RtpContentMap modifiedContentMap =
507 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
508
509 setRemoteContentMap(modifiedContentMap);
510
511 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
512
513 final org.webrtc.SessionDescription sdp =
514 new org.webrtc.SessionDescription(
515 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
516
517 try {
518 this.webRTCWrapper.setRemoteDescription(sdp).get();
519 } catch (final Exception e) {
520 final Throwable cause = Throwables.getRootCause(e);
521 Log.d(
522 Config.LOGTAG,
523 id.getAccount().getJid().asBareJid()
524 + ": unable to set remote description after receiving content-accept",
525 cause);
526 webRTCWrapper.close();
527 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
528 return;
529 }
530 processCandidates(receivedContentAccept.contents.entrySet());
531 updateEndUserState();
532 Log.d(
533 Config.LOGTAG,
534 id.getAccount().getJid().asBareJid()
535 + ": remote has accepted content-add "
536 + ContentAddition.summary(receivedContentAccept));
537 }
538
539 private void receiveContentModify(final JinglePacket jinglePacket) {
540 // TODO check session accepted
541 final Map<String, Content.Senders> modification =
542 Maps.transformEntries(
543 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
544 final boolean isInitiator = isInitiator();
545 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
546 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
547 final Set<String> currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet();
548 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
549 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
550 respondOk(jinglePacket);
551 final RtpContentMap modifiedContentMap;
552 try {
553 modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification);
554 } catch (final IllegalArgumentException e) {
555 webRTCWrapper.close();
556 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
557 return;
558 }
559 this.outgoingContentAdd = modifiedContentMap;
560 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add");
561 } else if (remoteContentMap != null && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
562 respondOk(jinglePacket);
563 final RtpContentMap modifiedRemoteContentMap;
564 try {
565 modifiedRemoteContentMap = remoteContentMap.modifiedSendersChecked(isInitiator, modification);
566 } catch (final IllegalArgumentException e) {
567 webRTCWrapper.close();
568 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
569 return;
570 }
571 final SessionDescription offer;
572 try {
573 offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator());
574 } catch (final IllegalArgumentException | NullPointerException e) {
575 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-modify to SDP", e);
576 webRTCWrapper.close();
577 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
578 return;
579 }
580 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": auto accepting content-modification");
581 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
582 } else {
583 Log.d(Config.LOGTAG,"received unsupported content modification "+modification);
584 respondWithItemNotFound(jinglePacket);
585 }
586 }
587
588 private void autoAcceptContentModify(final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
589 this.setRemoteContentMap(modifiedRemoteContentMap);
590 final org.webrtc.SessionDescription sdp =
591 new org.webrtc.SessionDescription(
592 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
593 try {
594 this.webRTCWrapper.setRemoteDescription(sdp).get();
595 // auto accept is only done when we already have tracks
596 final SessionDescription answer = setLocalSessionDescription();
597 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
598 modifyLocalContentMap(rtpContentMap);
599 // we do not need to send an answer but do we have to resend the candidates currently in SDP?
600 //resendCandidatesFromSdp(answer);
601 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
602 } catch (final Exception e) {
603 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
604 webRTCWrapper.close();
605 sendSessionTerminate(Reason.FAILED_APPLICATION);
606 }
607 }
608
609 private void resendCandidatesFromSdp(final SessionDescription answer) {
610 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
611 for(final SessionDescription.Media media : answer.media) {
612 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
613 if (Strings.isNullOrEmpty(mid)) {
614 continue;
615 }
616 for(final String sdpCandidate : media.attributes.get("candidate")) {
617 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
618 if (candidate != null) {
619 candidateBuilder.put(mid,candidate);
620 }
621 }
622 }
623 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build();
624 sendTransportInfo(candidates);
625 }
626
627 private void receiveContentReject(final JinglePacket jinglePacket) {
628 final RtpContentMap receivedContentReject;
629 try {
630 receivedContentReject = RtpContentMap.of(jinglePacket);
631 } catch (final RuntimeException e) {
632 Log.d(
633 Config.LOGTAG,
634 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
635 Throwables.getRootCause(e));
636 respondOk(jinglePacket);
637 this.webRTCWrapper.close();
638 sendSessionTerminate(Reason.of(e), e.getMessage());
639 return;
640 }
641
642 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
643 if (outgoingContentAdd == null) {
644 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
645 terminateWithOutOfOrder(jinglePacket);
646 return;
647 }
648 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
649 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
650 this.outgoingContentAdd = null;
651 respondOk(jinglePacket);
652 Log.d(Config.LOGTAG,jinglePacket.toString());
653 receiveContentReject(ourSummary);
654 } else {
655 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
656 terminateWithOutOfOrder(jinglePacket);
657 }
658 }
659
660 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
661 try {
662 this.webRTCWrapper.removeTrack(Media.VIDEO);
663 final RtpContentMap localContentMap = customRollback();
664 modifyLocalContentMap(localContentMap);
665 } catch (final Exception e) {
666 final Throwable cause = Throwables.getRootCause(e);
667 Log.d(
668 Config.LOGTAG,
669 id.getAccount().getJid().asBareJid()
670 + ": unable to rollback local description after receiving content-reject",
671 cause);
672 webRTCWrapper.close();
673 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
674 return;
675 }
676 Log.d(
677 Config.LOGTAG,
678 id.getAccount().getJid().asBareJid()
679 + ": remote has rejected our content-add "
680 + summary);
681 }
682
683 private void receiveContentRemove(final JinglePacket jinglePacket) {
684 final RtpContentMap receivedContentRemove;
685 try {
686 receivedContentRemove = RtpContentMap.of(jinglePacket);
687 receivedContentRemove.requireContentDescriptions();
688 } catch (final RuntimeException e) {
689 Log.d(
690 Config.LOGTAG,
691 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
692 Throwables.getRootCause(e));
693 respondOk(jinglePacket);
694 this.webRTCWrapper.close();
695 sendSessionTerminate(Reason.of(e), e.getMessage());
696 return;
697 }
698 respondOk(jinglePacket);
699 receiveContentRemove(receivedContentRemove);
700 }
701
702 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
703 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
704 final Set<ContentAddition.Summary> contentAddSummary =
705 incomingContentAdd == null
706 ? Collections.emptySet()
707 : ContentAddition.summary(incomingContentAdd);
708 final Set<ContentAddition.Summary> removeSummary =
709 ContentAddition.summary(receivedContentRemove);
710 if (contentAddSummary.equals(removeSummary)) {
711 this.incomingContentAdd = null;
712 updateEndUserState();
713 } else {
714 webRTCWrapper.close();
715 sendSessionTerminate(
716 Reason.FAILED_APPLICATION,
717 String.format(
718 "%s only supports %s as a means to retract a not yet accepted %s",
719 BuildConfig.APP_NAME,
720 JinglePacket.Action.CONTENT_REMOVE,
721 JinglePacket.Action.CONTENT_ADD));
722 }
723 }
724
725 public synchronized void retractContentAdd() {
726 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
727 if (outgoingContentAdd == null) {
728 throw new IllegalStateException("Not outgoing content add");
729 }
730 try {
731 webRTCWrapper.removeTrack(Media.VIDEO);
732 final RtpContentMap localContentMap = customRollback();
733 modifyLocalContentMap(localContentMap);
734 } catch (final Exception e) {
735 final Throwable cause = Throwables.getRootCause(e);
736 Log.d(
737 Config.LOGTAG,
738 id.getAccount().getJid().asBareJid()
739 + ": unable to rollback local description after trying to retract content-add",
740 cause);
741 webRTCWrapper.close();
742 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
743 return;
744 }
745 this.outgoingContentAdd = null;
746 final JinglePacket retract =
747 outgoingContentAdd
748 .toStub()
749 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
750 this.send(retract);
751 Log.d(
752 Config.LOGTAG,
753 id.getAccount().getJid()
754 + ": retract content-add "
755 + ContentAddition.summary(outgoingContentAdd));
756 }
757
758 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
759 final SessionDescription sdp = setLocalSessionDescription();
760 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
761 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
762 this.webRTCWrapper
763 .setRemoteDescription(
764 new org.webrtc.SessionDescription(
765 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
766 .get();
767 return localRtpContentMap;
768 }
769
770 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
771 final RtpContentMap currentRemote = getRemoteContentMap();
772 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
773 if (diff.isEmpty()) {
774 throw new IllegalStateException(
775 "Unexpected rollback condition. No difference between local and remote");
776 }
777 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
778 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
779 final RtpContentMap nextRemote =
780 currentRemote.addContent(
781 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
782 return SessionDescription.of(nextRemote, !isInitiator());
783 }
784 throw new IllegalStateException(
785 "Unexpected rollback condition. Senders were not uniformly none");
786 }
787
788 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
789 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
790 if (incomingContentAdd == null) {
791 throw new IllegalStateException("No incoming content add");
792 }
793
794 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
795 this.incomingContentAdd = null;
796 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
797 Log.d(Config.LOGTAG,"senders of incoming content-add: "+senders);
798 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
799 Log.d(Config.LOGTAG,"content addition is receive only. we want to upgrade to 'both'");
800 final RtpContentMap modifiedSenders = incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
801 final JinglePacket proposedContentModification = modifiedSenders.toStub().toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
802 proposedContentModification.setTo(id.with);
803 xmppConnectionService.sendIqPacket(id.account, proposedContentModification, (account, response) -> {
804 if (response.getType() == IqPacket.TYPE.RESULT) {
805 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has accepted our upgrade to senders=both");
806 acceptContentAdd(ContentAddition.summary(modifiedSenders), modifiedSenders);
807 } else {
808 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has rejected our upgrade to senders=both");
809 acceptContentAdd(contentAddition, incomingContentAdd);
810 }
811 });
812 }
813 } else {
814 throw new IllegalStateException("Accepted content add does not match pending content-add");
815 }
816 }
817
818 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
819 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
820 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
821 this.setRemoteContentMap(modifiedContentMap);
822
823 final SessionDescription offer;
824 try {
825 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
826 } catch (final IllegalArgumentException | NullPointerException e) {
827 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
828 webRTCWrapper.close();
829 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
830 return;
831 }
832 this.incomingContentAdd = null;
833 acceptContentAdd(contentAddition, offer);
834 }
835
836 private void acceptContentAdd(
837 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
838 final org.webrtc.SessionDescription sdp =
839 new org.webrtc.SessionDescription(
840 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
841 try {
842 this.webRTCWrapper.setRemoteDescription(sdp).get();
843
844 // TODO add tracks for 'media' where contentAddition.senders matches
845
846 // TODO if senders.sending(isInitiator())
847
848 this.webRTCWrapper.addTrack(Media.VIDEO);
849
850 // TODO add additional transceivers for recv only cases
851
852 final SessionDescription answer = setLocalSessionDescription();
853 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
854
855 final RtpContentMap contentAcceptMap =
856 rtpContentMap.toContentModification(
857 Collections2.transform(contentAddition, ca -> ca.name));
858 Log.d(
859 Config.LOGTAG,
860 id.getAccount().getJid().asBareJid()
861 + ": sending content-accept "
862 + ContentAddition.summary(contentAcceptMap));
863
864 addIceCandidatesFromBlackLog();
865
866 modifyLocalContentMap(rtpContentMap);
867 sendContentAccept(contentAcceptMap);
868 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
869 } catch (final Exception e) {
870 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
871 webRTCWrapper.close();
872 sendSessionTerminate(Reason.FAILED_APPLICATION);
873 }
874 }
875
876 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
877 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
878 send(jinglePacket);
879 }
880
881 public synchronized void rejectContentAdd() {
882 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
883 if (incomingContentAdd == null) {
884 throw new IllegalStateException("No incoming content add");
885 }
886 this.incomingContentAdd = null;
887 updateEndUserState();
888 rejectContentAdd(incomingContentAdd);
889 }
890
891 private void rejectContentAdd(final RtpContentMap contentMap) {
892 final JinglePacket jinglePacket =
893 contentMap
894 .toStub()
895 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
896 Log.d(
897 Config.LOGTAG,
898 id.getAccount().getJid().asBareJid()
899 + ": rejecting content "
900 + ContentAddition.summary(contentMap));
901 send(jinglePacket);
902 }
903
904 private boolean checkForIceRestart(
905 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
906 final RtpContentMap existing = getRemoteContentMap();
907 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
908 final IceUdpTransportInfo.Credentials newCredentials;
909 try {
910 existingCredentials = existing.getCredentials();
911 newCredentials = rtpContentMap.getDistinctCredentials();
912 } catch (final IllegalStateException e) {
913 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
914 return false;
915 }
916 if (existingCredentials.contains(newCredentials)) {
917 return false;
918 }
919 // TODO an alternative approach is to check if we already got an iq result to our
920 // ICE-restart
921 // and if that's the case we are seeing an answer.
922 // This might be more spec compliant but also more error prone potentially
923 final boolean isSignalStateStable = this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
924 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
925 final boolean isOffer = rtpContentMap.emptyCandidates();
926 final RtpContentMap restartContentMap;
927 try {
928 if (isOffer) {
929 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
930 restartContentMap =
931 existing.modifiedCredentials(
932 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
933 } else {
934 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
935 Log.d(
936 Config.LOGTAG,
937 "received confirmation of ICE restart"
938 + newCredentials
939 + " peer_setup="
940 + setup);
941 // DTLS setup attribute needs to be rewritten to reflect current peer state
942 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
943 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
944 }
945 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
946 return isOffer;
947 } else {
948 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
949 respondWithTieBreak(jinglePacket);
950 return true;
951 }
952 } catch (final Exception exception) {
953 respondOk(jinglePacket);
954 final Throwable rootCause = Throwables.getRootCause(exception);
955 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
956 // If this happens a termination is already in progress
957 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
958 return true;
959 }
960 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
961 webRTCWrapper.close();
962 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
963 return true;
964 }
965 }
966
967 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
968 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
969 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
970 throw new IllegalStateException("Invalid peer setup");
971 }
972 return peerSetup;
973 }
974
975 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
976 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
977 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
978 }
979 this.peerDtlsSetup = setup;
980 }
981
982 private boolean applyIceRestart(
983 final JinglePacket jinglePacket,
984 final RtpContentMap restartContentMap,
985 final boolean isOffer)
986 throws ExecutionException, InterruptedException {
987 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
988 final org.webrtc.SessionDescription.Type type =
989 isOffer
990 ? org.webrtc.SessionDescription.Type.OFFER
991 : org.webrtc.SessionDescription.Type.ANSWER;
992 org.webrtc.SessionDescription sdp =
993 new org.webrtc.SessionDescription(type, sessionDescription.toString());
994 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
995 if (isInitiator()) {
996 // We ignore the offer and respond with tie-break. This will clause the responder
997 // not to apply the content map
998 return false;
999 }
1000 }
1001 webRTCWrapper.setRemoteDescription(sdp).get();
1002 setRemoteContentMap(restartContentMap);
1003 if (isOffer) {
1004 final SessionDescription localSessionDescription = setLocalSessionDescription();
1005 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1006 // We need to respond OK before sending any candidates
1007 respondOk(jinglePacket);
1008 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1009 } else {
1010 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1011 }
1012 return true;
1013 }
1014
1015 private void processCandidates(
1016 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
1017 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
1018 processCandidate(content);
1019 }
1020 }
1021
1022 private void processCandidate(
1023 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
1024 final RtpContentMap rtpContentMap = getRemoteContentMap();
1025 final List<String> indices = toIdentificationTags(rtpContentMap);
1026 final String sdpMid = content.getKey(); // aka content name
1027 final IceUdpTransportInfo transport = content.getValue().transport;
1028 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1029
1030 // TODO check that credentials remained the same
1031
1032 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1033 final String sdp;
1034 try {
1035 sdp = candidate.toSdpAttribute(credentials.ufrag);
1036 } catch (final IllegalArgumentException e) {
1037 Log.d(
1038 Config.LOGTAG,
1039 id.account.getJid().asBareJid()
1040 + ": ignoring invalid ICE candidate "
1041 + e.getMessage());
1042 continue;
1043 }
1044 final int mLineIndex = indices.indexOf(sdpMid);
1045 if (mLineIndex < 0) {
1046 Log.w(
1047 Config.LOGTAG,
1048 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1049 }
1050 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1051 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1052 this.webRTCWrapper.addIceCandidate(iceCandidate);
1053 }
1054 }
1055
1056 private RtpContentMap getRemoteContentMap() {
1057 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1058 }
1059
1060 private RtpContentMap getLocalContentMap() {
1061 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1062 }
1063
1064 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1065 final Group originalGroup = rtpContentMap.group;
1066 final List<String> identificationTags =
1067 originalGroup == null
1068 ? rtpContentMap.getNames()
1069 : originalGroup.getIdentificationTags();
1070 if (identificationTags.size() == 0) {
1071 Log.w(
1072 Config.LOGTAG,
1073 id.account.getJid().asBareJid()
1074 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1075 }
1076 return identificationTags;
1077 }
1078
1079 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1080 final JinglePacket jinglePacket, final boolean expectVerification) {
1081 final RtpContentMap receivedContentMap;
1082 try {
1083 receivedContentMap = RtpContentMap.of(jinglePacket);
1084 } catch (final Exception e) {
1085 return Futures.immediateFailedFuture(e);
1086 }
1087 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1088 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1089 id.account
1090 .getAxolotlService()
1091 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1092 return Futures.transform(
1093 future,
1094 omemoVerifiedPayload -> {
1095 // TODO test if an exception here triggers a correct abort
1096 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1097 Log.d(
1098 Config.LOGTAG,
1099 id.account.getJid().asBareJid()
1100 + ": received verifiable DTLS fingerprint via "
1101 + omemoVerification);
1102 return omemoVerifiedPayload.getPayload();
1103 },
1104 MoreExecutors.directExecutor());
1105 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1106 return Futures.immediateFailedFuture(
1107 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1108 } else {
1109 return Futures.immediateFuture(receivedContentMap);
1110 }
1111 }
1112
1113 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1114 if (isInitiator()) {
1115 Log.d(
1116 Config.LOGTAG,
1117 String.format(
1118 "%s: received session-initiate even though we were initiating",
1119 id.account.getJid().asBareJid()));
1120 if (isTerminated()) {
1121 Log.d(
1122 Config.LOGTAG,
1123 String.format(
1124 "%s: got a reason to terminate with out-of-order. but already in state %s",
1125 id.account.getJid().asBareJid(), getState()));
1126 respondWithOutOfOrder(jinglePacket);
1127 } else {
1128 terminateWithOutOfOrder(jinglePacket);
1129 }
1130 return;
1131 }
1132 final Presence presence = id.getContact().getPresences().get(id.getWith().getResource());
1133 if (presence != null && presence.getServiceDiscoveryResult().getFeatures().contains("urn:ietf:rfc:3264")) webRTCWrapper.setRFC3264(true);
1134 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1135 Futures.addCallback(
1136 future,
1137 new FutureCallback<RtpContentMap>() {
1138 @Override
1139 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1140 receiveSessionInitiate(jinglePacket, rtpContentMap);
1141 }
1142
1143 @Override
1144 public void onFailure(@NonNull final Throwable throwable) {
1145 respondOk(jinglePacket);
1146 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1147 }
1148 },
1149 MoreExecutors.directExecutor());
1150 }
1151
1152 private void receiveSessionInitiate(
1153 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1154 try {
1155 contentMap.requireContentDescriptions();
1156 contentMap.requireDTLSFingerprint(true);
1157 } catch (final RuntimeException e) {
1158 Log.d(
1159 Config.LOGTAG,
1160 id.account.getJid().asBareJid() + ": improperly formatted contents",
1161 Throwables.getRootCause(e));
1162 respondOk(jinglePacket);
1163 sendSessionTerminate(Reason.of(e), e.getMessage());
1164 return;
1165 }
1166 Log.d(
1167 Config.LOGTAG,
1168 "processing session-init with " + contentMap.contents.size() + " contents");
1169 final State target;
1170 if (this.state == State.PROCEED) {
1171 Preconditions.checkState(
1172 proposedMedia != null && proposedMedia.size() > 0,
1173 "proposed media must be set when processing pre-approved session-initiate");
1174 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1175 sendSessionTerminate(
1176 Reason.SECURITY_ERROR,
1177 String.format(
1178 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1179 this.proposedMedia, contentMap.getMedia()));
1180 return;
1181 }
1182 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1183 } else {
1184 target = State.SESSION_INITIALIZED;
1185 }
1186 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1187 respondOk(jinglePacket);
1188 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1189 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1190 Log.d(
1191 Config.LOGTAG,
1192 id.account.getJid().asBareJid()
1193 + ": automatically accepting session-initiate");
1194 sendSessionAccept();
1195 } else {
1196 Log.d(
1197 Config.LOGTAG,
1198 id.account.getJid().asBareJid()
1199 + ": received not pre-approved session-initiate. start ringing");
1200 startRinging();
1201 }
1202 } else {
1203 Log.d(
1204 Config.LOGTAG,
1205 String.format(
1206 "%s: received session-initiate while in state %s",
1207 id.account.getJid().asBareJid(), state));
1208 terminateWithOutOfOrder(jinglePacket);
1209 }
1210 }
1211
1212 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1213 if (!isInitiator()) {
1214 Log.d(
1215 Config.LOGTAG,
1216 String.format(
1217 "%s: received session-accept even though we were responding",
1218 id.account.getJid().asBareJid()));
1219 terminateWithOutOfOrder(jinglePacket);
1220 return;
1221 }
1222 final ListenableFuture<RtpContentMap> future =
1223 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1224 Futures.addCallback(
1225 future,
1226 new FutureCallback<RtpContentMap>() {
1227 @Override
1228 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1229 receiveSessionAccept(jinglePacket, rtpContentMap);
1230 }
1231
1232 @Override
1233 public void onFailure(@NonNull final Throwable throwable) {
1234 respondOk(jinglePacket);
1235 Log.d(
1236 Config.LOGTAG,
1237 id.account.getJid().asBareJid()
1238 + ": improperly formatted contents in session-accept",
1239 throwable);
1240 webRTCWrapper.close();
1241 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1242 }
1243 },
1244 MoreExecutors.directExecutor());
1245 }
1246
1247 private void receiveSessionAccept(
1248 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1249 try {
1250 contentMap.requireContentDescriptions();
1251 contentMap.requireDTLSFingerprint();
1252 } catch (final RuntimeException e) {
1253 respondOk(jinglePacket);
1254 Log.d(
1255 Config.LOGTAG,
1256 id.account.getJid().asBareJid()
1257 + ": improperly formatted contents in session-accept",
1258 e);
1259 webRTCWrapper.close();
1260 sendSessionTerminate(Reason.of(e), e.getMessage());
1261 return;
1262 }
1263 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1264 if (!initiatorMedia.equals(contentMap.getMedia())) {
1265 sendSessionTerminate(
1266 Reason.SECURITY_ERROR,
1267 String.format(
1268 "Your session-included included media %s but our session-initiate was %s",
1269 this.proposedMedia, contentMap.getMedia()));
1270 return;
1271 }
1272 Log.d(
1273 Config.LOGTAG,
1274 "processing session-accept with " + contentMap.contents.size() + " contents");
1275 if (transition(State.SESSION_ACCEPTED)) {
1276 respondOk(jinglePacket);
1277 receiveSessionAccept(contentMap);
1278 } else {
1279 Log.d(
1280 Config.LOGTAG,
1281 String.format(
1282 "%s: received session-accept while in state %s",
1283 id.account.getJid().asBareJid(), state));
1284 respondOk(jinglePacket);
1285 }
1286 }
1287
1288 private void receiveSessionAccept(final RtpContentMap contentMap) {
1289 this.responderRtpContentMap = contentMap;
1290 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1291 final SessionDescription sessionDescription;
1292 try {
1293 sessionDescription = SessionDescription.of(contentMap, false);
1294 } catch (final IllegalArgumentException | NullPointerException e) {
1295 Log.d(
1296 Config.LOGTAG,
1297 id.account.getJid().asBareJid()
1298 + ": unable convert offer from session-accept to SDP",
1299 e);
1300 webRTCWrapper.close();
1301 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1302 return;
1303 }
1304 final org.webrtc.SessionDescription answer =
1305 new org.webrtc.SessionDescription(
1306 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1307 try {
1308 this.webRTCWrapper.setRemoteDescription(answer).get();
1309 } catch (final Exception e) {
1310 Log.d(
1311 Config.LOGTAG,
1312 id.account.getJid().asBareJid()
1313 + ": unable to set remote description after receiving session-accept",
1314 Throwables.getRootCause(e));
1315 webRTCWrapper.close();
1316 sendSessionTerminate(
1317 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1318 return;
1319 }
1320 processCandidates(contentMap.contents.entrySet());
1321 }
1322
1323 private void sendSessionAccept() {
1324 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1325 if (rtpContentMap == null) {
1326 throw new IllegalStateException("initiator RTP Content Map has not been set");
1327 }
1328 final SessionDescription offer;
1329 try {
1330 offer = SessionDescription.of(rtpContentMap, true);
1331 } catch (final IllegalArgumentException | NullPointerException e) {
1332 Log.d(
1333 Config.LOGTAG,
1334 id.account.getJid().asBareJid()
1335 + ": unable convert offer from session-initiate to SDP",
1336 e);
1337 webRTCWrapper.close();
1338 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1339 return;
1340 }
1341 sendSessionAccept(rtpContentMap.getMedia(), offer);
1342 }
1343
1344 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1345 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1346 }
1347
1348 private synchronized void sendSessionAccept(
1349 final Set<Media> media,
1350 final SessionDescription offer,
1351 final List<PeerConnection.IceServer> iceServers) {
1352 if (isTerminated()) {
1353 Log.w(
1354 Config.LOGTAG,
1355 id.account.getJid().asBareJid()
1356 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1357 return;
1358 }
1359 try {
1360 setupWebRTC(media, iceServers);
1361 } catch (final WebRTCWrapper.InitializationException e) {
1362 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1363 webRTCWrapper.close();
1364 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1365 return;
1366 }
1367 final org.webrtc.SessionDescription sdp =
1368 new org.webrtc.SessionDescription(
1369 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1370 try {
1371 this.webRTCWrapper.setRemoteDescription(sdp).get();
1372 addIceCandidatesFromBlackLog();
1373 org.webrtc.SessionDescription webRTCSessionDescription =
1374 this.webRTCWrapper.setLocalDescription().get();
1375 prepareSessionAccept(webRTCSessionDescription);
1376 } catch (final Exception e) {
1377 failureToAcceptSession(e);
1378 }
1379 }
1380
1381 private void failureToAcceptSession(final Throwable throwable) {
1382 if (isTerminated()) {
1383 return;
1384 }
1385 final Throwable rootCause = Throwables.getRootCause(throwable);
1386 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1387 webRTCWrapper.close();
1388 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1389 }
1390
1391 private void addIceCandidatesFromBlackLog() {
1392 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1393 while ((foo = this.pendingIceCandidates.poll()) != null) {
1394 processCandidate(foo);
1395 Log.d(
1396 Config.LOGTAG,
1397 id.account.getJid().asBareJid() + ": added candidate from back log");
1398 }
1399 }
1400
1401 private void prepareSessionAccept(
1402 final org.webrtc.SessionDescription initialWebRTCSessionDescription) {
1403 Futures.addCallback(
1404 webRTCWrapper.getRFC3264() ? Futures.withTimeout(iceGatheringComplete, 2, TimeUnit.SECONDS, JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE) : Futures.immediateFuture(null),
1405 new FutureCallback<Collection<IceCandidate>>() {
1406 @Override
1407 public void onSuccess(final Collection<IceCandidate> iceCandidates) {
1408 org.webrtc.SessionDescription webRTCSessionDescription =
1409 JingleRtpConnection.this.webRTCWrapper.getLocalDescription();
1410 final SessionDescription sessionDescription =
1411 SessionDescription.parse(webRTCSessionDescription.description);
1412 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1413 JingleRtpConnection.this.responderRtpContentMap = respondingRtpContentMap;
1414 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1415 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1416 prepareOutgoingContentMap(respondingRtpContentMap);
1417 Futures.addCallback(
1418 outgoingContentMapFuture,
1419 new FutureCallback<RtpContentMap>() {
1420 @Override
1421 public void onSuccess(final RtpContentMap outgoingContentMap) {
1422 sendSessionAccept(outgoingContentMap);
1423 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1424 }
1425
1426 @Override
1427 public void onFailure(@NonNull Throwable throwable) {
1428 failureToAcceptSession(throwable);
1429 }
1430 },
1431 MoreExecutors.directExecutor());
1432 }
1433
1434 @Override
1435 public void onFailure(@NonNull final Throwable throwable) {
1436 Log.e(Config.LOGTAG, "ICE gathering didn't finish clean: " + throwable);
1437 onSuccess(null);
1438 }
1439 },
1440 MoreExecutors.directExecutor()
1441 );
1442 }
1443
1444 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1445 if (isTerminated()) {
1446 Log.w(
1447 Config.LOGTAG,
1448 id.account.getJid().asBareJid()
1449 + ": preparing session accept was too slow. already terminated. nothing to do.");
1450 return;
1451 }
1452 transitionOrThrow(State.SESSION_ACCEPTED);
1453 final JinglePacket sessionAccept =
1454 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1455 send(sessionAccept);
1456 }
1457
1458 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1459 final RtpContentMap rtpContentMap) {
1460 if (this.omemoVerification.hasDeviceId()) {
1461 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1462 verifiedPayloadFuture =
1463 id.account
1464 .getAxolotlService()
1465 .encrypt(
1466 rtpContentMap,
1467 id.with,
1468 omemoVerification.getDeviceId());
1469 return Futures.transform(
1470 verifiedPayloadFuture,
1471 verifiedPayload -> {
1472 omemoVerification.setOrEnsureEqual(verifiedPayload);
1473 return verifiedPayload.getPayload();
1474 },
1475 MoreExecutors.directExecutor());
1476 } else {
1477 return Futures.immediateFuture(rtpContentMap);
1478 }
1479 }
1480
1481 synchronized void deliveryMessage(
1482 final Jid from,
1483 final Element message,
1484 final String serverMessageId,
1485 final long timestamp) {
1486 Log.d(
1487 Config.LOGTAG,
1488 id.account.getJid().asBareJid()
1489 + ": delivered message to JingleRtpConnection "
1490 + message);
1491 switch (message.getName()) {
1492 case "propose":
1493 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1494 break;
1495 case "proceed":
1496 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1497 break;
1498 case "retract":
1499 receiveRetract(from, serverMessageId, timestamp);
1500 break;
1501 case "reject":
1502 receiveReject(from, serverMessageId, timestamp);
1503 break;
1504 case "accept":
1505 receiveAccept(from, serverMessageId, timestamp);
1506 break;
1507 default:
1508 break;
1509 }
1510 }
1511
1512 void deliverFailedProceed(final String message) {
1513 Log.d(
1514 Config.LOGTAG,
1515 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1516 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1517 webRTCWrapper.close();
1518 Log.d(
1519 Config.LOGTAG,
1520 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1521 this.finish();
1522 }
1523 }
1524
1525 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1526 final boolean originatedFromMyself =
1527 from.asBareJid().equals(id.account.getJid().asBareJid());
1528 if (originatedFromMyself) {
1529 if (transition(State.ACCEPTED)) {
1530 acceptedOnOtherDevice(serverMsgId, timestamp);
1531 } else {
1532 Log.d(
1533 Config.LOGTAG,
1534 id.account.getJid().asBareJid()
1535 + ": unable to transition to accept because already in state="
1536 + this.state);
1537 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1538 }
1539 } else {
1540 Log.d(
1541 Config.LOGTAG,
1542 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1543 }
1544 }
1545
1546 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1547 if (serverMsgId != null) {
1548 this.message.setServerMsgId(serverMsgId);
1549 }
1550 this.message.setTime(timestamp);
1551 this.message.setCarbon(true); // indicate that call was accepted on other device
1552 this.writeLogMessageSuccess(0);
1553 this.xmppConnectionService
1554 .getNotificationService()
1555 .cancelIncomingCallNotification();
1556 this.finish();
1557 }
1558
1559 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1560 final boolean originatedFromMyself =
1561 from.asBareJid().equals(id.account.getJid().asBareJid());
1562 // reject from another one of my clients
1563 if (originatedFromMyself) {
1564 receiveRejectFromMyself(serverMsgId, timestamp);
1565 } else if (isInitiator()) {
1566 if (from.equals(id.with)) {
1567 receiveRejectFromResponder();
1568 } else {
1569 Log.d(
1570 Config.LOGTAG,
1571 id.account.getJid()
1572 + ": ignoring reject from "
1573 + from
1574 + " for session with "
1575 + id.with);
1576 }
1577 } else {
1578 Log.d(
1579 Config.LOGTAG,
1580 id.account.getJid()
1581 + ": ignoring reject from "
1582 + from
1583 + " for session with "
1584 + id.with);
1585 }
1586 }
1587
1588 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1589 if (transition(State.REJECTED)) {
1590 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1591 this.finish();
1592 if (serverMsgId != null) {
1593 this.message.setServerMsgId(serverMsgId);
1594 }
1595 this.message.setTime(timestamp);
1596 this.message.setCarbon(true); // indicate that call was rejected on other device
1597 writeLogMessageMissed();
1598 } else {
1599 Log.d(
1600 Config.LOGTAG,
1601 "not able to transition into REJECTED because already in " + this.state);
1602 }
1603 }
1604
1605 private void receiveRejectFromResponder() {
1606 if (isInState(State.PROCEED)) {
1607 Log.d(
1608 Config.LOGTAG,
1609 id.account.getJid()
1610 + ": received reject while still in proceed. callee reconsidered");
1611 closeTransitionLogFinish(State.REJECTED_RACED);
1612 return;
1613 }
1614 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1615 Log.d(
1616 Config.LOGTAG,
1617 id.account.getJid()
1618 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1619 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1620 return;
1621 }
1622 Log.d(
1623 Config.LOGTAG,
1624 id.account.getJid()
1625 + ": ignoring reject from responder because already in state "
1626 + this.state);
1627 }
1628
1629 private void receivePropose(
1630 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1631 final boolean originatedFromMyself =
1632 from.asBareJid().equals(id.account.getJid().asBareJid());
1633 if (originatedFromMyself) {
1634 Log.d(
1635 Config.LOGTAG,
1636 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1637 } else if (transition(
1638 State.PROPOSED,
1639 () -> {
1640 final Collection<RtpDescription> descriptions =
1641 Collections2.transform(
1642 Collections2.filter(
1643 propose.getDescriptions(),
1644 d -> d instanceof RtpDescription),
1645 input -> (RtpDescription) input);
1646 final Collection<Media> media =
1647 Collections2.transform(descriptions, RtpDescription::getMedia);
1648 Preconditions.checkState(
1649 !media.contains(Media.UNKNOWN),
1650 "RTP descriptions contain unknown media");
1651 Log.d(
1652 Config.LOGTAG,
1653 id.account.getJid().asBareJid()
1654 + ": received session proposal from "
1655 + from
1656 + " for "
1657 + media);
1658 this.proposedMedia = Sets.newHashSet(media);
1659 })) {
1660 if (serverMsgId != null) {
1661 this.message.setServerMsgId(serverMsgId);
1662 }
1663 this.message.setTime(timestamp);
1664 startRinging();
1665 } else {
1666 Log.d(
1667 Config.LOGTAG,
1668 id.account.getJid()
1669 + ": ignoring session proposal because already in "
1670 + state);
1671 }
1672 }
1673
1674 private void startRinging() {
1675 Log.d(
1676 Config.LOGTAG,
1677 id.account.getJid().asBareJid()
1678 + ": received call from "
1679 + id.with
1680 + ". start ringing");
1681 ringingTimeoutFuture =
1682 jingleConnectionManager.schedule(
1683 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1684 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1685 }
1686
1687 private synchronized void ringingTimeout() {
1688 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1689 switch (this.state) {
1690 case PROPOSED:
1691 message.markUnread();
1692 rejectCallFromProposed();
1693 break;
1694 case SESSION_INITIALIZED:
1695 message.markUnread();
1696 rejectCallFromSessionInitiate();
1697 break;
1698 }
1699 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1700 }
1701
1702 private void cancelRingingTimeout() {
1703 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1704 if (future != null && !future.isCancelled()) {
1705 future.cancel(false);
1706 }
1707 }
1708
1709 private void receiveProceed(
1710 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1711 final Set<Media> media =
1712 Preconditions.checkNotNull(
1713 this.proposedMedia, "Proposed media has to be set before handling proceed");
1714 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1715 if (from.equals(id.with)) {
1716 if (isInitiator()) {
1717 if (transition(State.PROCEED)) {
1718 if (serverMsgId != null) {
1719 this.message.setServerMsgId(serverMsgId);
1720 }
1721 this.message.setTime(timestamp);
1722 final Integer remoteDeviceId = proceed.getDeviceId();
1723 if (isOmemoEnabled()) {
1724 this.omemoVerification.setDeviceId(remoteDeviceId);
1725 } else {
1726 if (remoteDeviceId != null) {
1727 Log.d(
1728 Config.LOGTAG,
1729 id.account.getJid().asBareJid()
1730 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1731 }
1732 this.omemoVerification.setDeviceId(null);
1733 }
1734 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1735 } else {
1736 Log.d(
1737 Config.LOGTAG,
1738 String.format(
1739 "%s: ignoring proceed because already in %s",
1740 id.account.getJid().asBareJid(), this.state));
1741 }
1742 } else {
1743 Log.d(
1744 Config.LOGTAG,
1745 String.format(
1746 "%s: ignoring proceed because we were not initializing",
1747 id.account.getJid().asBareJid()));
1748 }
1749 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1750 if (transition(State.ACCEPTED)) {
1751 Log.d(
1752 Config.LOGTAG,
1753 id.account.getJid().asBareJid()
1754 + ": moved session with "
1755 + id.with
1756 + " into state accepted after received carbon copied proceed");
1757 acceptedOnOtherDevice(serverMsgId, timestamp);
1758 }
1759 } else {
1760 Log.d(
1761 Config.LOGTAG,
1762 String.format(
1763 "%s: ignoring proceed from %s. was expected from %s",
1764 id.account.getJid().asBareJid(), from, id.with));
1765 }
1766 }
1767
1768 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1769 if (from.equals(id.with)) {
1770 final State target =
1771 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1772 if (transition(target)) {
1773 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1774 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1775 Log.d(
1776 Config.LOGTAG,
1777 id.account.getJid().asBareJid()
1778 + ": session with "
1779 + id.with
1780 + " has been retracted (serverMsgId="
1781 + serverMsgId
1782 + ")");
1783 if (serverMsgId != null) {
1784 this.message.setServerMsgId(serverMsgId);
1785 }
1786 this.message.setTime(timestamp);
1787 if (target == State.RETRACTED) {
1788 this.message.markUnread();
1789 }
1790 writeLogMessageMissed();
1791 finish();
1792 } else {
1793 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1794 }
1795 } else {
1796 // TODO parse retract from self
1797 Log.d(
1798 Config.LOGTAG,
1799 id.account.getJid().asBareJid()
1800 + ": received retract from "
1801 + from
1802 + ". expected retract from"
1803 + id.with
1804 + ". ignoring");
1805 }
1806 }
1807
1808 public void sendSessionInitiate() {
1809 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1810 }
1811
1812 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1813 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1814 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1815 }
1816
1817 private synchronized void sendSessionInitiate(
1818 final Set<Media> media,
1819 final State targetState,
1820 final List<PeerConnection.IceServer> iceServers) {
1821 if (isTerminated()) {
1822 Log.w(
1823 Config.LOGTAG,
1824 id.account.getJid().asBareJid()
1825 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1826 return;
1827 }
1828 try {
1829 setupWebRTC(media, iceServers);
1830 } catch (final WebRTCWrapper.InitializationException e) {
1831 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1832 webRTCWrapper.close();
1833 sendRetract(Reason.ofThrowable(e));
1834 return;
1835 }
1836 try {
1837 org.webrtc.SessionDescription webRTCSessionDescription =
1838 this.webRTCWrapper.setLocalDescription().get();
1839 prepareSessionInitiate(webRTCSessionDescription, targetState);
1840 } catch (final Exception e) {
1841 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1842 // exceptions
1843 failureToInitiateSession(e, targetState);
1844 }
1845 }
1846
1847 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1848 if (isTerminated()) {
1849 return;
1850 }
1851 Log.d(
1852 Config.LOGTAG,
1853 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1854 Throwables.getRootCause(throwable));
1855 webRTCWrapper.close();
1856 final Reason reason = Reason.ofThrowable(throwable);
1857 if (isInState(targetState)) {
1858 sendSessionTerminate(reason, throwable.getMessage());
1859 } else {
1860 sendRetract(reason);
1861 }
1862 }
1863
1864 private void sendRetract(final Reason reason) {
1865 // TODO embed reason into retract
1866 sendJingleMessage("retract", id.with.asBareJid());
1867 transitionOrThrow(reasonToState(reason));
1868 this.finish();
1869 }
1870
1871 private void prepareSessionInitiate(
1872 final org.webrtc.SessionDescription initialWebRTCSessionDescription, final State targetState) {
1873 Futures.addCallback(
1874 webRTCWrapper.getRFC3264() ? Futures.withTimeout(iceGatheringComplete, 2, TimeUnit.SECONDS, JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE) : Futures.immediateFuture(null),
1875 new FutureCallback<Collection<IceCandidate>>() {
1876 @Override
1877 public void onSuccess(final Collection<IceCandidate> iceCandidates) {
1878 org.webrtc.SessionDescription webRTCSessionDescription =
1879 JingleRtpConnection.this.webRTCWrapper.getLocalDescription();
1880 final SessionDescription sessionDescription =
1881 SessionDescription.parse(webRTCSessionDescription.description);
1882 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1883 JingleRtpConnection.this.initiatorRtpContentMap = rtpContentMap;
1884 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1885 encryptSessionInitiate(rtpContentMap);
1886 Futures.addCallback(
1887 outgoingContentMapFuture,
1888 new FutureCallback<RtpContentMap>() {
1889 @Override
1890 public void onSuccess(final RtpContentMap outgoingContentMap) {
1891 sendSessionInitiate(outgoingContentMap, targetState);
1892 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1893 }
1894
1895 @Override
1896 public void onFailure(@NonNull final Throwable throwable) {
1897 failureToInitiateSession(throwable, targetState);
1898 }
1899 },
1900 MoreExecutors.directExecutor()
1901 );
1902 }
1903
1904 @Override
1905 public void onFailure(@NonNull final Throwable throwable) {
1906 Log.e(Config.LOGTAG, "ICE gathering didn't finish clean: " + throwable);
1907 onSuccess(null);
1908 }
1909 },
1910 MoreExecutors.directExecutor()
1911 );
1912 }
1913
1914 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1915 if (isTerminated()) {
1916 Log.w(
1917 Config.LOGTAG,
1918 id.account.getJid().asBareJid()
1919 + ": preparing session was too slow. already terminated. nothing to do.");
1920 return;
1921 }
1922 this.transitionOrThrow(targetState);
1923 final JinglePacket sessionInitiate =
1924 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1925 send(sessionInitiate);
1926 }
1927
1928 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1929 final RtpContentMap rtpContentMap) {
1930 if (this.omemoVerification.hasDeviceId()) {
1931 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1932 verifiedPayloadFuture =
1933 id.account
1934 .getAxolotlService()
1935 .encrypt(
1936 rtpContentMap,
1937 id.with,
1938 omemoVerification.getDeviceId());
1939 final ListenableFuture<RtpContentMap> future =
1940 Futures.transform(
1941 verifiedPayloadFuture,
1942 verifiedPayload -> {
1943 omemoVerification.setSessionFingerprint(
1944 verifiedPayload.getFingerprint());
1945 return verifiedPayload.getPayload();
1946 },
1947 MoreExecutors.directExecutor());
1948 if (Config.REQUIRE_RTP_VERIFICATION) {
1949 return future;
1950 }
1951 return Futures.catching(
1952 future,
1953 CryptoFailedException.class,
1954 e -> {
1955 Log.w(
1956 Config.LOGTAG,
1957 id.account.getJid().asBareJid()
1958 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1959 e);
1960 return rtpContentMap;
1961 },
1962 MoreExecutors.directExecutor());
1963 } else {
1964 return Futures.immediateFuture(rtpContentMap);
1965 }
1966 }
1967
1968 private void sendSessionTerminate(final Reason reason) {
1969 sendSessionTerminate(reason, null);
1970 }
1971
1972 private void sendSessionTerminate(final Reason reason, final String text) {
1973 final State previous = this.state;
1974 final State target = reasonToState(reason);
1975 transitionOrThrow(target);
1976 if (previous != State.NULL) {
1977 writeLogMessage(target);
1978 }
1979 final JinglePacket jinglePacket =
1980 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1981 jinglePacket.setReason(reason, text);
1982 Log.d(Config.LOGTAG, jinglePacket.toString());
1983 send(jinglePacket);
1984 finish();
1985 }
1986
1987 private void sendTransportInfo(
1988 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1989 final RtpContentMap transportInfo;
1990 try {
1991 final RtpContentMap rtpContentMap =
1992 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1993 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1994 } catch (final Exception e) {
1995 Log.d(
1996 Config.LOGTAG,
1997 id.account.getJid().asBareJid()
1998 + ": unable to prepare transport-info from candidate for content="
1999 + contentName);
2000 return;
2001 }
2002 final JinglePacket jinglePacket =
2003 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2004 send(jinglePacket);
2005 }
2006
2007 private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
2008 // TODO send all candidates in one transport-info
2009 }
2010
2011
2012 private void send(final JinglePacket jinglePacket) {
2013 jinglePacket.setTo(id.with);
2014 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
2015 }
2016
2017 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
2018 if (response.getType() == IqPacket.TYPE.ERROR) {
2019 handleIqErrorResponse(response);
2020 return;
2021 }
2022 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2023 handleIqTimeoutResponse(response);
2024 }
2025 }
2026
2027 private void handleIqErrorResponse(final IqPacket response) {
2028 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
2029 final String errorCondition = response.getErrorCondition();
2030 Log.d(
2031 Config.LOGTAG,
2032 id.account.getJid().asBareJid()
2033 + ": received IQ-error from "
2034 + response.getFrom()
2035 + " in RTP session. "
2036 + errorCondition);
2037 if (isTerminated()) {
2038 Log.i(
2039 Config.LOGTAG,
2040 id.account.getJid().asBareJid()
2041 + ": ignoring error because session was already terminated");
2042 return;
2043 }
2044 this.webRTCWrapper.close();
2045 final State target;
2046 if (Arrays.asList(
2047 "service-unavailable",
2048 "recipient-unavailable",
2049 "remote-server-not-found",
2050 "remote-server-timeout")
2051 .contains(errorCondition)) {
2052 target = State.TERMINATED_CONNECTIVITY_ERROR;
2053 } else {
2054 target = State.TERMINATED_APPLICATION_FAILURE;
2055 }
2056 transitionOrThrow(target);
2057 this.finish();
2058 }
2059
2060 private void handleIqTimeoutResponse(final IqPacket response) {
2061 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
2062 Log.d(
2063 Config.LOGTAG,
2064 id.account.getJid().asBareJid()
2065 + ": received IQ timeout in RTP session with "
2066 + id.with
2067 + ". terminating with connectivity error");
2068 if (isTerminated()) {
2069 Log.i(
2070 Config.LOGTAG,
2071 id.account.getJid().asBareJid()
2072 + ": ignoring error because session was already terminated");
2073 return;
2074 }
2075 this.webRTCWrapper.close();
2076 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
2077 this.finish();
2078 }
2079
2080 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
2081 Log.d(
2082 Config.LOGTAG,
2083 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
2084 this.webRTCWrapper.close();
2085 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
2086 respondWithOutOfOrder(jinglePacket);
2087 this.finish();
2088 }
2089
2090 private void respondWithTieBreak(final JinglePacket jinglePacket) {
2091 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
2092 }
2093
2094 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
2095 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
2096 }
2097
2098 private void respondWithItemNotFound(final JinglePacket jinglePacket) {
2099 respondWithJingleError(jinglePacket, null, "item-not-found", "cancel");
2100 }
2101
2102 void respondWithJingleError(
2103 final IqPacket original,
2104 String jingleCondition,
2105 String condition,
2106 String conditionType) {
2107 jingleConnectionManager.respondWithJingleError(
2108 id.account, original, jingleCondition, condition, conditionType);
2109 }
2110
2111 private void respondOk(final JinglePacket jinglePacket) {
2112 xmppConnectionService.sendIqPacket(
2113 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
2114 }
2115
2116 public RtpEndUserState getEndUserState() {
2117 switch (this.state) {
2118 case NULL:
2119 case PROPOSED:
2120 case SESSION_INITIALIZED:
2121 if (isInitiator()) {
2122 return RtpEndUserState.RINGING;
2123 } else {
2124 return RtpEndUserState.INCOMING_CALL;
2125 }
2126 case PROCEED:
2127 if (isInitiator()) {
2128 return RtpEndUserState.RINGING;
2129 } else {
2130 return RtpEndUserState.ACCEPTING_CALL;
2131 }
2132 case SESSION_INITIALIZED_PRE_APPROVED:
2133 if (isInitiator()) {
2134 return RtpEndUserState.RINGING;
2135 } else {
2136 return RtpEndUserState.CONNECTING;
2137 }
2138 case SESSION_ACCEPTED:
2139 final ContentAddition ca = getPendingContentAddition();
2140 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2141 return RtpEndUserState.INCOMING_CONTENT_ADD;
2142 }
2143 return getPeerConnectionStateAsEndUserState();
2144 case REJECTED:
2145 case REJECTED_RACED:
2146 case TERMINATED_DECLINED_OR_BUSY:
2147 if (isInitiator()) {
2148 return RtpEndUserState.DECLINED_OR_BUSY;
2149 } else {
2150 return RtpEndUserState.ENDED;
2151 }
2152 case TERMINATED_SUCCESS:
2153 case ACCEPTED:
2154 case RETRACTED:
2155 case TERMINATED_CANCEL_OR_TIMEOUT:
2156 return RtpEndUserState.ENDED;
2157 case RETRACTED_RACED:
2158 if (isInitiator()) {
2159 return RtpEndUserState.ENDED;
2160 } else {
2161 return RtpEndUserState.RETRACTED;
2162 }
2163 case TERMINATED_CONNECTIVITY_ERROR:
2164 return zeroDuration()
2165 ? RtpEndUserState.CONNECTIVITY_ERROR
2166 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2167 case TERMINATED_APPLICATION_FAILURE:
2168 return RtpEndUserState.APPLICATION_ERROR;
2169 case TERMINATED_SECURITY_ERROR:
2170 return RtpEndUserState.SECURITY_ERROR;
2171 }
2172 throw new IllegalStateException(
2173 String.format("%s has no equivalent EndUserState", this.state));
2174 }
2175
2176 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2177 final PeerConnection.PeerConnectionState state;
2178 try {
2179 state = webRTCWrapper.getState();
2180 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2181 // We usually close the WebRTCWrapper *before* transitioning so we might still
2182 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2183 return RtpEndUserState.ENDING_CALL;
2184 }
2185 switch (state) {
2186 case CONNECTED:
2187 return RtpEndUserState.CONNECTED;
2188 case NEW:
2189 case CONNECTING:
2190 return RtpEndUserState.CONNECTING;
2191 case CLOSED:
2192 return RtpEndUserState.ENDING_CALL;
2193 default:
2194 return zeroDuration()
2195 ? RtpEndUserState.CONNECTIVITY_ERROR
2196 : RtpEndUserState.RECONNECTING;
2197 }
2198 }
2199
2200 public ContentAddition getPendingContentAddition() {
2201 final RtpContentMap in = this.incomingContentAdd;
2202 final RtpContentMap out = this.outgoingContentAdd;
2203 if (out != null) {
2204 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2205 } else if (in != null) {
2206 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2207 } else {
2208 return null;
2209 }
2210 }
2211
2212 public Set<Media> getMedia() {
2213 final State current = getState();
2214 if (current == State.NULL) {
2215 if (isInitiator()) {
2216 return Preconditions.checkNotNull(
2217 this.proposedMedia, "RTP connection has not been initialized properly");
2218 }
2219 throw new IllegalStateException("RTP connection has not been initialized yet");
2220 }
2221 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2222 return Preconditions.checkNotNull(
2223 this.proposedMedia, "RTP connection has not been initialized properly");
2224 }
2225 final RtpContentMap localContentMap = getLocalContentMap();
2226 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2227 if (localContentMap != null) {
2228 return localContentMap.getMedia();
2229 } else if (initiatorContentMap != null) {
2230 return initiatorContentMap.getMedia();
2231 } else if (isTerminated()) {
2232 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2233 } else {
2234 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2235 }
2236 }
2237
2238 public boolean isVerified() {
2239 final String fingerprint = this.omemoVerification.getFingerprint();
2240 if (fingerprint == null) {
2241 return false;
2242 }
2243 final FingerprintStatus status =
2244 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2245 return status != null && status.isVerified();
2246 }
2247
2248 public boolean addMedia(final Media media) {
2249 final Set<Media> currentMedia = getMedia();
2250 if (currentMedia.contains(media)) {
2251 throw new IllegalStateException(String.format("%s has already been proposed", media));
2252 }
2253 // TODO add state protection - can only add while ACCEPTED or so
2254 Log.d(Config.LOGTAG,"adding media: "+media);
2255 return webRTCWrapper.addTrack(media);
2256 }
2257
2258 public synchronized void acceptCall() {
2259 switch (this.state) {
2260 case PROPOSED:
2261 cancelRingingTimeout();
2262 acceptCallFromProposed();
2263 break;
2264 case SESSION_INITIALIZED:
2265 cancelRingingTimeout();
2266 acceptCallFromSessionInitialized();
2267 break;
2268 case ACCEPTED:
2269 Log.w(
2270 Config.LOGTAG,
2271 id.account.getJid().asBareJid()
2272 + ": the call has already been accepted with another client. UI was just lagging behind");
2273 break;
2274 case PROCEED:
2275 case SESSION_ACCEPTED:
2276 Log.w(
2277 Config.LOGTAG,
2278 id.account.getJid().asBareJid()
2279 + ": the call has already been accepted. user probably double tapped the UI");
2280 break;
2281 default:
2282 throw new IllegalStateException("Can not accept call from " + this.state);
2283 }
2284 }
2285
2286 public void notifyPhoneCall() {
2287 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2288 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2289 rejectCall();
2290 } else {
2291 endCall();
2292 }
2293 }
2294
2295 public synchronized void rejectCall() {
2296 if (isTerminated()) {
2297 Log.w(
2298 Config.LOGTAG,
2299 id.account.getJid().asBareJid()
2300 + ": received rejectCall() when session has already been terminated. nothing to do");
2301 return;
2302 }
2303 switch (this.state) {
2304 case PROPOSED:
2305 rejectCallFromProposed();
2306 break;
2307 case SESSION_INITIALIZED:
2308 rejectCallFromSessionInitiate();
2309 break;
2310 default:
2311 throw new IllegalStateException("Can not reject call from " + this.state);
2312 }
2313 }
2314
2315 public synchronized void endCall() {
2316 if (isTerminated()) {
2317 Log.w(
2318 Config.LOGTAG,
2319 id.account.getJid().asBareJid()
2320 + ": received endCall() when session has already been terminated. nothing to do");
2321 return;
2322 }
2323 if (isInState(State.PROPOSED) && !isInitiator()) {
2324 rejectCallFromProposed();
2325 return;
2326 }
2327 if (isInState(State.PROCEED)) {
2328 if (isInitiator()) {
2329 retractFromProceed();
2330 } else {
2331 rejectCallFromProceed();
2332 }
2333 return;
2334 }
2335 if (isInitiator()
2336 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2337 this.webRTCWrapper.close();
2338 sendSessionTerminate(Reason.CANCEL);
2339 return;
2340 }
2341 if (isInState(State.SESSION_INITIALIZED)) {
2342 rejectCallFromSessionInitiate();
2343 return;
2344 }
2345 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2346 this.webRTCWrapper.close();
2347 sendSessionTerminate(Reason.SUCCESS);
2348 return;
2349 }
2350 if (isInState(
2351 State.TERMINATED_APPLICATION_FAILURE,
2352 State.TERMINATED_CONNECTIVITY_ERROR,
2353 State.TERMINATED_DECLINED_OR_BUSY)) {
2354 Log.d(
2355 Config.LOGTAG,
2356 "ignoring request to end call because already in state " + this.state);
2357 return;
2358 }
2359 throw new IllegalStateException(
2360 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2361 }
2362
2363 private void retractFromProceed() {
2364 Log.d(Config.LOGTAG, "retract from proceed");
2365 this.sendJingleMessage("retract");
2366 closeTransitionLogFinish(State.RETRACTED_RACED);
2367 }
2368
2369 private void closeTransitionLogFinish(final State state) {
2370 this.webRTCWrapper.close();
2371 transitionOrThrow(state);
2372 writeLogMessage(state);
2373 finish();
2374 }
2375
2376 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2377 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2378 final Presence presence = id.getContact().getPresences().get(id.getWith().getResource());
2379 if (presence != null && presence.getServiceDiscoveryResult().getFeatures().contains("urn:ietf:rfc:3264")) webRTCWrapper.setRFC3264(true);
2380 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2381 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2382 }
2383
2384 private void acceptCallFromProposed() {
2385 transitionOrThrow(State.PROCEED);
2386 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2387 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2388 this.sendJingleMessage("proceed");
2389 }
2390
2391 private void rejectCallFromProposed() {
2392 transitionOrThrow(State.REJECTED);
2393 writeLogMessageMissed();
2394 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2395 this.sendJingleMessage("reject");
2396 finish();
2397 }
2398
2399 private void rejectCallFromProceed() {
2400 this.sendJingleMessage("reject");
2401 closeTransitionLogFinish(State.REJECTED_RACED);
2402 }
2403
2404 private void rejectCallFromSessionInitiate() {
2405 webRTCWrapper.close();
2406 sendSessionTerminate(Reason.DECLINE);
2407 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2408 }
2409
2410 private void sendJingleMessage(final String action) {
2411 sendJingleMessage(action, id.with);
2412 }
2413
2414 private void sendJingleMessage(final String action, final Jid to) {
2415 final MessagePacket messagePacket = new MessagePacket();
2416 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2417 messagePacket.setTo(to);
2418 final Element intent =
2419 messagePacket
2420 .addChild(action, Namespace.JINGLE_MESSAGE)
2421 .setAttribute("id", id.sessionId);
2422 if ("proceed".equals(action)) {
2423 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2424 if (isOmemoEnabled()) {
2425 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2426 final Element device =
2427 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2428 device.setAttribute("id", deviceId);
2429 }
2430 }
2431 messagePacket.addChild("store", "urn:xmpp:hints");
2432 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2433 }
2434
2435 private boolean isOmemoEnabled() {
2436 final Conversational conversational = message.getConversation();
2437 if (conversational instanceof Conversation) {
2438 return ((Conversation) conversational).getNextEncryption()
2439 == Message.ENCRYPTION_AXOLOTL;
2440 }
2441 return false;
2442 }
2443
2444 private void acceptCallFromSessionInitialized() {
2445 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2446 sendSessionAccept();
2447 }
2448
2449 private synchronized boolean isInState(State... state) {
2450 return Arrays.asList(state).contains(this.state);
2451 }
2452
2453 private boolean transition(final State target) {
2454 return transition(target, null);
2455 }
2456
2457 private synchronized boolean transition(final State target, final Runnable runnable) {
2458 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2459 if (validTransitions != null && validTransitions.contains(target)) {
2460 this.state = target;
2461 if (runnable != null) {
2462 runnable.run();
2463 }
2464 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2465 updateEndUserState();
2466 updateOngoingCallNotification();
2467 return true;
2468 } else {
2469 return false;
2470 }
2471 }
2472
2473 void transitionOrThrow(final State target) {
2474 if (!transition(target)) {
2475 throw new IllegalStateException(
2476 String.format("Unable to transition from %s to %s", this.state, target));
2477 }
2478 }
2479
2480 @Override
2481 public void onIceCandidate(final IceCandidate iceCandidate) {
2482 final RtpContentMap rtpContentMap =
2483 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2484 final IceUdpTransportInfo.Credentials credentials;
2485 try {
2486 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2487 } catch (final IllegalArgumentException e) {
2488 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2489 return;
2490 }
2491 final String uFrag = credentials.ufrag;
2492 final IceUdpTransportInfo.Candidate candidate =
2493 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2494 if (candidate == null) {
2495 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2496 return;
2497 }
2498 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2499 sendTransportInfo(iceCandidate.sdpMid, candidate);
2500 }
2501
2502 @Override
2503 public void onIceGatheringComplete(Collection<IceCandidate> iceCandidates) {
2504 iceGatheringComplete.set(iceCandidates);
2505 }
2506
2507 @Override
2508 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2509 Log.d(
2510 Config.LOGTAG,
2511 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2512 this.stateHistory.add(newState);
2513 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2514 this.sessionDuration.start();
2515 updateOngoingCallNotification();
2516 } else if (this.sessionDuration.isRunning()) {
2517 this.sessionDuration.stop();
2518 updateOngoingCallNotification();
2519 }
2520
2521 final boolean neverConnected =
2522 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2523
2524 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2525 if (neverConnected) {
2526 if (isTerminated()) {
2527 Log.d(
2528 Config.LOGTAG,
2529 id.account.getJid().asBareJid()
2530 + ": not sending session-terminate after connectivity error because session is already in state "
2531 + this.state);
2532 return;
2533 }
2534 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2535 return;
2536 } else {
2537 this.restartIce();
2538 }
2539 }
2540 updateEndUserState();
2541 }
2542
2543 private void restartIce() {
2544 this.stateHistory.clear();
2545 this.webRTCWrapper.restartIceAsync();
2546 }
2547
2548 @Override
2549 public void onRenegotiationNeeded() {
2550 this.webRTCWrapper.execute(this::renegotiate);
2551 }
2552
2553 private void renegotiate() {
2554 final SessionDescription sessionDescription;
2555 try {
2556 sessionDescription = setLocalSessionDescription();
2557 } catch (final Exception e) {
2558 final Throwable cause = Throwables.getRootCause(e);
2559 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2560 webRTCWrapper.close();
2561 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2562 return;
2563 }
2564 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2565 final RtpContentMap currentContentMap = getLocalContentMap();
2566 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2567 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2568
2569 Log.d(
2570 Config.LOGTAG,
2571 id.getAccount().getJid().asBareJid()
2572 + ": renegotiate. iceRestart="
2573 + iceRestart
2574 + " content id diff="
2575 + diff);
2576
2577 if (diff.hasModifications() && iceRestart) {
2578 webRTCWrapper.close();
2579 sendSessionTerminate(
2580 Reason.FAILED_APPLICATION,
2581 "WebRTC unexpectedly tried to modify content and transport at once");
2582 return;
2583 }
2584
2585 if (iceRestart) {
2586 initiateIceRestart(rtpContentMap);
2587 return;
2588 } else if (diff.isEmpty()) {
2589 Log.d(
2590 Config.LOGTAG,
2591 "renegotiation. nothing to do. SignalingState="
2592 + this.webRTCWrapper.getSignalingState());
2593 }
2594
2595 if (diff.added.size() > 0) {
2596 modifyLocalContentMap(rtpContentMap);
2597 sendContentAdd(rtpContentMap, diff.added);
2598 }
2599 }
2600
2601 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2602 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2603 final JinglePacket jinglePacket =
2604 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2605 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2606 jinglePacket.setTo(id.with);
2607 xmppConnectionService.sendIqPacket(
2608 id.account,
2609 jinglePacket,
2610 (account, response) -> {
2611 if (response.getType() == IqPacket.TYPE.RESULT) {
2612 Log.d(Config.LOGTAG, "received success to our ice restart");
2613 setLocalContentMap(rtpContentMap);
2614 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2615 return;
2616 }
2617 if (response.getType() == IqPacket.TYPE.ERROR) {
2618 if (isTieBreak(response)) {
2619 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2620 return;
2621 }
2622 handleIqErrorResponse(response);
2623 }
2624 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2625 handleIqTimeoutResponse(response);
2626 }
2627 });
2628 }
2629
2630 private boolean isTieBreak(final IqPacket response) {
2631 final Element error = response.findChild("error");
2632 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2633 }
2634
2635 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2636 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2637 this.outgoingContentAdd = contentAdd;
2638 final JinglePacket jinglePacket =
2639 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2640 jinglePacket.setTo(id.with);
2641 xmppConnectionService.sendIqPacket(
2642 id.account,
2643 jinglePacket,
2644 (connection, response) -> {
2645 if (response.getType() == IqPacket.TYPE.RESULT) {
2646 Log.d(
2647 Config.LOGTAG,
2648 id.getAccount().getJid().asBareJid()
2649 + ": received ACK to our content-add");
2650 return;
2651 }
2652 if (response.getType() == IqPacket.TYPE.ERROR) {
2653 if (isTieBreak(response)) {
2654 this.outgoingContentAdd = null;
2655 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2656 return;
2657 }
2658 handleIqErrorResponse(response);
2659 }
2660 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2661 handleIqTimeoutResponse(response);
2662 }
2663 });
2664 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2665 }
2666
2667 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2668 if (isInitiator()) {
2669 this.initiatorRtpContentMap = rtpContentMap;
2670 } else {
2671 this.responderRtpContentMap = rtpContentMap;
2672 }
2673 }
2674
2675 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2676 if (isInitiator()) {
2677 this.responderRtpContentMap = rtpContentMap;
2678 } else {
2679 this.initiatorRtpContentMap = rtpContentMap;
2680 }
2681 }
2682
2683 // this method is to be used for content map modifications that modify media
2684 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2685 final RtpContentMap activeContents = rtpContentMap.activeContents();
2686 setLocalContentMap(activeContents);
2687 this.webRTCWrapper.switchSpeakerPhonePreference(
2688 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2689 updateEndUserState();
2690 }
2691
2692 private SessionDescription setLocalSessionDescription()
2693 throws ExecutionException, InterruptedException {
2694 final org.webrtc.SessionDescription sessionDescription =
2695 this.webRTCWrapper.setLocalDescription().get();
2696 return SessionDescription.parse(sessionDescription.description);
2697 }
2698
2699 private void closeWebRTCSessionAfterFailedConnection() {
2700 this.webRTCWrapper.close();
2701 synchronized (this) {
2702 if (isTerminated()) {
2703 Log.d(
2704 Config.LOGTAG,
2705 id.account.getJid().asBareJid()
2706 + ": no need to send session-terminate after failed connection. Other party already did");
2707 return;
2708 }
2709 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2710 }
2711 }
2712
2713 public boolean zeroDuration() {
2714 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2715 }
2716
2717 public long getCallDuration() {
2718 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2719 }
2720
2721 public AppRTCAudioManager getAudioManager() {
2722 return webRTCWrapper.getAudioManager();
2723 }
2724
2725 public boolean isMicrophoneEnabled() {
2726 return webRTCWrapper.isMicrophoneEnabled();
2727 }
2728
2729 public boolean setMicrophoneEnabled(final boolean enabled) {
2730 return webRTCWrapper.setMicrophoneEnabled(enabled);
2731 }
2732
2733 public boolean isVideoEnabled() {
2734 return webRTCWrapper.isVideoEnabled();
2735 }
2736
2737 public void setVideoEnabled(final boolean enabled) {
2738 webRTCWrapper.setVideoEnabled(enabled);
2739 }
2740
2741 public boolean isCameraSwitchable() {
2742 return webRTCWrapper.isCameraSwitchable();
2743 }
2744
2745 public boolean isFrontCamera() {
2746 return webRTCWrapper.isFrontCamera();
2747 }
2748
2749 public ListenableFuture<Boolean> switchCamera() {
2750 return webRTCWrapper.switchCamera();
2751 }
2752
2753 @Override
2754 public void onAudioDeviceChanged(
2755 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2756 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2757 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2758 selectedAudioDevice, availableAudioDevices);
2759 }
2760
2761 private void updateEndUserState() {
2762 final RtpEndUserState endUserState = getEndUserState();
2763 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2764 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2765 id.account, id.with, id.sessionId, endUserState);
2766 }
2767
2768 private void updateOngoingCallNotification() {
2769 final State state = this.state;
2770 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2771 final boolean reconnecting;
2772 if (state == State.SESSION_ACCEPTED) {
2773 reconnecting =
2774 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2775 } else {
2776 reconnecting = false;
2777 }
2778 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2779 } else {
2780 xmppConnectionService.removeOngoingCall();
2781 }
2782 }
2783
2784 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2785 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2786 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2787 request.setTo(id.account.getDomain());
2788 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2789 xmppConnectionService.sendIqPacket(
2790 id.account,
2791 request,
2792 (account, response) -> {
2793 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2794 new ImmutableList.Builder<>();
2795 if (response.getType() == IqPacket.TYPE.RESULT) {
2796 final Element services =
2797 response.findChild(
2798 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2799 final List<Element> children =
2800 services == null
2801 ? Collections.emptyList()
2802 : services.getChildren();
2803 for (final Element child : children) {
2804 if ("service".equals(child.getName())) {
2805 final String type = child.getAttribute("type");
2806 final String host = child.getAttribute("host");
2807 final String sport = child.getAttribute("port");
2808 final Integer port =
2809 sport == null ? null : Ints.tryParse(sport);
2810 final String transport = child.getAttribute("transport");
2811 final String username = child.getAttribute("username");
2812 final String password = child.getAttribute("password");
2813 if (Strings.isNullOrEmpty(host) || port == null) {
2814 continue;
2815 }
2816 if (port < 0 || port > 65535) {
2817 continue;
2818 }
2819
2820
2821
2822
2823 if (Arrays.asList("stun", "stuns", "turn", "turns")
2824 .contains(type)
2825 && Arrays.asList("udp", "tcp").contains(transport)) {
2826 if (Arrays.asList("stuns", "turns").contains(type)
2827 && "udp".equals(transport)) {
2828 Log.d(
2829 Config.LOGTAG,
2830 id.account.getJid().asBareJid()
2831 + ": skipping invalid combination of udp/tls in external services");
2832 continue;
2833 }
2834
2835 // STUN URLs do not support a query section since M110
2836 final String uri;
2837 if (Arrays.asList("stun","stuns").contains(type)) {
2838 uri = String.format("%s:%s%s", type, IP.wrapIPv6(host),port);
2839 } else {
2840 uri = String.format(
2841 "%s:%s:%s?transport=%s",
2842 type,
2843 IP.wrapIPv6(host),
2844 port,
2845 transport);
2846 }
2847
2848 final PeerConnection.IceServer.Builder iceServerBuilder =
2849 PeerConnection.IceServer.builder(uri);
2850 iceServerBuilder.setTlsCertPolicy(
2851 PeerConnection.TlsCertPolicy
2852 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2853 if (username != null && password != null) {
2854 iceServerBuilder.setUsername(username);
2855 iceServerBuilder.setPassword(password);
2856 } else if (Arrays.asList("turn", "turns").contains(type)) {
2857 // The WebRTC spec requires throwing an
2858 // InvalidAccessError when username (from libwebrtc
2859 // source coder)
2860 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2861 Log.d(
2862 Config.LOGTAG,
2863 id.account.getJid().asBareJid()
2864 + ": skipping "
2865 + type
2866 + "/"
2867 + transport
2868 + " without username and password");
2869 continue;
2870 }
2871 final PeerConnection.IceServer iceServer =
2872 iceServerBuilder.createIceServer();
2873 Log.d(
2874 Config.LOGTAG,
2875 id.account.getJid().asBareJid()
2876 + ": discovered ICE Server: "
2877 + iceServer);
2878 listBuilder.add(iceServer);
2879 }
2880 }
2881 }
2882 }
2883 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2884 if (iceServers.size() == 0) {
2885 Log.w(
2886 Config.LOGTAG,
2887 id.account.getJid().asBareJid()
2888 + ": no ICE server found "
2889 + response);
2890 }
2891 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2892 });
2893 } else {
2894 Log.w(
2895 Config.LOGTAG,
2896 id.account.getJid().asBareJid() + ": has no external service discovery");
2897 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2898 }
2899 }
2900
2901 private void finish() {
2902 if (isTerminated()) {
2903 this.cancelRingingTimeout();
2904 this.webRTCWrapper.verifyClosed();
2905 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2906 this.jingleConnectionManager.finishConnectionOrThrow(this);
2907 try {
2908 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2909 log.getParentFile().mkdirs();
2910 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2911 } catch (final IOException e) { }
2912 } else {
2913 throw new IllegalStateException(
2914 String.format("Unable to call finish from %s", this.state));
2915 }
2916 }
2917
2918 private void writeLogMessage(final State state) {
2919 final long duration = getCallDuration();
2920 if (state == State.TERMINATED_SUCCESS
2921 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2922 writeLogMessageSuccess(duration);
2923 } else {
2924 writeLogMessageMissed();
2925 }
2926 }
2927
2928 private void writeLogMessageSuccess(final long duration) {
2929 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2930 this.writeMessage();
2931 }
2932
2933 private void writeLogMessageMissed() {
2934 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2935 this.writeMessage();
2936 }
2937
2938 private void writeMessage() {
2939 final Conversational conversational = message.getConversation();
2940 if (conversational instanceof Conversation) {
2941 ((Conversation) conversational).add(this.message);
2942 xmppConnectionService.createMessageAsync(message);
2943 xmppConnectionService.updateConversationUi();
2944 } else {
2945 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2946 }
2947 }
2948
2949 public State getState() {
2950 return this.state;
2951 }
2952
2953 boolean isTerminated() {
2954 return TERMINATED.contains(this.state);
2955 }
2956
2957 public Optional<VideoTrack> getLocalVideoTrack() {
2958 return webRTCWrapper.getLocalVideoTrack();
2959 }
2960
2961 public Optional<VideoTrack> getRemoteVideoTrack() {
2962 return webRTCWrapper.getRemoteVideoTrack();
2963 }
2964
2965 public EglBase.Context getEglBaseContext() {
2966 return webRTCWrapper.getEglBaseContext();
2967 }
2968
2969 void setProposedMedia(final Set<Media> media) {
2970 this.proposedMedia = media;
2971 }
2972
2973 public void fireStateUpdate() {
2974 final RtpEndUserState endUserState = getEndUserState();
2975 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2976 id.account, id.with, id.sessionId, endUserState);
2977 }
2978
2979 public boolean isSwitchToVideoAvailable() {
2980 final boolean prerequisite =
2981 Media.audioOnly(getMedia())
2982 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2983 .contains(getEndUserState());
2984 return prerequisite && remoteHasVideoFeature();
2985 }
2986
2987 private boolean remoteHasVideoFeature() {
2988 final Contact contact = id.getContact();
2989 final Presence presence =
2990 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2991 final ServiceDiscoveryResult serviceDiscoveryResult =
2992 presence == null ? null : presence.getServiceDiscoveryResult();
2993 final List<String> features =
2994 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
2995 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
2996 }
2997
2998 private interface OnIceServersDiscovered {
2999 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
3000 }
3001}