1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Optional;
9import com.google.common.base.Preconditions;
10import com.google.common.base.Stopwatch;
11import com.google.common.base.Strings;
12import com.google.common.base.Throwables;
13import com.google.common.collect.Collections2;
14import com.google.common.collect.ImmutableList;
15import com.google.common.collect.ImmutableMap;
16import com.google.common.collect.Sets;
17import com.google.common.primitives.Ints;
18import com.google.common.util.concurrent.FutureCallback;
19import com.google.common.util.concurrent.Futures;
20import com.google.common.util.concurrent.ListenableFuture;
21import com.google.common.util.concurrent.MoreExecutors;
22
23import org.webrtc.DtmfSender;
24import org.webrtc.EglBase;
25import org.webrtc.IceCandidate;
26import org.webrtc.PeerConnection;
27import org.webrtc.VideoTrack;
28
29import java.util.Arrays;
30import java.util.Collection;
31import java.util.Collections;
32import java.util.LinkedList;
33import java.util.List;
34import java.util.Map;
35import java.util.Queue;
36import java.util.Set;
37import java.util.concurrent.ExecutionException;
38import java.util.concurrent.ScheduledFuture;
39import java.util.concurrent.TimeUnit;
40
41import eu.siacs.conversations.Config;
42import eu.siacs.conversations.crypto.axolotl.AxolotlService;
43import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
44import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
45import eu.siacs.conversations.entities.Account;
46import eu.siacs.conversations.entities.Conversation;
47import eu.siacs.conversations.entities.Conversational;
48import eu.siacs.conversations.entities.Message;
49import eu.siacs.conversations.entities.RtpSessionStatus;
50import eu.siacs.conversations.services.AppRTCAudioManager;
51import eu.siacs.conversations.utils.IP;
52import eu.siacs.conversations.xml.Element;
53import eu.siacs.conversations.xml.Namespace;
54import eu.siacs.conversations.xmpp.Jid;
55import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
56import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
57import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
58import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
59import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
60import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
61import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
62import eu.siacs.conversations.xmpp.stanzas.IqPacket;
63import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
64
65public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
66
67 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
68 State.PROCEED,
69 State.SESSION_INITIALIZED_PRE_APPROVED,
70 State.SESSION_ACCEPTED
71 );
72 private static final long BUSY_TIME_OUT = 30;
73 private static final List<State> TERMINATED = Arrays.asList(
74 State.ACCEPTED,
75 State.REJECTED,
76 State.REJECTED_RACED,
77 State.RETRACTED,
78 State.RETRACTED_RACED,
79 State.TERMINATED_SUCCESS,
80 State.TERMINATED_DECLINED_OR_BUSY,
81 State.TERMINATED_CONNECTIVITY_ERROR,
82 State.TERMINATED_CANCEL_OR_TIMEOUT,
83 State.TERMINATED_APPLICATION_FAILURE,
84 State.TERMINATED_SECURITY_ERROR
85 );
86
87 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
88
89 static {
90 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
91 transitionBuilder.put(State.NULL, ImmutableList.of(
92 State.PROPOSED,
93 State.SESSION_INITIALIZED,
94 State.TERMINATED_APPLICATION_FAILURE,
95 State.TERMINATED_SECURITY_ERROR
96 ));
97 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
98 State.ACCEPTED,
99 State.PROCEED,
100 State.REJECTED,
101 State.RETRACTED,
102 State.TERMINATED_APPLICATION_FAILURE,
103 State.TERMINATED_SECURITY_ERROR,
104 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
105 ));
106 transitionBuilder.put(State.PROCEED, ImmutableList.of(
107 State.REJECTED_RACED,
108 State.RETRACTED_RACED,
109 State.SESSION_INITIALIZED_PRE_APPROVED,
110 State.TERMINATED_SUCCESS,
111 State.TERMINATED_APPLICATION_FAILURE,
112 State.TERMINATED_SECURITY_ERROR,
113 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
114 ));
115 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
116 State.SESSION_ACCEPTED,
117 State.TERMINATED_SUCCESS,
118 State.TERMINATED_DECLINED_OR_BUSY,
119 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
120 State.TERMINATED_CANCEL_OR_TIMEOUT,
121 State.TERMINATED_APPLICATION_FAILURE,
122 State.TERMINATED_SECURITY_ERROR
123 ));
124 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
125 State.SESSION_ACCEPTED,
126 State.TERMINATED_SUCCESS,
127 State.TERMINATED_DECLINED_OR_BUSY,
128 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
129 State.TERMINATED_CANCEL_OR_TIMEOUT,
130 State.TERMINATED_APPLICATION_FAILURE,
131 State.TERMINATED_SECURITY_ERROR
132 ));
133 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
134 State.TERMINATED_SUCCESS,
135 State.TERMINATED_DECLINED_OR_BUSY,
136 State.TERMINATED_CONNECTIVITY_ERROR,
137 State.TERMINATED_CANCEL_OR_TIMEOUT,
138 State.TERMINATED_APPLICATION_FAILURE,
139 State.TERMINATED_SECURITY_ERROR
140 ));
141 VALID_TRANSITIONS = transitionBuilder.build();
142 }
143
144 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
145 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>> pendingIceCandidates = new LinkedList<>();
146 private final OmemoVerification omemoVerification = new OmemoVerification();
147 private final Message message;
148 private State state = State.NULL;
149 private StateTransitionException stateTransitionException;
150 private Set<Media> proposedMedia;
151 private RtpContentMap initiatorRtpContentMap;
152 private RtpContentMap responderRtpContentMap;
153 private IceUdpTransportInfo.Setup peerDtlsSetup;
154 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
155 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
156 private ScheduledFuture<?> ringingTimeoutFuture;
157
158 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
159 super(jingleConnectionManager, id, initiator);
160 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
161 id.account,
162 id.with.asBareJid(),
163 false,
164 false
165 );
166 this.message = new Message(
167 conversation,
168 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
169 Message.TYPE_RTP_SESSION,
170 id.sessionId
171 );
172 }
173
174 private static State reasonToState(Reason reason) {
175 switch (reason) {
176 case SUCCESS:
177 return State.TERMINATED_SUCCESS;
178 case DECLINE:
179 case BUSY:
180 return State.TERMINATED_DECLINED_OR_BUSY;
181 case CANCEL:
182 case TIMEOUT:
183 return State.TERMINATED_CANCEL_OR_TIMEOUT;
184 case SECURITY_ERROR:
185 return State.TERMINATED_SECURITY_ERROR;
186 case FAILED_APPLICATION:
187 case UNSUPPORTED_TRANSPORTS:
188 case UNSUPPORTED_APPLICATIONS:
189 return State.TERMINATED_APPLICATION_FAILURE;
190 default:
191 return State.TERMINATED_CONNECTIVITY_ERROR;
192 }
193 }
194
195 @Override
196 synchronized void deliverPacket(final JinglePacket jinglePacket) {
197 switch (jinglePacket.getAction()) {
198 case SESSION_INITIATE:
199 receiveSessionInitiate(jinglePacket);
200 break;
201 case TRANSPORT_INFO:
202 receiveTransportInfo(jinglePacket);
203 break;
204 case SESSION_ACCEPT:
205 receiveSessionAccept(jinglePacket);
206 break;
207 case SESSION_TERMINATE:
208 receiveSessionTerminate(jinglePacket);
209 break;
210 default:
211 respondOk(jinglePacket);
212 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
213 break;
214 }
215 }
216
217 @Override
218 synchronized void notifyRebound() {
219 if (isTerminated()) {
220 return;
221 }
222 webRTCWrapper.close();
223 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
224 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
225 }
226 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
227 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
228 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
229 } else {
230 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
231 finish();
232 }
233 }
234
235 public boolean applyDtmfTone(String tone) {
236 return webRTCWrapper.applyDtmfTone(tone);
237 }
238
239 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
240 respondOk(jinglePacket);
241 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
242 final State previous = this.state;
243 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
244 if (TERMINATED.contains(previous)) {
245 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
246 return;
247 }
248 webRTCWrapper.close();
249 final State target = reasonToState(wrapper.reason);
250 transitionOrThrow(target);
251 writeLogMessage(target);
252 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
253 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
254 }
255 finish();
256 }
257
258 private void receiveTransportInfo(final JinglePacket jinglePacket) {
259 //Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to INITIALIZED only after transport-info has been received
260 if (isInState(State.NULL, State.PROCEED, State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
261 final RtpContentMap contentMap;
262 try {
263 contentMap = RtpContentMap.of(jinglePacket);
264 } catch (final IllegalArgumentException | NullPointerException e) {
265 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
266 respondOk(jinglePacket);
267 return;
268 }
269 receiveTransportInfo(jinglePacket, contentMap);
270 } else {
271 if (isTerminated()) {
272 respondOk(jinglePacket);
273 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated");
274 } else {
275 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
276 terminateWithOutOfOrder(jinglePacket);
277 }
278 }
279 }
280
281 private void receiveTransportInfo(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
282 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
283 if (this.state == State.SESSION_ACCEPTED) {
284 //zero candidates + modified credentials are an ICE restart offer
285 if (checkForIceRestart(jinglePacket, contentMap)) {
286 return;
287 }
288 respondOk(jinglePacket);
289 try {
290 processCandidates(candidates);
291 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
292 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
293 }
294 } else {
295 respondOk(jinglePacket);
296 pendingIceCandidates.addAll(candidates);
297 }
298 }
299
300 private boolean checkForIceRestart(final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
301 final RtpContentMap existing = getRemoteContentMap();
302 final IceUdpTransportInfo.Credentials existingCredentials;
303 final IceUdpTransportInfo.Credentials newCredentials;
304 try {
305 existingCredentials = existing.getCredentials();
306 newCredentials = rtpContentMap.getCredentials();
307 } catch (final IllegalStateException e) {
308 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
309 return false;
310 }
311 if (existingCredentials.equals(newCredentials)) {
312 return false;
313 }
314 //TODO an alternative approach is to check if we already got an iq result to our ICE-restart
315 // and if that's the case we are seeing an answer.
316 // This might be more spec compliant but also more error prone potentially
317 final boolean isOffer = rtpContentMap.emptyCandidates();
318 final RtpContentMap restartContentMap;
319 try {
320 if (isOffer) {
321 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
322 restartContentMap = existing.modifiedCredentials(newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
323 } else {
324 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
325 Log.d(Config.LOGTAG, "received confirmation of ICE restart" + newCredentials + " peer_setup=" + setup);
326 // DTLS setup attribute needs to be rewritten to reflect current peer state
327 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
328 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
329 }
330 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
331 return isOffer;
332 } else {
333 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
334 respondWithTieBreak(jinglePacket);
335 return true;
336 }
337 } catch (final Exception exception) {
338 respondOk(jinglePacket);
339 final Throwable rootCause = Throwables.getRootCause(exception);
340 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
341 //If this happens a termination is already in progress
342 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
343 return true;
344 }
345 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
346 webRTCWrapper.close();
347 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
348 return true;
349 }
350 }
351
352 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
353 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
354 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
355 throw new IllegalStateException("Invalid peer setup");
356 }
357 return peerSetup;
358 }
359
360 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
361 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
362 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
363 }
364 this.peerDtlsSetup = setup;
365 }
366
367 private boolean applyIceRestart(final JinglePacket jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer) throws ExecutionException, InterruptedException {
368 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap);
369 final org.webrtc.SessionDescription.Type type = isOffer ? org.webrtc.SessionDescription.Type.OFFER : org.webrtc.SessionDescription.Type.ANSWER;
370 org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(type, sessionDescription.toString());
371 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
372 if (isInitiator()) {
373 //We ignore the offer and respond with tie-break. This will clause the responder not to apply the content map
374 return false;
375 }
376 }
377 webRTCWrapper.setRemoteDescription(sdp).get();
378 setRemoteContentMap(restartContentMap);
379 if (isOffer) {
380 webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
381 final SessionDescription localSessionDescription = setLocalSessionDescription();
382 setLocalContentMap(RtpContentMap.of(localSessionDescription));
383 //We need to respond OK before sending any candidates
384 respondOk(jinglePacket);
385 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
386 } else {
387 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
388 }
389 return true;
390 }
391
392 private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
393 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
394 processCandidate(content);
395 }
396 }
397
398 private void processCandidate(final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
399 final RtpContentMap rtpContentMap = getRemoteContentMap();
400 final List<String> indices = toIdentificationTags(rtpContentMap);
401 final String sdpMid = content.getKey(); //aka content name
402 final IceUdpTransportInfo transport = content.getValue().transport;
403 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
404
405 //TODO check that credentials remained the same
406
407 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
408 final String sdp;
409 try {
410 sdp = candidate.toSdpAttribute(credentials.ufrag);
411 } catch (final IllegalArgumentException e) {
412 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
413 continue;
414 }
415 final int mLineIndex = indices.indexOf(sdpMid);
416 if (mLineIndex < 0) {
417 Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
418 }
419 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
420 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
421 this.webRTCWrapper.addIceCandidate(iceCandidate);
422 }
423 }
424
425 private RtpContentMap getRemoteContentMap() {
426 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
427 }
428
429 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
430 final Group originalGroup = rtpContentMap.group;
431 final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
432 if (identificationTags.size() == 0) {
433 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
434 }
435 return identificationTags;
436 }
437
438 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
439 final RtpContentMap receivedContentMap;
440 try {
441 receivedContentMap = RtpContentMap.of(jinglePacket);
442 } catch (final Exception e) {
443 return Futures.immediateFailedFuture(e);
444 }
445 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
446 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
447 return Futures.transform(future, omemoVerifiedPayload -> {
448 //TODO test if an exception here triggers a correct abort
449 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
450 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received verifiable DTLS fingerprint via " + omemoVerification);
451 return omemoVerifiedPayload.getPayload();
452 }, MoreExecutors.directExecutor());
453 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
454 return Futures.immediateFailedFuture(
455 new SecurityException("DTLS fingerprint was unexpectedly not verifiable")
456 );
457 } else {
458 return Futures.immediateFuture(receivedContentMap);
459 }
460 }
461
462 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
463 if (isInitiator()) {
464 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
465 if (isTerminated()) {
466 Log.d(Config.LOGTAG, String.format(
467 "%s: got a reason to terminate with out-of-order. but already in state %s",
468 id.account.getJid().asBareJid(),
469 getState()
470 ));
471 respondWithOutOfOrder(jinglePacket);
472 } else {
473 terminateWithOutOfOrder(jinglePacket);
474 }
475 return;
476 }
477 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
478 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
479 @Override
480 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
481 receiveSessionInitiate(jinglePacket, rtpContentMap);
482 }
483
484 @Override
485 public void onFailure(@NonNull final Throwable throwable) {
486 respondOk(jinglePacket);
487 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
488 }
489 }, MoreExecutors.directExecutor());
490 }
491
492 private void receiveSessionInitiate(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
493 try {
494 contentMap.requireContentDescriptions();
495 contentMap.requireDTLSFingerprint(true);
496 } catch (final RuntimeException e) {
497 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e));
498 respondOk(jinglePacket);
499 sendSessionTerminate(Reason.of(e), e.getMessage());
500 return;
501 }
502 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
503 final State target;
504 if (this.state == State.PROCEED) {
505 Preconditions.checkState(
506 proposedMedia != null && proposedMedia.size() > 0,
507 "proposed media must be set when processing pre-approved session-initiate"
508 );
509 if (!this.proposedMedia.equals(contentMap.getMedia())) {
510 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
511 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
512 this.proposedMedia,
513 contentMap.getMedia()
514 ));
515 return;
516 }
517 target = State.SESSION_INITIALIZED_PRE_APPROVED;
518 } else {
519 target = State.SESSION_INITIALIZED;
520 }
521 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
522 respondOk(jinglePacket);
523 pendingIceCandidates.addAll(contentMap.contents.entrySet());
524 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
525 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
526 sendSessionAccept();
527 } else {
528 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
529 startRinging();
530 }
531 } else {
532 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
533 terminateWithOutOfOrder(jinglePacket);
534 }
535 }
536
537 private void receiveSessionAccept(final JinglePacket jinglePacket) {
538 if (!isInitiator()) {
539 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
540 terminateWithOutOfOrder(jinglePacket);
541 return;
542 }
543 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
544 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
545 @Override
546 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
547 receiveSessionAccept(jinglePacket, rtpContentMap);
548 }
549
550 @Override
551 public void onFailure(@NonNull final Throwable throwable) {
552 respondOk(jinglePacket);
553 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", throwable);
554 webRTCWrapper.close();
555 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
556 }
557 }, MoreExecutors.directExecutor());
558 }
559
560 private void receiveSessionAccept(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
561 try {
562 contentMap.requireContentDescriptions();
563 contentMap.requireDTLSFingerprint();
564 } catch (final RuntimeException e) {
565 respondOk(jinglePacket);
566 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
567 webRTCWrapper.close();
568 sendSessionTerminate(Reason.of(e), e.getMessage());
569 return;
570 }
571 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
572 if (!initiatorMedia.equals(contentMap.getMedia())) {
573 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
574 "Your session-included included media %s but our session-initiate was %s",
575 this.proposedMedia,
576 contentMap.getMedia()
577 ));
578 return;
579 }
580 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
581 if (transition(State.SESSION_ACCEPTED)) {
582 respondOk(jinglePacket);
583 receiveSessionAccept(contentMap);
584 } else {
585 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
586 respondOk(jinglePacket);
587 }
588 }
589
590 private void receiveSessionAccept(final RtpContentMap contentMap) {
591 this.responderRtpContentMap = contentMap;
592 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
593 final SessionDescription sessionDescription;
594 try {
595 sessionDescription = SessionDescription.of(contentMap);
596 } catch (final IllegalArgumentException | NullPointerException e) {
597 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
598 webRTCWrapper.close();
599 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
600 return;
601 }
602 final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
603 org.webrtc.SessionDescription.Type.ANSWER,
604 sessionDescription.toString()
605 );
606 try {
607 this.webRTCWrapper.setRemoteDescription(answer).get();
608 } catch (final Exception e) {
609 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e));
610 webRTCWrapper.close();
611 sendSessionTerminate(Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
612 return;
613 }
614 processCandidates(contentMap.contents.entrySet());
615 }
616
617 private void sendSessionAccept() {
618 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
619 if (rtpContentMap == null) {
620 throw new IllegalStateException("initiator RTP Content Map has not been set");
621 }
622 final SessionDescription offer;
623 try {
624 offer = SessionDescription.of(rtpContentMap);
625 } catch (final IllegalArgumentException | NullPointerException e) {
626 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
627 webRTCWrapper.close();
628 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
629 return;
630 }
631 sendSessionAccept(rtpContentMap.getMedia(), offer);
632 }
633
634 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
635 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
636 }
637
638 private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
639 if (isTerminated()) {
640 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
641 return;
642 }
643 try {
644 setupWebRTC(media, iceServers);
645 } catch (final WebRTCWrapper.InitializationException e) {
646 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
647 webRTCWrapper.close();
648 sendSessionTerminate(Reason.FAILED_APPLICATION);
649 return;
650 }
651 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
652 org.webrtc.SessionDescription.Type.OFFER,
653 offer.toString()
654 );
655 try {
656 this.webRTCWrapper.setRemoteDescription(sdp).get();
657 addIceCandidatesFromBlackLog();
658 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
659 prepareSessionAccept(webRTCSessionDescription);
660 } catch (final Exception e) {
661 failureToAcceptSession(e);
662 }
663 }
664
665 private void failureToAcceptSession(final Throwable throwable) {
666 if (isTerminated()) {
667 return;
668 }
669 final Throwable rootCause = Throwables.getRootCause(throwable);
670 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
671 webRTCWrapper.close();
672 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
673 }
674
675 private void addIceCandidatesFromBlackLog() {
676 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
677 while ((foo = this.pendingIceCandidates.poll()) != null) {
678 processCandidate(foo);
679 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidate from back log");
680 }
681 }
682
683 private void prepareSessionAccept(final org.webrtc.SessionDescription webRTCSessionDescription) {
684 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
685 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
686 this.responderRtpContentMap = respondingRtpContentMap;
687 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
688 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
689 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = prepareOutgoingContentMap(respondingRtpContentMap);
690 Futures.addCallback(outgoingContentMapFuture,
691 new FutureCallback<RtpContentMap>() {
692 @Override
693 public void onSuccess(final RtpContentMap outgoingContentMap) {
694 sendSessionAccept(outgoingContentMap);
695 }
696
697 @Override
698 public void onFailure(@NonNull Throwable throwable) {
699 failureToAcceptSession(throwable);
700 }
701 },
702 MoreExecutors.directExecutor()
703 );
704 }
705
706 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
707 if (isTerminated()) {
708 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session accept was too slow. already terminated. nothing to do.");
709 return;
710 }
711 transitionOrThrow(State.SESSION_ACCEPTED);
712 final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
713 send(sessionAccept);
714 }
715
716 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(final RtpContentMap rtpContentMap) {
717 if (this.omemoVerification.hasDeviceId()) {
718 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
719 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
720 return Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
721 omemoVerification.setOrEnsureEqual(verifiedPayload);
722 return verifiedPayload.getPayload();
723 }, MoreExecutors.directExecutor());
724 } else {
725 return Futures.immediateFuture(rtpContentMap);
726 }
727 }
728
729 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
730 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
731 switch (message.getName()) {
732 case "propose":
733 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
734 break;
735 case "proceed":
736 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
737 break;
738 case "retract":
739 receiveRetract(from, serverMessageId, timestamp);
740 break;
741 case "reject":
742 receiveReject(from, serverMessageId, timestamp);
743 break;
744 case "accept":
745 receiveAccept(from, serverMessageId, timestamp);
746 break;
747 default:
748 break;
749 }
750 }
751
752 void deliverFailedProceed() {
753 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
754 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
755 webRTCWrapper.close();
756 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
757 this.finish();
758 }
759 }
760
761 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
762 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
763 if (originatedFromMyself) {
764 if (transition(State.ACCEPTED)) {
765 if (serverMsgId != null) {
766 this.message.setServerMsgId(serverMsgId);
767 }
768 this.message.setTime(timestamp);
769 this.message.setCarbon(true); //indicate that call was accepted on other device
770 this.writeLogMessageSuccess(0);
771 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
772 this.finish();
773 } else {
774 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
775 }
776 } else {
777 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
778 }
779 }
780
781 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
782 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
783 //reject from another one of my clients
784 if (originatedFromMyself) {
785 receiveRejectFromMyself(serverMsgId, timestamp);
786 } else if (isInitiator()) {
787 if (from.equals(id.with)) {
788 receiveRejectFromResponder();
789 } else {
790 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
791 }
792 } else {
793 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
794 }
795 }
796
797 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
798 if (transition(State.REJECTED)) {
799 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
800 this.finish();
801 if (serverMsgId != null) {
802 this.message.setServerMsgId(serverMsgId);
803 }
804 this.message.setTime(timestamp);
805 this.message.setCarbon(true); //indicate that call was rejected on other device
806 writeLogMessageMissed();
807 } else {
808 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
809 }
810 }
811
812 private void receiveRejectFromResponder() {
813 if (isInState(State.PROCEED)) {
814 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered");
815 closeTransitionLogFinish(State.REJECTED_RACED);
816 return;
817 }
818 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
819 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
820 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
821 return;
822 }
823 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state);
824 }
825
826 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
827 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
828 if (originatedFromMyself) {
829 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
830 } else if (transition(State.PROPOSED, () -> {
831 final Collection<RtpDescription> descriptions = Collections2.transform(
832 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
833 input -> (RtpDescription) input
834 );
835 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
836 Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
837 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
838 this.proposedMedia = Sets.newHashSet(media);
839 })) {
840 if (serverMsgId != null) {
841 this.message.setServerMsgId(serverMsgId);
842 }
843 this.message.setTime(timestamp);
844 startRinging();
845 } else {
846 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
847 }
848 }
849
850 private void startRinging() {
851 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
852 ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
853 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
854 }
855
856 private synchronized void ringingTimeout() {
857 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
858 switch (this.state) {
859 case PROPOSED:
860 message.markUnread();
861 rejectCallFromProposed();
862 break;
863 case SESSION_INITIALIZED:
864 message.markUnread();
865 rejectCallFromSessionInitiate();
866 break;
867 }
868 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
869 }
870
871 private void cancelRingingTimeout() {
872 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
873 if (future != null && !future.isCancelled()) {
874 future.cancel(false);
875 }
876 }
877
878 private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
879 final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
880 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
881 if (from.equals(id.with)) {
882 if (isInitiator()) {
883 if (transition(State.PROCEED)) {
884 if (serverMsgId != null) {
885 this.message.setServerMsgId(serverMsgId);
886 }
887 this.message.setTime(timestamp);
888 final Integer remoteDeviceId = proceed.getDeviceId();
889 if (isOmemoEnabled()) {
890 this.omemoVerification.setDeviceId(remoteDeviceId);
891 } else {
892 if (remoteDeviceId != null) {
893 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
894 }
895 this.omemoVerification.setDeviceId(null);
896 }
897 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
898 } else {
899 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
900 }
901 } else {
902 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
903 }
904 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
905 if (transition(State.ACCEPTED)) {
906 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
907 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
908 this.finish();
909 }
910 } else {
911 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
912 }
913 }
914
915 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
916 if (from.equals(id.with)) {
917 final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
918 if (transition(target)) {
919 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
920 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
921 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
922 if (serverMsgId != null) {
923 this.message.setServerMsgId(serverMsgId);
924 }
925 this.message.setTime(timestamp);
926 if (target == State.RETRACTED) {
927 this.message.markUnread();
928 }
929 writeLogMessageMissed();
930 finish();
931 } else {
932 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
933 }
934 } else {
935 //TODO parse retract from self
936 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
937 }
938 }
939
940 public void sendSessionInitiate() {
941 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
942 }
943
944 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
945 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
946 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
947 }
948
949 private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
950 if (isTerminated()) {
951 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
952 return;
953 }
954 try {
955 setupWebRTC(media, iceServers);
956 } catch (final WebRTCWrapper.InitializationException e) {
957 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
958 webRTCWrapper.close();
959 sendRetract(Reason.ofThrowable(e));
960 return;
961 }
962 try {
963 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
964 prepareSessionInitiate(webRTCSessionDescription, targetState);
965 } catch (final Exception e) {
966 //TODO sending the error text is worthwhile as well. Especially for FailureToSet exceptions
967 failureToInitiateSession(e, targetState);
968 }
969 }
970
971 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
972 if (isTerminated()) {
973 return;
974 }
975 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(throwable));
976 webRTCWrapper.close();
977 final Reason reason = Reason.ofThrowable(throwable);
978 if (isInState(targetState)) {
979 sendSessionTerminate(reason);
980 } else {
981 sendRetract(reason);
982 }
983 }
984
985 private void sendRetract(final Reason reason) {
986 //TODO embed reason into retract
987 sendJingleMessage("retract", id.with.asBareJid());
988 transitionOrThrow(reasonToState(reason));
989 this.finish();
990 }
991
992 private void prepareSessionInitiate(final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
993 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
994 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
995 this.initiatorRtpContentMap = rtpContentMap;
996 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
997 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = encryptSessionInitiate(rtpContentMap);
998 Futures.addCallback(outgoingContentMapFuture, new FutureCallback<RtpContentMap>() {
999 @Override
1000 public void onSuccess(final RtpContentMap outgoingContentMap) {
1001 sendSessionInitiate(outgoingContentMap, targetState);
1002 }
1003
1004 @Override
1005 public void onFailure(@NonNull final Throwable throwable) {
1006 failureToInitiateSession(throwable, targetState);
1007 }
1008 }, MoreExecutors.directExecutor());
1009 }
1010
1011 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1012 if (isTerminated()) {
1013 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session was too slow. already terminated. nothing to do.");
1014 return;
1015 }
1016 this.transitionOrThrow(targetState);
1017 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1018 send(sessionInitiate);
1019 }
1020
1021 private ListenableFuture<RtpContentMap> encryptSessionInitiate(final RtpContentMap rtpContentMap) {
1022 if (this.omemoVerification.hasDeviceId()) {
1023 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
1024 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
1025 final ListenableFuture<RtpContentMap> future = Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
1026 omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint());
1027 return verifiedPayload.getPayload();
1028 }, MoreExecutors.directExecutor());
1029 if (Config.REQUIRE_RTP_VERIFICATION) {
1030 return future;
1031 }
1032 return Futures.catching(
1033 future,
1034 CryptoFailedException.class,
1035 e -> {
1036 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e);
1037 return rtpContentMap;
1038 },
1039 MoreExecutors.directExecutor()
1040 );
1041 } else {
1042 return Futures.immediateFuture(rtpContentMap);
1043 }
1044 }
1045
1046 private void sendSessionTerminate(final Reason reason) {
1047 sendSessionTerminate(reason, null);
1048 }
1049
1050 private void sendSessionTerminate(final Reason reason, final String text) {
1051 final State previous = this.state;
1052 final State target = reasonToState(reason);
1053 transitionOrThrow(target);
1054 if (previous != State.NULL) {
1055 writeLogMessage(target);
1056 }
1057 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1058 jinglePacket.setReason(reason, text);
1059 Log.d(Config.LOGTAG, jinglePacket.toString());
1060 send(jinglePacket);
1061 finish();
1062 }
1063
1064 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
1065 final RtpContentMap transportInfo;
1066 try {
1067 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1068 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1069 } catch (final Exception e) {
1070 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
1071 return;
1072 }
1073 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1074 send(jinglePacket);
1075 }
1076
1077 private void send(final JinglePacket jinglePacket) {
1078 jinglePacket.setTo(id.with);
1079 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1080 }
1081
1082 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1083 if (response.getType() == IqPacket.TYPE.ERROR) {
1084 handleIqErrorResponse(response);
1085 return;
1086 }
1087 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1088 handleIqTimeoutResponse(response);
1089 }
1090 }
1091
1092 private void handleIqErrorResponse(final IqPacket response) {
1093 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1094 final String errorCondition = response.getErrorCondition();
1095 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
1096 if (isTerminated()) {
1097 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
1098 return;
1099 }
1100 this.webRTCWrapper.close();
1101 final State target;
1102 if (Arrays.asList(
1103 "service-unavailable",
1104 "recipient-unavailable",
1105 "remote-server-not-found",
1106 "remote-server-timeout"
1107 ).contains(errorCondition)) {
1108 target = State.TERMINATED_CONNECTIVITY_ERROR;
1109 } else {
1110 target = State.TERMINATED_APPLICATION_FAILURE;
1111 }
1112 transitionOrThrow(target);
1113 this.finish();
1114 }
1115
1116 private void handleIqTimeoutResponse(final IqPacket response) {
1117 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1118 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
1119 if (isTerminated()) {
1120 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
1121 return;
1122 }
1123 this.webRTCWrapper.close();
1124 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
1125 this.finish();
1126 }
1127
1128 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1129 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1130 this.webRTCWrapper.close();
1131 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1132 respondWithOutOfOrder(jinglePacket);
1133 this.finish();
1134 }
1135
1136 private void respondWithTieBreak(final JinglePacket jinglePacket) {
1137 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
1138 }
1139
1140 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1141 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
1142 }
1143
1144 void respondWithJingleError(final IqPacket original, String jingleCondition, String condition, String conditionType) {
1145 jingleConnectionManager.respondWithJingleError(id.account, original, jingleCondition, condition, conditionType);
1146 }
1147
1148 private void respondOk(final JinglePacket jinglePacket) {
1149 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1150 }
1151
1152 public void throwStateTransitionException() {
1153 final StateTransitionException exception = this.stateTransitionException;
1154 if (exception != null) {
1155 throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception);
1156 }
1157 }
1158
1159 public RtpEndUserState getEndUserState() {
1160 switch (this.state) {
1161 case NULL:
1162 case PROPOSED:
1163 case SESSION_INITIALIZED:
1164 if (isInitiator()) {
1165 return RtpEndUserState.RINGING;
1166 } else {
1167 return RtpEndUserState.INCOMING_CALL;
1168 }
1169 case PROCEED:
1170 if (isInitiator()) {
1171 return RtpEndUserState.RINGING;
1172 } else {
1173 return RtpEndUserState.ACCEPTING_CALL;
1174 }
1175 case SESSION_INITIALIZED_PRE_APPROVED:
1176 if (isInitiator()) {
1177 return RtpEndUserState.RINGING;
1178 } else {
1179 return RtpEndUserState.CONNECTING;
1180 }
1181 case SESSION_ACCEPTED:
1182 return getPeerConnectionStateAsEndUserState();
1183 case REJECTED:
1184 case REJECTED_RACED:
1185 case TERMINATED_DECLINED_OR_BUSY:
1186 if (isInitiator()) {
1187 return RtpEndUserState.DECLINED_OR_BUSY;
1188 } else {
1189 return RtpEndUserState.ENDED;
1190 }
1191 case TERMINATED_SUCCESS:
1192 case ACCEPTED:
1193 case RETRACTED:
1194 case TERMINATED_CANCEL_OR_TIMEOUT:
1195 return RtpEndUserState.ENDED;
1196 case RETRACTED_RACED:
1197 if (isInitiator()) {
1198 return RtpEndUserState.ENDED;
1199 } else {
1200 return RtpEndUserState.RETRACTED;
1201 }
1202 case TERMINATED_CONNECTIVITY_ERROR:
1203 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1204 case TERMINATED_APPLICATION_FAILURE:
1205 return RtpEndUserState.APPLICATION_ERROR;
1206 case TERMINATED_SECURITY_ERROR:
1207 return RtpEndUserState.SECURITY_ERROR;
1208 }
1209 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
1210 }
1211
1212
1213 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
1214 final PeerConnection.PeerConnectionState state;
1215 try {
1216 state = webRTCWrapper.getState();
1217 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
1218 //We usually close the WebRTCWrapper *before* transitioning so we might still
1219 //be in SESSION_ACCEPTED even though the peerConnection has been torn down
1220 return RtpEndUserState.ENDING_CALL;
1221 }
1222 switch (state) {
1223 case CONNECTED:
1224 return RtpEndUserState.CONNECTED;
1225 case NEW:
1226 case CONNECTING:
1227 return RtpEndUserState.CONNECTING;
1228 case CLOSED:
1229 return RtpEndUserState.ENDING_CALL;
1230 default:
1231 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.RECONNECTING;
1232 }
1233 }
1234
1235 public Set<Media> getMedia() {
1236 final State current = getState();
1237 if (current == State.NULL) {
1238 if (isInitiator()) {
1239 return Preconditions.checkNotNull(
1240 this.proposedMedia,
1241 "RTP connection has not been initialized properly"
1242 );
1243 }
1244 throw new IllegalStateException("RTP connection has not been initialized yet");
1245 }
1246 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
1247 return Preconditions.checkNotNull(
1248 this.proposedMedia,
1249 "RTP connection has not been initialized properly"
1250 );
1251 }
1252 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
1253 if (initiatorContentMap != null) {
1254 return initiatorContentMap.getMedia();
1255 } else if (isTerminated()) {
1256 return Collections.emptySet(); //we might fail before we ever got a chance to set media
1257 } else {
1258 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
1259 }
1260 }
1261
1262
1263 public boolean isVerified() {
1264 final String fingerprint = this.omemoVerification.getFingerprint();
1265 if (fingerprint == null) {
1266 return false;
1267 }
1268 final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint);
1269 return status != null && status.isVerified();
1270 }
1271
1272 public synchronized void acceptCall() {
1273 switch (this.state) {
1274 case PROPOSED:
1275 cancelRingingTimeout();
1276 acceptCallFromProposed();
1277 break;
1278 case SESSION_INITIALIZED:
1279 cancelRingingTimeout();
1280 acceptCallFromSessionInitialized();
1281 break;
1282 case ACCEPTED:
1283 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind");
1284 break;
1285 case PROCEED:
1286 case SESSION_ACCEPTED:
1287 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI");
1288 break;
1289 default:
1290 throw new IllegalStateException("Can not accept call from " + this.state);
1291 }
1292 }
1293
1294
1295 public void notifyPhoneCall() {
1296 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1297 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1298 rejectCall();
1299 } else {
1300 endCall();
1301 }
1302 }
1303
1304 public synchronized void rejectCall() {
1305 if (isTerminated()) {
1306 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do");
1307 return;
1308 }
1309 switch (this.state) {
1310 case PROPOSED:
1311 rejectCallFromProposed();
1312 break;
1313 case SESSION_INITIALIZED:
1314 rejectCallFromSessionInitiate();
1315 break;
1316 default:
1317 throw new IllegalStateException("Can not reject call from " + this.state);
1318 }
1319 }
1320
1321 public synchronized void endCall() {
1322 if (isTerminated()) {
1323 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
1324 return;
1325 }
1326 if (isInState(State.PROPOSED) && !isInitiator()) {
1327 rejectCallFromProposed();
1328 return;
1329 }
1330 if (isInState(State.PROCEED)) {
1331 if (isInitiator()) {
1332 retractFromProceed();
1333 } else {
1334 rejectCallFromProceed();
1335 }
1336 return;
1337 }
1338 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1339 this.webRTCWrapper.close();
1340 sendSessionTerminate(Reason.CANCEL);
1341 return;
1342 }
1343 if (isInState(State.SESSION_INITIALIZED)) {
1344 rejectCallFromSessionInitiate();
1345 return;
1346 }
1347 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1348 this.webRTCWrapper.close();
1349 sendSessionTerminate(Reason.SUCCESS);
1350 return;
1351 }
1352 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
1353 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
1354 return;
1355 }
1356 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1357 }
1358
1359 private void retractFromProceed() {
1360 Log.d(Config.LOGTAG, "retract from proceed");
1361 this.sendJingleMessage("retract");
1362 closeTransitionLogFinish(State.RETRACTED_RACED);
1363 }
1364
1365 private void closeTransitionLogFinish(final State state) {
1366 this.webRTCWrapper.close();
1367 transitionOrThrow(state);
1368 writeLogMessage(state);
1369 finish();
1370 }
1371
1372 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
1373 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1374 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1375 if (media.contains(Media.VIDEO)) {
1376 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1377 } else {
1378 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1379 }
1380 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1381 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1382 }
1383
1384 private void acceptCallFromProposed() {
1385 transitionOrThrow(State.PROCEED);
1386 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1387 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1388 this.sendJingleMessage("proceed");
1389 }
1390
1391 private void rejectCallFromProposed() {
1392 transitionOrThrow(State.REJECTED);
1393 writeLogMessageMissed();
1394 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1395 this.sendJingleMessage("reject");
1396 finish();
1397 }
1398
1399 private void rejectCallFromProceed() {
1400 this.sendJingleMessage("reject");
1401 closeTransitionLogFinish(State.REJECTED_RACED);
1402 }
1403
1404 private void rejectCallFromSessionInitiate() {
1405 webRTCWrapper.close();
1406 sendSessionTerminate(Reason.DECLINE);
1407 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1408 }
1409
1410 private void sendJingleMessage(final String action) {
1411 sendJingleMessage(action, id.with);
1412 }
1413
1414 private void sendJingleMessage(final String action, final Jid to) {
1415 final MessagePacket messagePacket = new MessagePacket();
1416 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
1417 messagePacket.setTo(to);
1418 final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
1419 if ("proceed".equals(action)) {
1420 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1421 if (isOmemoEnabled()) {
1422 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1423 final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1424 device.setAttribute("id", deviceId);
1425 }
1426 }
1427 messagePacket.addChild("store", "urn:xmpp:hints");
1428 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1429 }
1430
1431 private boolean isOmemoEnabled() {
1432 final Conversational conversational = message.getConversation();
1433 if (conversational instanceof Conversation) {
1434 return ((Conversation) conversational).getNextEncryption() == Message.ENCRYPTION_AXOLOTL;
1435 }
1436 return false;
1437 }
1438
1439 private void acceptCallFromSessionInitialized() {
1440 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1441 sendSessionAccept();
1442 }
1443
1444 private synchronized boolean isInState(State... state) {
1445 return Arrays.asList(state).contains(this.state);
1446 }
1447
1448 private boolean transition(final State target) {
1449 return transition(target, null);
1450 }
1451
1452 private synchronized boolean transition(final State target, final Runnable runnable) {
1453 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1454 if (validTransitions != null && validTransitions.contains(target)) {
1455 this.state = target;
1456 this.stateTransitionException = new StateTransitionException(target);
1457 if (runnable != null) {
1458 runnable.run();
1459 }
1460 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1461 updateEndUserState();
1462 updateOngoingCallNotification();
1463 return true;
1464 } else {
1465 return false;
1466 }
1467 }
1468
1469 void transitionOrThrow(final State target) {
1470 if (!transition(target)) {
1471 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
1472 }
1473 }
1474
1475 @Override
1476 public void onIceCandidate(final IceCandidate iceCandidate) {
1477 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1478 final String ufrag = rtpContentMap.getCredentials().ufrag;
1479 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, ufrag);
1480 if (candidate == null) {
1481 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate.toString());
1482 return;
1483 }
1484 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
1485 sendTransportInfo(iceCandidate.sdpMid, candidate);
1486 }
1487
1488 @Override
1489 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
1490 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
1491 this.stateHistory.add(newState);
1492 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
1493 this.sessionDuration.start();
1494 updateOngoingCallNotification();
1495 } else if (this.sessionDuration.isRunning()) {
1496 this.sessionDuration.stop();
1497 updateOngoingCallNotification();
1498 }
1499
1500 final boolean neverConnected = !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
1501
1502 if (newState == PeerConnection.PeerConnectionState.FAILED) {
1503 if (neverConnected) {
1504 if (isTerminated()) {
1505 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
1506 return;
1507 }
1508 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
1509 return;
1510 } else {
1511 webRTCWrapper.restartIce();
1512 }
1513 }
1514 updateEndUserState();
1515 }
1516
1517 @Override
1518 public void onRenegotiationNeeded() {
1519 this.webRTCWrapper.execute(this::initiateIceRestart);
1520 }
1521
1522 private void initiateIceRestart() {
1523 //TODO discover new TURN/STUN credentials
1524 this.stateHistory.clear();
1525 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
1526 final SessionDescription sessionDescription;
1527 try {
1528 sessionDescription = setLocalSessionDescription();
1529 } catch (final Exception e) {
1530 final Throwable cause = Throwables.getRootCause(e);
1531 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
1532 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
1533 return;
1534 }
1535 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
1536 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
1537 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1538 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
1539 jinglePacket.setTo(id.with);
1540 xmppConnectionService.sendIqPacket(id.account, jinglePacket, (account, response) -> {
1541 if (response.getType() == IqPacket.TYPE.RESULT) {
1542 Log.d(Config.LOGTAG, "received success to our ice restart");
1543 setLocalContentMap(rtpContentMap);
1544 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1545 return;
1546 }
1547 if (response.getType() == IqPacket.TYPE.ERROR) {
1548 final Element error = response.findChild("error");
1549 if (error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS)) {
1550 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
1551 return;
1552 }
1553 handleIqErrorResponse(response);
1554 }
1555 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1556 handleIqTimeoutResponse(response);
1557 }
1558 });
1559 }
1560
1561 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
1562 if (isInitiator()) {
1563 this.initiatorRtpContentMap = rtpContentMap;
1564 } else {
1565 this.responderRtpContentMap = rtpContentMap;
1566 }
1567 }
1568
1569 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
1570 if (isInitiator()) {
1571 this.responderRtpContentMap = rtpContentMap;
1572 } else {
1573 this.initiatorRtpContentMap = rtpContentMap;
1574 }
1575 }
1576
1577 private SessionDescription setLocalSessionDescription() throws ExecutionException, InterruptedException {
1578 final org.webrtc.SessionDescription sessionDescription = this.webRTCWrapper.setLocalDescription().get();
1579 return SessionDescription.parse(sessionDescription.description);
1580 }
1581
1582 private void closeWebRTCSessionAfterFailedConnection() {
1583 this.webRTCWrapper.close();
1584 synchronized (this) {
1585 if (isTerminated()) {
1586 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did");
1587 return;
1588 }
1589 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
1590 }
1591 }
1592
1593 public boolean zeroDuration() {
1594 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
1595 }
1596
1597 public long getCallDuration() {
1598 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
1599 }
1600
1601 public AppRTCAudioManager getAudioManager() {
1602 return webRTCWrapper.getAudioManager();
1603 }
1604
1605 public boolean isMicrophoneEnabled() {
1606 return webRTCWrapper.isMicrophoneEnabled();
1607 }
1608
1609 public boolean setMicrophoneEnabled(final boolean enabled) {
1610 return webRTCWrapper.setMicrophoneEnabled(enabled);
1611 }
1612
1613 public boolean isVideoEnabled() {
1614 return webRTCWrapper.isVideoEnabled();
1615 }
1616
1617 public void setVideoEnabled(final boolean enabled) {
1618 webRTCWrapper.setVideoEnabled(enabled);
1619 }
1620
1621 public boolean isCameraSwitchable() {
1622 return webRTCWrapper.isCameraSwitchable();
1623 }
1624
1625 public boolean isFrontCamera() {
1626 return webRTCWrapper.isFrontCamera();
1627 }
1628
1629 public ListenableFuture<Boolean> switchCamera() {
1630 return webRTCWrapper.switchCamera();
1631 }
1632
1633 @Override
1634 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1635 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
1636 }
1637
1638 private void updateEndUserState() {
1639 final RtpEndUserState endUserState = getEndUserState();
1640 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
1641 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1642 }
1643
1644 private void updateOngoingCallNotification() {
1645 final State state = this.state;
1646 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
1647 final boolean reconnecting;
1648 if (state == State.SESSION_ACCEPTED) {
1649 reconnecting = getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
1650 } else {
1651 reconnecting = false;
1652 }
1653 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
1654 } else {
1655 xmppConnectionService.removeOngoingCall();
1656 }
1657 }
1658
1659 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
1660 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
1661 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
1662 request.setTo(id.account.getDomain());
1663 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1664 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
1665 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
1666 if (response.getType() == IqPacket.TYPE.RESULT) {
1667 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1668 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
1669 for (final Element child : children) {
1670 if ("service".equals(child.getName())) {
1671 final String type = child.getAttribute("type");
1672 final String host = child.getAttribute("host");
1673 final String sport = child.getAttribute("port");
1674 final Integer port = sport == null ? null : Ints.tryParse(sport);
1675 final String transport = child.getAttribute("transport");
1676 final String username = child.getAttribute("username");
1677 final String password = child.getAttribute("password");
1678 if (Strings.isNullOrEmpty(host) || port == null) {
1679 continue;
1680 }
1681 if (port < 0 || port > 65535) {
1682 continue;
1683 }
1684 if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) {
1685 if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) {
1686 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services");
1687 continue;
1688 }
1689 final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer
1690 .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport));
1691 iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK);
1692 if (username != null && password != null) {
1693 iceServerBuilder.setUsername(username);
1694 iceServerBuilder.setPassword(password);
1695 } else if (Arrays.asList("turn", "turns").contains(type)) {
1696 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
1697 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
1698 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
1699 continue;
1700 }
1701 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
1702 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
1703 listBuilder.add(iceServer);
1704 }
1705 }
1706 }
1707 }
1708 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
1709 if (iceServers.size() == 0) {
1710 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
1711 }
1712 onIceServersDiscovered.onIceServersDiscovered(iceServers);
1713 });
1714 } else {
1715 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
1716 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
1717 }
1718 }
1719
1720 private void finish() {
1721 if (isTerminated()) {
1722 this.cancelRingingTimeout();
1723 this.webRTCWrapper.verifyClosed();
1724 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
1725 this.jingleConnectionManager.finishConnectionOrThrow(this);
1726 } else {
1727 throw new IllegalStateException(String.format("Unable to call finish from %s", this.state));
1728 }
1729 }
1730
1731 private void writeLogMessage(final State state) {
1732 final long duration = getCallDuration();
1733 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
1734 writeLogMessageSuccess(duration);
1735 } else {
1736 writeLogMessageMissed();
1737 }
1738 }
1739
1740 private void writeLogMessageSuccess(final long duration) {
1741 this.message.setBody(new RtpSessionStatus(true, duration).toString());
1742 this.writeMessage();
1743 }
1744
1745 private void writeLogMessageMissed() {
1746 this.message.setBody(new RtpSessionStatus(false, 0).toString());
1747 this.writeMessage();
1748 }
1749
1750 private void writeMessage() {
1751 final Conversational conversational = message.getConversation();
1752 if (conversational instanceof Conversation) {
1753 ((Conversation) conversational).add(this.message);
1754 xmppConnectionService.createMessageAsync(message);
1755 xmppConnectionService.updateConversationUi();
1756 } else {
1757 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1758 }
1759 }
1760
1761 public State getState() {
1762 return this.state;
1763 }
1764
1765 boolean isTerminated() {
1766 return TERMINATED.contains(this.state);
1767 }
1768
1769 public Optional<VideoTrack> getLocalVideoTrack() {
1770 return webRTCWrapper.getLocalVideoTrack();
1771 }
1772
1773 public Optional<VideoTrack> getRemoteVideoTrack() {
1774 return webRTCWrapper.getRemoteVideoTrack();
1775 }
1776
1777 public EglBase.Context getEglBaseContext() {
1778 return webRTCWrapper.getEglBaseContext();
1779 }
1780
1781 void setProposedMedia(final Set<Media> media) {
1782 this.proposedMedia = media;
1783 }
1784
1785 public void fireStateUpdate() {
1786 final RtpEndUserState endUserState = getEndUserState();
1787 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1788 }
1789
1790 private interface OnIceServersDiscovered {
1791 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1792 }
1793
1794 private static class StateTransitionException extends Exception {
1795 private final State state;
1796
1797 private StateTransitionException(final State state) {
1798 this.state = state;
1799 }
1800 }
1801}