1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Optional;
9import com.google.common.base.Preconditions;
10import com.google.common.base.Stopwatch;
11import com.google.common.base.Strings;
12import com.google.common.base.Throwables;
13import com.google.common.collect.Collections2;
14import com.google.common.collect.ImmutableList;
15import com.google.common.collect.ImmutableMap;
16import com.google.common.collect.Sets;
17import com.google.common.primitives.Ints;
18import com.google.common.util.concurrent.FutureCallback;
19import com.google.common.util.concurrent.Futures;
20import com.google.common.util.concurrent.ListenableFuture;
21import com.google.common.util.concurrent.MoreExecutors;
22
23import org.webrtc.DtmfSender;
24import org.webrtc.EglBase;
25import org.webrtc.IceCandidate;
26import org.webrtc.PeerConnection;
27import org.webrtc.VideoTrack;
28
29import java.util.Arrays;
30import java.util.Collection;
31import java.util.Collections;
32import java.util.LinkedList;
33import java.util.List;
34import java.util.Map;
35import java.util.Queue;
36import java.util.Set;
37import java.util.concurrent.ExecutionException;
38import java.util.concurrent.ScheduledFuture;
39import java.util.concurrent.TimeUnit;
40
41import eu.siacs.conversations.Config;
42import eu.siacs.conversations.crypto.axolotl.AxolotlService;
43import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
44import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
45import eu.siacs.conversations.entities.Account;
46import eu.siacs.conversations.entities.Conversation;
47import eu.siacs.conversations.entities.Conversational;
48import eu.siacs.conversations.entities.Message;
49import eu.siacs.conversations.entities.RtpSessionStatus;
50import eu.siacs.conversations.services.AppRTCAudioManager;
51import eu.siacs.conversations.utils.IP;
52import eu.siacs.conversations.xml.Element;
53import eu.siacs.conversations.xml.Namespace;
54import eu.siacs.conversations.xmpp.Jid;
55import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
56import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
57import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
58import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
59import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
60import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
61import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
62import eu.siacs.conversations.xmpp.stanzas.IqPacket;
63import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
64
65public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
66
67 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
68 State.PROCEED,
69 State.SESSION_INITIALIZED_PRE_APPROVED,
70 State.SESSION_ACCEPTED
71 );
72 private static final long BUSY_TIME_OUT = 30;
73 private static final List<State> TERMINATED = Arrays.asList(
74 State.ACCEPTED,
75 State.REJECTED,
76 State.REJECTED_RACED,
77 State.RETRACTED,
78 State.RETRACTED_RACED,
79 State.TERMINATED_SUCCESS,
80 State.TERMINATED_DECLINED_OR_BUSY,
81 State.TERMINATED_CONNECTIVITY_ERROR,
82 State.TERMINATED_CANCEL_OR_TIMEOUT,
83 State.TERMINATED_APPLICATION_FAILURE,
84 State.TERMINATED_SECURITY_ERROR
85 );
86
87 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
88
89 static {
90 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
91 transitionBuilder.put(State.NULL, ImmutableList.of(
92 State.PROPOSED,
93 State.SESSION_INITIALIZED,
94 State.TERMINATED_APPLICATION_FAILURE,
95 State.TERMINATED_SECURITY_ERROR
96 ));
97 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
98 State.ACCEPTED,
99 State.PROCEED,
100 State.REJECTED,
101 State.RETRACTED,
102 State.TERMINATED_APPLICATION_FAILURE,
103 State.TERMINATED_SECURITY_ERROR,
104 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
105 ));
106 transitionBuilder.put(State.PROCEED, ImmutableList.of(
107 State.REJECTED_RACED,
108 State.RETRACTED_RACED,
109 State.SESSION_INITIALIZED_PRE_APPROVED,
110 State.TERMINATED_SUCCESS,
111 State.TERMINATED_APPLICATION_FAILURE,
112 State.TERMINATED_SECURITY_ERROR,
113 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
114 ));
115 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
116 State.SESSION_ACCEPTED,
117 State.TERMINATED_SUCCESS,
118 State.TERMINATED_DECLINED_OR_BUSY,
119 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
120 State.TERMINATED_CANCEL_OR_TIMEOUT,
121 State.TERMINATED_APPLICATION_FAILURE,
122 State.TERMINATED_SECURITY_ERROR
123 ));
124 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
125 State.SESSION_ACCEPTED,
126 State.TERMINATED_SUCCESS,
127 State.TERMINATED_DECLINED_OR_BUSY,
128 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
129 State.TERMINATED_CANCEL_OR_TIMEOUT,
130 State.TERMINATED_APPLICATION_FAILURE,
131 State.TERMINATED_SECURITY_ERROR
132 ));
133 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
134 State.TERMINATED_SUCCESS,
135 State.TERMINATED_DECLINED_OR_BUSY,
136 State.TERMINATED_CONNECTIVITY_ERROR,
137 State.TERMINATED_CANCEL_OR_TIMEOUT,
138 State.TERMINATED_APPLICATION_FAILURE,
139 State.TERMINATED_SECURITY_ERROR
140 ));
141 VALID_TRANSITIONS = transitionBuilder.build();
142 }
143
144 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
145 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>> pendingIceCandidates = new LinkedList<>();
146 private final OmemoVerification omemoVerification = new OmemoVerification();
147 private final Message message;
148 private State state = State.NULL;
149 private StateTransitionException stateTransitionException;
150 private Set<Media> proposedMedia;
151 private RtpContentMap initiatorRtpContentMap;
152 private RtpContentMap responderRtpContentMap;
153 private IceUdpTransportInfo.Setup peerDtlsSetup;
154 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
155 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
156 private ScheduledFuture<?> ringingTimeoutFuture;
157
158 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
159 super(jingleConnectionManager, id, initiator);
160 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
161 id.account,
162 id.with.asBareJid(),
163 false,
164 false
165 );
166 this.message = new Message(
167 conversation,
168 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
169 Message.TYPE_RTP_SESSION,
170 id.sessionId
171 );
172 }
173
174 private static State reasonToState(Reason reason) {
175 switch (reason) {
176 case SUCCESS:
177 return State.TERMINATED_SUCCESS;
178 case DECLINE:
179 case BUSY:
180 return State.TERMINATED_DECLINED_OR_BUSY;
181 case CANCEL:
182 case TIMEOUT:
183 return State.TERMINATED_CANCEL_OR_TIMEOUT;
184 case SECURITY_ERROR:
185 return State.TERMINATED_SECURITY_ERROR;
186 case FAILED_APPLICATION:
187 case UNSUPPORTED_TRANSPORTS:
188 case UNSUPPORTED_APPLICATIONS:
189 return State.TERMINATED_APPLICATION_FAILURE;
190 default:
191 return State.TERMINATED_CONNECTIVITY_ERROR;
192 }
193 }
194
195 @Override
196 synchronized void deliverPacket(final JinglePacket jinglePacket) {
197 switch (jinglePacket.getAction()) {
198 case SESSION_INITIATE:
199 receiveSessionInitiate(jinglePacket);
200 break;
201 case TRANSPORT_INFO:
202 receiveTransportInfo(jinglePacket);
203 break;
204 case SESSION_ACCEPT:
205 receiveSessionAccept(jinglePacket);
206 break;
207 case SESSION_TERMINATE:
208 receiveSessionTerminate(jinglePacket);
209 break;
210 default:
211 respondOk(jinglePacket);
212 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
213 break;
214 }
215 }
216
217 @Override
218 synchronized void notifyRebound() {
219 if (isTerminated()) {
220 return;
221 }
222 webRTCWrapper.close();
223 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
224 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
225 }
226 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
227 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
228 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
229 } else {
230 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
231 finish();
232 }
233 }
234
235 public boolean applyDtmfTone(String tone) {
236 return webRTCWrapper.applyDtmfTone(tone);
237 }
238
239 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
240 respondOk(jinglePacket);
241 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
242 final State previous = this.state;
243 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
244 if (TERMINATED.contains(previous)) {
245 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
246 return;
247 }
248 webRTCWrapper.close();
249 final State target = reasonToState(wrapper.reason);
250 transitionOrThrow(target);
251 writeLogMessage(target);
252 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
253 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
254 }
255 finish();
256 }
257
258 private void receiveTransportInfo(final JinglePacket jinglePacket) {
259 //Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to INITIALIZED only after transport-info has been received
260 if (isInState(State.NULL, State.PROCEED, State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
261 final RtpContentMap contentMap;
262 try {
263 contentMap = RtpContentMap.of(jinglePacket);
264 } catch (final IllegalArgumentException | NullPointerException e) {
265 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
266 respondOk(jinglePacket);
267 return;
268 }
269 receiveTransportInfo(jinglePacket, contentMap);
270 } else {
271 if (isTerminated()) {
272 respondOk(jinglePacket);
273 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated");
274 } else {
275 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
276 terminateWithOutOfOrder(jinglePacket);
277 }
278 }
279 }
280
281 private void receiveTransportInfo(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
282 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
283 if (this.state == State.SESSION_ACCEPTED) {
284 //zero candidates + modified credentials are an ICE restart offer
285 if (checkForIceRestart(jinglePacket, contentMap)) {
286 return;
287 }
288 respondOk(jinglePacket);
289 try {
290 processCandidates(candidates);
291 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
292 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
293 }
294 } else {
295 respondOk(jinglePacket);
296 pendingIceCandidates.addAll(candidates);
297 }
298 }
299
300 private boolean checkForIceRestart(final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
301 final RtpContentMap existing = getRemoteContentMap();
302 final IceUdpTransportInfo.Credentials existingCredentials;
303 final IceUdpTransportInfo.Credentials newCredentials;
304 try {
305 existingCredentials = existing.getCredentials();
306 newCredentials = rtpContentMap.getCredentials();
307 } catch (final IllegalStateException e) {
308 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
309 return false;
310 }
311 if (existingCredentials.equals(newCredentials)) {
312 return false;
313 }
314 //TODO an alternative approach is to check if we already got an iq result to our ICE-restart
315 // and if that's the case we are seeing an answer.
316 // This might be more spec compliant but also more error prone potentially
317 final boolean isOffer = rtpContentMap.emptyCandidates();
318 final RtpContentMap restartContentMap;
319 try {
320 if (isOffer) {
321 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
322 restartContentMap = existing.modifiedCredentials(newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
323 } else {
324 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
325 Log.d(Config.LOGTAG, "received confirmation of ICE restart" + newCredentials + " peer_setup=" + setup);
326 // DTLS setup attribute needs to be rewritten to reflect current peer state
327 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
328 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
329 }
330 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
331 return isOffer;
332 } else {
333 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
334 respondWithTieBreak(jinglePacket);
335 return true;
336 }
337 } catch (final Exception exception) {
338 respondOk(jinglePacket);
339 final Throwable rootCause = Throwables.getRootCause(exception);
340 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
341 //If this happens a termination is already in progress
342 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
343 return true;
344 }
345 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
346 webRTCWrapper.close();
347 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
348 return true;
349 }
350 }
351
352 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
353 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
354 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
355 throw new IllegalStateException("Invalid peer setup");
356 }
357 return peerSetup;
358 }
359
360 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
361 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
362 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
363 }
364 this.peerDtlsSetup = setup;
365 }
366
367 private boolean applyIceRestart(final JinglePacket jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer) throws ExecutionException, InterruptedException {
368 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap);
369 final org.webrtc.SessionDescription.Type type = isOffer ? org.webrtc.SessionDescription.Type.OFFER : org.webrtc.SessionDescription.Type.ANSWER;
370 org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(type, sessionDescription.toString());
371 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
372 if (isInitiator()) {
373 //We ignore the offer and respond with tie-break. This will clause the responder not to apply the content map
374 return false;
375 }
376 }
377 webRTCWrapper.setRemoteDescription(sdp).get();
378 setRemoteContentMap(restartContentMap);
379 if (isOffer) {
380 webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
381 final SessionDescription localSessionDescription = setLocalSessionDescription();
382 setLocalContentMap(RtpContentMap.of(localSessionDescription));
383 //We need to respond OK before sending any candidates
384 respondOk(jinglePacket);
385 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
386 } else {
387 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
388 }
389 return true;
390 }
391
392 private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
393 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
394 processCandidate(content);
395 }
396 }
397
398 private void processCandidate(final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
399 final RtpContentMap rtpContentMap = getRemoteContentMap();
400 final List<String> indices = toIdentificationTags(rtpContentMap);
401 final String sdpMid = content.getKey(); //aka content name
402 final IceUdpTransportInfo transport = content.getValue().transport;
403 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
404
405 //TODO check that credentials remained the same
406
407 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
408 final String sdp;
409 try {
410 sdp = candidate.toSdpAttribute(credentials.ufrag);
411 } catch (final IllegalArgumentException e) {
412 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
413 continue;
414 }
415 final int mLineIndex = indices.indexOf(sdpMid);
416 if (mLineIndex < 0) {
417 Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
418 }
419 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
420 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
421 this.webRTCWrapper.addIceCandidate(iceCandidate);
422 }
423 }
424
425 private RtpContentMap getRemoteContentMap() {
426 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
427 }
428
429 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
430 final Group originalGroup = rtpContentMap.group;
431 final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
432 if (identificationTags.size() == 0) {
433 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
434 }
435 return identificationTags;
436 }
437
438 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
439 final RtpContentMap receivedContentMap;
440 try {
441 receivedContentMap = RtpContentMap.of(jinglePacket);
442 } catch (final Exception e) {
443 return Futures.immediateFailedFuture(e);
444 }
445 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
446 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
447 return Futures.transform(future, omemoVerifiedPayload -> {
448 //TODO test if an exception here triggers a correct abort
449 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
450 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received verifiable DTLS fingerprint via " + omemoVerification);
451 return omemoVerifiedPayload.getPayload();
452 }, MoreExecutors.directExecutor());
453 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
454 return Futures.immediateFailedFuture(
455 new SecurityException("DTLS fingerprint was unexpectedly not verifiable")
456 );
457 } else {
458 return Futures.immediateFuture(receivedContentMap);
459 }
460 }
461
462 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
463 if (isInitiator()) {
464 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
465 if (isTerminated()) {
466 Log.d(Config.LOGTAG, String.format(
467 "%s: got a reason to terminate with out-of-order. but already in state %s",
468 id.account.getJid().asBareJid(),
469 getState()
470 ));
471 respondWithOutOfOrder(jinglePacket);
472 } else {
473 terminateWithOutOfOrder(jinglePacket);
474 }
475 return;
476 }
477 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
478 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
479 @Override
480 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
481 receiveSessionInitiate(jinglePacket, rtpContentMap);
482 }
483
484 @Override
485 public void onFailure(@NonNull final Throwable throwable) {
486 respondOk(jinglePacket);
487 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
488 }
489 }, MoreExecutors.directExecutor());
490 }
491
492 private void receiveSessionInitiate(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
493 try {
494 contentMap.requireContentDescriptions();
495 contentMap.requireDTLSFingerprint(true);
496 } catch (final RuntimeException e) {
497 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e));
498 respondOk(jinglePacket);
499 sendSessionTerminate(Reason.of(e), e.getMessage());
500 return;
501 }
502 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
503 final State target;
504 if (this.state == State.PROCEED) {
505 Preconditions.checkState(
506 proposedMedia != null && proposedMedia.size() > 0,
507 "proposed media must be set when processing pre-approved session-initiate"
508 );
509 if (!this.proposedMedia.equals(contentMap.getMedia())) {
510 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
511 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
512 this.proposedMedia,
513 contentMap.getMedia()
514 ));
515 return;
516 }
517 target = State.SESSION_INITIALIZED_PRE_APPROVED;
518 } else {
519 target = State.SESSION_INITIALIZED;
520 }
521 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
522 respondOk(jinglePacket);
523 pendingIceCandidates.addAll(contentMap.contents.entrySet());
524 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
525 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
526 sendSessionAccept();
527 } else {
528 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
529 startRinging();
530 }
531 } else {
532 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
533 terminateWithOutOfOrder(jinglePacket);
534 }
535 }
536
537 private void receiveSessionAccept(final JinglePacket jinglePacket) {
538 if (!isInitiator()) {
539 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
540 terminateWithOutOfOrder(jinglePacket);
541 return;
542 }
543 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
544 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
545 @Override
546 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
547 receiveSessionAccept(jinglePacket, rtpContentMap);
548 }
549
550 @Override
551 public void onFailure(@NonNull final Throwable throwable) {
552 respondOk(jinglePacket);
553 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", throwable);
554 webRTCWrapper.close();
555 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
556 }
557 }, MoreExecutors.directExecutor());
558 }
559
560 private void receiveSessionAccept(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
561 try {
562 contentMap.requireContentDescriptions();
563 contentMap.requireDTLSFingerprint();
564 } catch (final RuntimeException e) {
565 respondOk(jinglePacket);
566 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
567 webRTCWrapper.close();
568 sendSessionTerminate(Reason.of(e), e.getMessage());
569 return;
570 }
571 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
572 if (!initiatorMedia.equals(contentMap.getMedia())) {
573 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
574 "Your session-included included media %s but our session-initiate was %s",
575 this.proposedMedia,
576 contentMap.getMedia()
577 ));
578 return;
579 }
580 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
581 if (transition(State.SESSION_ACCEPTED)) {
582 respondOk(jinglePacket);
583 receiveSessionAccept(contentMap);
584 } else {
585 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
586 respondOk(jinglePacket);
587 }
588 }
589
590 private void receiveSessionAccept(final RtpContentMap contentMap) {
591 this.responderRtpContentMap = contentMap;
592 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
593 final SessionDescription sessionDescription;
594 try {
595 sessionDescription = SessionDescription.of(contentMap);
596 } catch (final IllegalArgumentException | NullPointerException e) {
597 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
598 webRTCWrapper.close();
599 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
600 return;
601 }
602 final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
603 org.webrtc.SessionDescription.Type.ANSWER,
604 sessionDescription.toString()
605 );
606 try {
607 this.webRTCWrapper.setRemoteDescription(answer).get();
608 } catch (final Exception e) {
609 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e));
610 webRTCWrapper.close();
611 sendSessionTerminate(Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
612 return;
613 }
614 processCandidates(contentMap.contents.entrySet());
615 }
616
617 private void sendSessionAccept() {
618 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
619 if (rtpContentMap == null) {
620 throw new IllegalStateException("initiator RTP Content Map has not been set");
621 }
622 final SessionDescription offer;
623 try {
624 offer = SessionDescription.of(rtpContentMap);
625 } catch (final IllegalArgumentException | NullPointerException e) {
626 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
627 webRTCWrapper.close();
628 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
629 return;
630 }
631 sendSessionAccept(rtpContentMap.getMedia(), offer);
632 }
633
634 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
635 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
636 }
637
638 private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
639 if (isTerminated()) {
640 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
641 return;
642 }
643 try {
644 setupWebRTC(media, iceServers);
645 } catch (final WebRTCWrapper.InitializationException e) {
646 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
647 webRTCWrapper.close();
648 sendSessionTerminate(Reason.FAILED_APPLICATION);
649 return;
650 }
651 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
652 org.webrtc.SessionDescription.Type.OFFER,
653 offer.toString()
654 );
655 try {
656 this.webRTCWrapper.setRemoteDescription(sdp).get();
657 addIceCandidatesFromBlackLog();
658 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
659 prepareSessionAccept(webRTCSessionDescription);
660 } catch (final Exception e) {
661 failureToAcceptSession(e);
662 }
663 }
664
665 private void failureToAcceptSession(final Throwable throwable) {
666 if (isTerminated()) {
667 return;
668 }
669 final Throwable rootCause = Throwables.getRootCause(throwable);
670 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
671 webRTCWrapper.close();
672 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
673 }
674
675 private void addIceCandidatesFromBlackLog() {
676 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
677 while ((foo = this.pendingIceCandidates.poll()) != null) {
678 processCandidate(foo);
679 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidate from back log");
680 }
681 }
682
683 private void prepareSessionAccept(final org.webrtc.SessionDescription webRTCSessionDescription) {
684 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
685 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
686 this.responderRtpContentMap = respondingRtpContentMap;
687 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
688 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
689 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = prepareOutgoingContentMap(respondingRtpContentMap);
690 Futures.addCallback(outgoingContentMapFuture,
691 new FutureCallback<RtpContentMap>() {
692 @Override
693 public void onSuccess(final RtpContentMap outgoingContentMap) {
694 sendSessionAccept(outgoingContentMap);
695 }
696
697 @Override
698 public void onFailure(@NonNull Throwable throwable) {
699 failureToAcceptSession(throwable);
700 }
701 },
702 MoreExecutors.directExecutor()
703 );
704 }
705
706 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
707 if (isTerminated()) {
708 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session accept was too slow. already terminated. nothing to do.");
709 return;
710 }
711 transitionOrThrow(State.SESSION_ACCEPTED);
712 final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
713 send(sessionAccept);
714 }
715
716 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(final RtpContentMap rtpContentMap) {
717 if (this.omemoVerification.hasDeviceId()) {
718 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
719 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
720 return Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
721 omemoVerification.setOrEnsureEqual(verifiedPayload);
722 return verifiedPayload.getPayload();
723 }, MoreExecutors.directExecutor());
724 } else {
725 return Futures.immediateFuture(rtpContentMap);
726 }
727 }
728
729 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
730 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
731 switch (message.getName()) {
732 case "propose":
733 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
734 break;
735 case "proceed":
736 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
737 break;
738 case "retract":
739 receiveRetract(from, serverMessageId, timestamp);
740 break;
741 case "reject":
742 receiveReject(from, serverMessageId, timestamp);
743 break;
744 case "accept":
745 receiveAccept(from, serverMessageId, timestamp);
746 break;
747 default:
748 break;
749 }
750 }
751
752 void deliverFailedProceed() {
753 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
754 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
755 webRTCWrapper.close();
756 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
757 this.finish();
758 }
759 }
760
761 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
762 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
763 if (originatedFromMyself) {
764 if (transition(State.ACCEPTED)) {
765 if (serverMsgId != null) {
766 this.message.setServerMsgId(serverMsgId);
767 }
768 this.message.setTime(timestamp);
769 this.message.setCarbon(true); //indicate that call was accepted on other device
770 this.writeLogMessageSuccess(0);
771 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
772 this.finish();
773 } else {
774 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
775 }
776 } else {
777 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
778 }
779 }
780
781 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
782 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
783 //reject from another one of my clients
784 if (originatedFromMyself) {
785 receiveRejectFromMyself(serverMsgId, timestamp);
786 } else if (isInitiator()) {
787 if (from.equals(id.with)) {
788 receiveRejectFromResponder();
789 } else {
790 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
791 }
792 } else {
793 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
794 }
795 }
796
797 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
798 if (transition(State.REJECTED)) {
799 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
800 this.finish();
801 if (serverMsgId != null) {
802 this.message.setServerMsgId(serverMsgId);
803 }
804 this.message.setTime(timestamp);
805 this.message.setCarbon(true); //indicate that call was rejected on other device
806 writeLogMessageMissed();
807 } else {
808 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
809 }
810 }
811
812 private void receiveRejectFromResponder() {
813 if (isInState(State.PROCEED)) {
814 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered");
815 closeTransitionLogFinish(State.REJECTED_RACED);
816 return;
817 }
818 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
819 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
820 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
821 return;
822 }
823 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state);
824 }
825
826 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
827 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
828 if (originatedFromMyself) {
829 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
830 } else if (transition(State.PROPOSED, () -> {
831 final Collection<RtpDescription> descriptions = Collections2.transform(
832 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
833 input -> (RtpDescription) input
834 );
835 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
836 Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
837 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
838 this.proposedMedia = Sets.newHashSet(media);
839 })) {
840 if (serverMsgId != null) {
841 this.message.setServerMsgId(serverMsgId);
842 }
843 this.message.setTime(timestamp);
844 startRinging();
845 } else {
846 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
847 }
848 }
849
850 private void startRinging() {
851 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
852 ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
853 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
854 }
855
856 private synchronized void ringingTimeout() {
857 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
858 switch (this.state) {
859 case PROPOSED:
860 message.markUnread();
861 rejectCallFromProposed();
862 break;
863 case SESSION_INITIALIZED:
864 message.markUnread();
865 rejectCallFromSessionInitiate();
866 break;
867 }
868 }
869
870 private void cancelRingingTimeout() {
871 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
872 if (future != null && !future.isCancelled()) {
873 future.cancel(false);
874 }
875 }
876
877 private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
878 final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
879 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
880 if (from.equals(id.with)) {
881 if (isInitiator()) {
882 if (transition(State.PROCEED)) {
883 if (serverMsgId != null) {
884 this.message.setServerMsgId(serverMsgId);
885 }
886 this.message.setTime(timestamp);
887 final Integer remoteDeviceId = proceed.getDeviceId();
888 if (isOmemoEnabled()) {
889 this.omemoVerification.setDeviceId(remoteDeviceId);
890 } else {
891 if (remoteDeviceId != null) {
892 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
893 }
894 this.omemoVerification.setDeviceId(null);
895 }
896 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
897 } else {
898 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
899 }
900 } else {
901 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
902 }
903 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
904 if (transition(State.ACCEPTED)) {
905 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
906 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
907 this.finish();
908 }
909 } else {
910 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
911 }
912 }
913
914 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
915 if (from.equals(id.with)) {
916 final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
917 if (transition(target)) {
918 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
919 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
920 if (serverMsgId != null) {
921 this.message.setServerMsgId(serverMsgId);
922 }
923 this.message.setTime(timestamp);
924 if (target == State.RETRACTED) {
925 this.message.markUnread();
926 }
927 writeLogMessageMissed();
928 finish();
929 } else {
930 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
931 }
932 } else {
933 //TODO parse retract from self
934 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
935 }
936 }
937
938 public void sendSessionInitiate() {
939 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
940 }
941
942 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
943 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
944 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
945 }
946
947 private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
948 if (isTerminated()) {
949 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
950 return;
951 }
952 try {
953 setupWebRTC(media, iceServers);
954 } catch (final WebRTCWrapper.InitializationException e) {
955 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
956 webRTCWrapper.close();
957 sendRetract(Reason.ofThrowable(e));
958 return;
959 }
960 try {
961 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
962 prepareSessionInitiate(webRTCSessionDescription, targetState);
963 } catch (final Exception e) {
964 //TODO sending the error text is worthwhile as well. Especially for FailureToSet exceptions
965 failureToInitiateSession(e, targetState);
966 }
967 }
968
969 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
970 if (isTerminated()) {
971 return;
972 }
973 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(throwable));
974 webRTCWrapper.close();
975 final Reason reason = Reason.ofThrowable(throwable);
976 if (isInState(targetState)) {
977 sendSessionTerminate(reason);
978 } else {
979 sendRetract(reason);
980 }
981 }
982
983 private void sendRetract(final Reason reason) {
984 //TODO embed reason into retract
985 sendJingleMessage("retract", id.with.asBareJid());
986 transitionOrThrow(reasonToState(reason));
987 this.finish();
988 }
989
990 private void prepareSessionInitiate(final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
991 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
992 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
993 this.initiatorRtpContentMap = rtpContentMap;
994 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
995 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = encryptSessionInitiate(rtpContentMap);
996 Futures.addCallback(outgoingContentMapFuture, new FutureCallback<RtpContentMap>() {
997 @Override
998 public void onSuccess(final RtpContentMap outgoingContentMap) {
999 sendSessionInitiate(outgoingContentMap, targetState);
1000 }
1001
1002 @Override
1003 public void onFailure(@NonNull final Throwable throwable) {
1004 failureToInitiateSession(throwable, targetState);
1005 }
1006 }, MoreExecutors.directExecutor());
1007 }
1008
1009 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1010 if (isTerminated()) {
1011 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session was too slow. already terminated. nothing to do.");
1012 return;
1013 }
1014 this.transitionOrThrow(targetState);
1015 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1016 send(sessionInitiate);
1017 }
1018
1019 private ListenableFuture<RtpContentMap> encryptSessionInitiate(final RtpContentMap rtpContentMap) {
1020 if (this.omemoVerification.hasDeviceId()) {
1021 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
1022 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
1023 final ListenableFuture<RtpContentMap> future = Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
1024 omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint());
1025 return verifiedPayload.getPayload();
1026 }, MoreExecutors.directExecutor());
1027 if (Config.REQUIRE_RTP_VERIFICATION) {
1028 return future;
1029 }
1030 return Futures.catching(
1031 future,
1032 CryptoFailedException.class,
1033 e -> {
1034 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e);
1035 return rtpContentMap;
1036 },
1037 MoreExecutors.directExecutor()
1038 );
1039 } else {
1040 return Futures.immediateFuture(rtpContentMap);
1041 }
1042 }
1043
1044 private void sendSessionTerminate(final Reason reason) {
1045 sendSessionTerminate(reason, null);
1046 }
1047
1048 private void sendSessionTerminate(final Reason reason, final String text) {
1049 final State previous = this.state;
1050 final State target = reasonToState(reason);
1051 transitionOrThrow(target);
1052 if (previous != State.NULL) {
1053 writeLogMessage(target);
1054 }
1055 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1056 jinglePacket.setReason(reason, text);
1057 Log.d(Config.LOGTAG, jinglePacket.toString());
1058 send(jinglePacket);
1059 finish();
1060 }
1061
1062 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
1063 final RtpContentMap transportInfo;
1064 try {
1065 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1066 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1067 } catch (final Exception e) {
1068 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
1069 return;
1070 }
1071 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1072 send(jinglePacket);
1073 }
1074
1075 private void send(final JinglePacket jinglePacket) {
1076 jinglePacket.setTo(id.with);
1077 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1078 }
1079
1080 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1081 if (response.getType() == IqPacket.TYPE.ERROR) {
1082 handleIqErrorResponse(response);
1083 return;
1084 }
1085 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1086 handleIqTimeoutResponse(response);
1087 }
1088 }
1089
1090 private void handleIqErrorResponse(final IqPacket response) {
1091 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1092 final String errorCondition = response.getErrorCondition();
1093 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
1094 if (isTerminated()) {
1095 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
1096 return;
1097 }
1098 this.webRTCWrapper.close();
1099 final State target;
1100 if (Arrays.asList(
1101 "service-unavailable",
1102 "recipient-unavailable",
1103 "remote-server-not-found",
1104 "remote-server-timeout"
1105 ).contains(errorCondition)) {
1106 target = State.TERMINATED_CONNECTIVITY_ERROR;
1107 } else {
1108 target = State.TERMINATED_APPLICATION_FAILURE;
1109 }
1110 transitionOrThrow(target);
1111 this.finish();
1112 }
1113
1114 private void handleIqTimeoutResponse(final IqPacket response) {
1115 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1116 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
1117 if (isTerminated()) {
1118 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
1119 return;
1120 }
1121 this.webRTCWrapper.close();
1122 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
1123 this.finish();
1124 }
1125
1126 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1127 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1128 this.webRTCWrapper.close();
1129 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1130 respondWithOutOfOrder(jinglePacket);
1131 this.finish();
1132 }
1133
1134 private void respondWithTieBreak(final JinglePacket jinglePacket) {
1135 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
1136 }
1137
1138 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1139 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
1140 }
1141
1142 void respondWithJingleError(final IqPacket original, String jingleCondition, String condition, String conditionType) {
1143 jingleConnectionManager.respondWithJingleError(id.account, original, jingleCondition, condition, conditionType);
1144 }
1145
1146 private void respondOk(final JinglePacket jinglePacket) {
1147 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1148 }
1149
1150 public void throwStateTransitionException() {
1151 final StateTransitionException exception = this.stateTransitionException;
1152 if (exception != null) {
1153 throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception);
1154 }
1155 }
1156
1157 public RtpEndUserState getEndUserState() {
1158 switch (this.state) {
1159 case NULL:
1160 case PROPOSED:
1161 case SESSION_INITIALIZED:
1162 if (isInitiator()) {
1163 return RtpEndUserState.RINGING;
1164 } else {
1165 return RtpEndUserState.INCOMING_CALL;
1166 }
1167 case PROCEED:
1168 if (isInitiator()) {
1169 return RtpEndUserState.RINGING;
1170 } else {
1171 return RtpEndUserState.ACCEPTING_CALL;
1172 }
1173 case SESSION_INITIALIZED_PRE_APPROVED:
1174 if (isInitiator()) {
1175 return RtpEndUserState.RINGING;
1176 } else {
1177 return RtpEndUserState.CONNECTING;
1178 }
1179 case SESSION_ACCEPTED:
1180 return getPeerConnectionStateAsEndUserState();
1181 case REJECTED:
1182 case REJECTED_RACED:
1183 case TERMINATED_DECLINED_OR_BUSY:
1184 if (isInitiator()) {
1185 return RtpEndUserState.DECLINED_OR_BUSY;
1186 } else {
1187 return RtpEndUserState.ENDED;
1188 }
1189 case TERMINATED_SUCCESS:
1190 case ACCEPTED:
1191 case RETRACTED:
1192 case TERMINATED_CANCEL_OR_TIMEOUT:
1193 return RtpEndUserState.ENDED;
1194 case RETRACTED_RACED:
1195 if (isInitiator()) {
1196 return RtpEndUserState.ENDED;
1197 } else {
1198 return RtpEndUserState.RETRACTED;
1199 }
1200 case TERMINATED_CONNECTIVITY_ERROR:
1201 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1202 case TERMINATED_APPLICATION_FAILURE:
1203 return RtpEndUserState.APPLICATION_ERROR;
1204 case TERMINATED_SECURITY_ERROR:
1205 return RtpEndUserState.SECURITY_ERROR;
1206 }
1207 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
1208 }
1209
1210
1211 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
1212 final PeerConnection.PeerConnectionState state;
1213 try {
1214 state = webRTCWrapper.getState();
1215 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
1216 //We usually close the WebRTCWrapper *before* transitioning so we might still
1217 //be in SESSION_ACCEPTED even though the peerConnection has been torn down
1218 return RtpEndUserState.ENDING_CALL;
1219 }
1220 switch (state) {
1221 case CONNECTED:
1222 return RtpEndUserState.CONNECTED;
1223 case NEW:
1224 case CONNECTING:
1225 return RtpEndUserState.CONNECTING;
1226 case CLOSED:
1227 return RtpEndUserState.ENDING_CALL;
1228 default:
1229 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.RECONNECTING;
1230 }
1231 }
1232
1233 public Set<Media> getMedia() {
1234 final State current = getState();
1235 if (current == State.NULL) {
1236 if (isInitiator()) {
1237 return Preconditions.checkNotNull(
1238 this.proposedMedia,
1239 "RTP connection has not been initialized properly"
1240 );
1241 }
1242 throw new IllegalStateException("RTP connection has not been initialized yet");
1243 }
1244 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
1245 return Preconditions.checkNotNull(
1246 this.proposedMedia,
1247 "RTP connection has not been initialized properly"
1248 );
1249 }
1250 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
1251 if (initiatorContentMap != null) {
1252 return initiatorContentMap.getMedia();
1253 } else if (isTerminated()) {
1254 return Collections.emptySet(); //we might fail before we ever got a chance to set media
1255 } else {
1256 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
1257 }
1258 }
1259
1260
1261 public boolean isVerified() {
1262 final String fingerprint = this.omemoVerification.getFingerprint();
1263 if (fingerprint == null) {
1264 return false;
1265 }
1266 final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint);
1267 return status != null && status.isVerified();
1268 }
1269
1270 public synchronized void acceptCall() {
1271 switch (this.state) {
1272 case PROPOSED:
1273 cancelRingingTimeout();
1274 acceptCallFromProposed();
1275 break;
1276 case SESSION_INITIALIZED:
1277 cancelRingingTimeout();
1278 acceptCallFromSessionInitialized();
1279 break;
1280 case ACCEPTED:
1281 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind");
1282 break;
1283 case PROCEED:
1284 case SESSION_ACCEPTED:
1285 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI");
1286 break;
1287 default:
1288 throw new IllegalStateException("Can not accept call from " + this.state);
1289 }
1290 }
1291
1292
1293 public void notifyPhoneCall() {
1294 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1295 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1296 rejectCall();
1297 } else {
1298 endCall();
1299 }
1300 }
1301
1302 public synchronized void rejectCall() {
1303 if (isTerminated()) {
1304 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do");
1305 return;
1306 }
1307 switch (this.state) {
1308 case PROPOSED:
1309 rejectCallFromProposed();
1310 break;
1311 case SESSION_INITIALIZED:
1312 rejectCallFromSessionInitiate();
1313 break;
1314 default:
1315 throw new IllegalStateException("Can not reject call from " + this.state);
1316 }
1317 }
1318
1319 public synchronized void endCall() {
1320 if (isTerminated()) {
1321 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
1322 return;
1323 }
1324 if (isInState(State.PROPOSED) && !isInitiator()) {
1325 rejectCallFromProposed();
1326 return;
1327 }
1328 if (isInState(State.PROCEED)) {
1329 if (isInitiator()) {
1330 retractFromProceed();
1331 } else {
1332 rejectCallFromProceed();
1333 }
1334 return;
1335 }
1336 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1337 this.webRTCWrapper.close();
1338 sendSessionTerminate(Reason.CANCEL);
1339 return;
1340 }
1341 if (isInState(State.SESSION_INITIALIZED)) {
1342 rejectCallFromSessionInitiate();
1343 return;
1344 }
1345 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1346 this.webRTCWrapper.close();
1347 sendSessionTerminate(Reason.SUCCESS);
1348 return;
1349 }
1350 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
1351 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
1352 return;
1353 }
1354 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1355 }
1356
1357 private void retractFromProceed() {
1358 Log.d(Config.LOGTAG, "retract from proceed");
1359 this.sendJingleMessage("retract");
1360 closeTransitionLogFinish(State.RETRACTED_RACED);
1361 }
1362
1363 private void closeTransitionLogFinish(final State state) {
1364 this.webRTCWrapper.close();
1365 transitionOrThrow(state);
1366 writeLogMessage(state);
1367 finish();
1368 }
1369
1370 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
1371 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1372 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1373 if (media.contains(Media.VIDEO)) {
1374 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1375 } else {
1376 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1377 }
1378 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1379 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1380 }
1381
1382 private void acceptCallFromProposed() {
1383 transitionOrThrow(State.PROCEED);
1384 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1385 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1386 this.sendJingleMessage("proceed");
1387 }
1388
1389 private void rejectCallFromProposed() {
1390 transitionOrThrow(State.REJECTED);
1391 writeLogMessageMissed();
1392 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1393 this.sendJingleMessage("reject");
1394 finish();
1395 }
1396
1397 private void rejectCallFromProceed() {
1398 this.sendJingleMessage("reject");
1399 closeTransitionLogFinish(State.REJECTED_RACED);
1400 }
1401
1402 private void rejectCallFromSessionInitiate() {
1403 webRTCWrapper.close();
1404 sendSessionTerminate(Reason.DECLINE);
1405 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1406 }
1407
1408 private void sendJingleMessage(final String action) {
1409 sendJingleMessage(action, id.with);
1410 }
1411
1412 private void sendJingleMessage(final String action, final Jid to) {
1413 final MessagePacket messagePacket = new MessagePacket();
1414 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
1415 messagePacket.setTo(to);
1416 final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
1417 if ("proceed".equals(action)) {
1418 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1419 if (isOmemoEnabled()) {
1420 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1421 final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1422 device.setAttribute("id", deviceId);
1423 }
1424 }
1425 messagePacket.addChild("store", "urn:xmpp:hints");
1426 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1427 }
1428
1429 private boolean isOmemoEnabled() {
1430 final Conversational conversational = message.getConversation();
1431 if (conversational instanceof Conversation) {
1432 return ((Conversation) conversational).getNextEncryption() == Message.ENCRYPTION_AXOLOTL;
1433 }
1434 return false;
1435 }
1436
1437 private void acceptCallFromSessionInitialized() {
1438 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1439 sendSessionAccept();
1440 }
1441
1442 private synchronized boolean isInState(State... state) {
1443 return Arrays.asList(state).contains(this.state);
1444 }
1445
1446 private boolean transition(final State target) {
1447 return transition(target, null);
1448 }
1449
1450 private synchronized boolean transition(final State target, final Runnable runnable) {
1451 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1452 if (validTransitions != null && validTransitions.contains(target)) {
1453 this.state = target;
1454 this.stateTransitionException = new StateTransitionException(target);
1455 if (runnable != null) {
1456 runnable.run();
1457 }
1458 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1459 updateEndUserState();
1460 updateOngoingCallNotification();
1461 return true;
1462 } else {
1463 return false;
1464 }
1465 }
1466
1467 void transitionOrThrow(final State target) {
1468 if (!transition(target)) {
1469 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
1470 }
1471 }
1472
1473 @Override
1474 public void onIceCandidate(final IceCandidate iceCandidate) {
1475 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1476 final String ufrag = rtpContentMap.getCredentials().ufrag;
1477 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, ufrag);
1478 if (candidate == null) {
1479 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate.toString());
1480 return;
1481 }
1482 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
1483 sendTransportInfo(iceCandidate.sdpMid, candidate);
1484 }
1485
1486 @Override
1487 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
1488 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
1489 this.stateHistory.add(newState);
1490 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
1491 this.sessionDuration.start();
1492 updateOngoingCallNotification();
1493 } else if (this.sessionDuration.isRunning()) {
1494 this.sessionDuration.stop();
1495 updateOngoingCallNotification();
1496 }
1497
1498 final boolean neverConnected = !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
1499
1500 if (newState == PeerConnection.PeerConnectionState.FAILED) {
1501 if (neverConnected) {
1502 if (isTerminated()) {
1503 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
1504 return;
1505 }
1506 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
1507 return;
1508 } else {
1509 webRTCWrapper.restartIce();
1510 }
1511 }
1512 updateEndUserState();
1513 }
1514
1515 @Override
1516 public void onRenegotiationNeeded() {
1517 this.webRTCWrapper.execute(this::initiateIceRestart);
1518 }
1519
1520 private void initiateIceRestart() {
1521 //TODO discover new TURN/STUN credentials
1522 this.stateHistory.clear();
1523 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
1524 final SessionDescription sessionDescription;
1525 try {
1526 sessionDescription = setLocalSessionDescription();
1527 } catch (final Exception e) {
1528 final Throwable cause = Throwables.getRootCause(e);
1529 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
1530 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
1531 return;
1532 }
1533 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
1534 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
1535 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1536 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
1537 jinglePacket.setTo(id.with);
1538 xmppConnectionService.sendIqPacket(id.account, jinglePacket, (account, response) -> {
1539 if (response.getType() == IqPacket.TYPE.RESULT) {
1540 Log.d(Config.LOGTAG, "received success to our ice restart");
1541 setLocalContentMap(rtpContentMap);
1542 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1543 return;
1544 }
1545 if (response.getType() == IqPacket.TYPE.ERROR) {
1546 final Element error = response.findChild("error");
1547 if (error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS)) {
1548 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
1549 return;
1550 }
1551 handleIqErrorResponse(response);
1552 }
1553 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1554 handleIqTimeoutResponse(response);
1555 }
1556 });
1557 }
1558
1559 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
1560 if (isInitiator()) {
1561 this.initiatorRtpContentMap = rtpContentMap;
1562 } else {
1563 this.responderRtpContentMap = rtpContentMap;
1564 }
1565 }
1566
1567 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
1568 if (isInitiator()) {
1569 this.responderRtpContentMap = rtpContentMap;
1570 } else {
1571 this.initiatorRtpContentMap = rtpContentMap;
1572 }
1573 }
1574
1575 private SessionDescription setLocalSessionDescription() throws ExecutionException, InterruptedException {
1576 final org.webrtc.SessionDescription sessionDescription = this.webRTCWrapper.setLocalDescription().get();
1577 return SessionDescription.parse(sessionDescription.description);
1578 }
1579
1580 private void closeWebRTCSessionAfterFailedConnection() {
1581 this.webRTCWrapper.close();
1582 synchronized (this) {
1583 if (isTerminated()) {
1584 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did");
1585 return;
1586 }
1587 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
1588 }
1589 }
1590
1591 public boolean zeroDuration() {
1592 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
1593 }
1594
1595 public long getCallDuration() {
1596 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
1597 }
1598
1599 public AppRTCAudioManager getAudioManager() {
1600 return webRTCWrapper.getAudioManager();
1601 }
1602
1603 public boolean isMicrophoneEnabled() {
1604 return webRTCWrapper.isMicrophoneEnabled();
1605 }
1606
1607 public boolean setMicrophoneEnabled(final boolean enabled) {
1608 return webRTCWrapper.setMicrophoneEnabled(enabled);
1609 }
1610
1611 public boolean isVideoEnabled() {
1612 return webRTCWrapper.isVideoEnabled();
1613 }
1614
1615 public void setVideoEnabled(final boolean enabled) {
1616 webRTCWrapper.setVideoEnabled(enabled);
1617 }
1618
1619 public boolean isCameraSwitchable() {
1620 return webRTCWrapper.isCameraSwitchable();
1621 }
1622
1623 public boolean isFrontCamera() {
1624 return webRTCWrapper.isFrontCamera();
1625 }
1626
1627 public ListenableFuture<Boolean> switchCamera() {
1628 return webRTCWrapper.switchCamera();
1629 }
1630
1631 @Override
1632 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1633 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
1634 }
1635
1636 private void updateEndUserState() {
1637 final RtpEndUserState endUserState = getEndUserState();
1638 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
1639 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1640 }
1641
1642 private void updateOngoingCallNotification() {
1643 final State state = this.state;
1644 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
1645 final boolean reconnecting;
1646 if (state == State.SESSION_ACCEPTED) {
1647 reconnecting = getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
1648 } else {
1649 reconnecting = false;
1650 }
1651 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
1652 } else {
1653 xmppConnectionService.removeOngoingCall();
1654 }
1655 }
1656
1657 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
1658 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
1659 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
1660 request.setTo(id.account.getDomain());
1661 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1662 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
1663 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
1664 if (response.getType() == IqPacket.TYPE.RESULT) {
1665 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1666 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
1667 for (final Element child : children) {
1668 if ("service".equals(child.getName())) {
1669 final String type = child.getAttribute("type");
1670 final String host = child.getAttribute("host");
1671 final String sport = child.getAttribute("port");
1672 final Integer port = sport == null ? null : Ints.tryParse(sport);
1673 final String transport = child.getAttribute("transport");
1674 final String username = child.getAttribute("username");
1675 final String password = child.getAttribute("password");
1676 if (Strings.isNullOrEmpty(host) || port == null) {
1677 continue;
1678 }
1679 if (port < 0 || port > 65535) {
1680 continue;
1681 }
1682 if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) {
1683 if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) {
1684 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services");
1685 continue;
1686 }
1687 final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer
1688 .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport));
1689 iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK);
1690 if (username != null && password != null) {
1691 iceServerBuilder.setUsername(username);
1692 iceServerBuilder.setPassword(password);
1693 } else if (Arrays.asList("turn", "turns").contains(type)) {
1694 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
1695 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
1696 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
1697 continue;
1698 }
1699 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
1700 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
1701 listBuilder.add(iceServer);
1702 }
1703 }
1704 }
1705 }
1706 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
1707 if (iceServers.size() == 0) {
1708 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
1709 }
1710 onIceServersDiscovered.onIceServersDiscovered(iceServers);
1711 });
1712 } else {
1713 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
1714 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
1715 }
1716 }
1717
1718 private void finish() {
1719 if (isTerminated()) {
1720 this.cancelRingingTimeout();
1721 this.webRTCWrapper.verifyClosed();
1722 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
1723 this.jingleConnectionManager.finishConnectionOrThrow(this);
1724 } else {
1725 throw new IllegalStateException(String.format("Unable to call finish from %s", this.state));
1726 }
1727 }
1728
1729 private void writeLogMessage(final State state) {
1730 final long duration = getCallDuration();
1731 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
1732 writeLogMessageSuccess(duration);
1733 } else {
1734 writeLogMessageMissed();
1735 }
1736 }
1737
1738 private void writeLogMessageSuccess(final long duration) {
1739 this.message.setBody(new RtpSessionStatus(true, duration).toString());
1740 this.writeMessage();
1741 }
1742
1743 private void writeLogMessageMissed() {
1744 this.message.setBody(new RtpSessionStatus(false, 0).toString());
1745 this.writeMessage();
1746 }
1747
1748 private void writeMessage() {
1749 final Conversational conversational = message.getConversation();
1750 if (conversational instanceof Conversation) {
1751 ((Conversation) conversational).add(this.message);
1752 xmppConnectionService.createMessageAsync(message);
1753 xmppConnectionService.updateConversationUi();
1754 } else {
1755 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1756 }
1757 }
1758
1759 public State getState() {
1760 return this.state;
1761 }
1762
1763 boolean isTerminated() {
1764 return TERMINATED.contains(this.state);
1765 }
1766
1767 public Optional<VideoTrack> getLocalVideoTrack() {
1768 return webRTCWrapper.getLocalVideoTrack();
1769 }
1770
1771 public Optional<VideoTrack> getRemoteVideoTrack() {
1772 return webRTCWrapper.getRemoteVideoTrack();
1773 }
1774
1775 public EglBase.Context getEglBaseContext() {
1776 return webRTCWrapper.getEglBaseContext();
1777 }
1778
1779 void setProposedMedia(final Set<Media> media) {
1780 this.proposedMedia = media;
1781 }
1782
1783 public void fireStateUpdate() {
1784 final RtpEndUserState endUserState = getEndUserState();
1785 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1786 }
1787
1788 private interface OnIceServersDiscovered {
1789 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1790 }
1791
1792 private static class StateTransitionException extends Exception {
1793 private final State state;
1794
1795 private StateTransitionException(final State state) {
1796 this.state = state;
1797 }
1798 }
1799}