1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Optional;
9import com.google.common.base.Preconditions;
10import com.google.common.base.Stopwatch;
11import com.google.common.base.Strings;
12import com.google.common.base.Throwables;
13import com.google.common.collect.Collections2;
14import com.google.common.collect.ImmutableList;
15import com.google.common.collect.ImmutableMap;
16import com.google.common.collect.Sets;
17import com.google.common.primitives.Ints;
18import com.google.common.util.concurrent.FutureCallback;
19import com.google.common.util.concurrent.Futures;
20import com.google.common.util.concurrent.ListenableFuture;
21import com.google.common.util.concurrent.MoreExecutors;
22
23import org.webrtc.EglBase;
24import org.webrtc.IceCandidate;
25import org.webrtc.PeerConnection;
26import org.webrtc.VideoTrack;
27
28import java.util.Arrays;
29import java.util.Collection;
30import java.util.Collections;
31import java.util.LinkedList;
32import java.util.List;
33import java.util.Map;
34import java.util.Queue;
35import java.util.Set;
36import java.util.concurrent.ExecutionException;
37import java.util.concurrent.ScheduledFuture;
38import java.util.concurrent.TimeUnit;
39
40import eu.siacs.conversations.Config;
41import eu.siacs.conversations.crypto.axolotl.AxolotlService;
42import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
43import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
44import eu.siacs.conversations.entities.Account;
45import eu.siacs.conversations.entities.Conversation;
46import eu.siacs.conversations.entities.Conversational;
47import eu.siacs.conversations.entities.Message;
48import eu.siacs.conversations.entities.RtpSessionStatus;
49import eu.siacs.conversations.services.AppRTCAudioManager;
50import eu.siacs.conversations.utils.IP;
51import eu.siacs.conversations.xml.Element;
52import eu.siacs.conversations.xml.Namespace;
53import eu.siacs.conversations.xmpp.Jid;
54import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
55import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
56import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
57import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
58import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
59import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
60import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
61import eu.siacs.conversations.xmpp.stanzas.IqPacket;
62import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
63
64public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
65
66 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
67 State.PROCEED,
68 State.SESSION_INITIALIZED_PRE_APPROVED,
69 State.SESSION_ACCEPTED
70 );
71 private static final long BUSY_TIME_OUT = 30;
72 private static final List<State> TERMINATED = Arrays.asList(
73 State.ACCEPTED,
74 State.REJECTED,
75 State.REJECTED_RACED,
76 State.RETRACTED,
77 State.RETRACTED_RACED,
78 State.TERMINATED_SUCCESS,
79 State.TERMINATED_DECLINED_OR_BUSY,
80 State.TERMINATED_CONNECTIVITY_ERROR,
81 State.TERMINATED_CANCEL_OR_TIMEOUT,
82 State.TERMINATED_APPLICATION_FAILURE,
83 State.TERMINATED_SECURITY_ERROR
84 );
85
86 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
87
88 static {
89 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
90 transitionBuilder.put(State.NULL, ImmutableList.of(
91 State.PROPOSED,
92 State.SESSION_INITIALIZED,
93 State.TERMINATED_APPLICATION_FAILURE,
94 State.TERMINATED_SECURITY_ERROR
95 ));
96 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
97 State.ACCEPTED,
98 State.PROCEED,
99 State.REJECTED,
100 State.RETRACTED,
101 State.TERMINATED_APPLICATION_FAILURE,
102 State.TERMINATED_SECURITY_ERROR,
103 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
104 ));
105 transitionBuilder.put(State.PROCEED, ImmutableList.of(
106 State.REJECTED_RACED,
107 State.RETRACTED_RACED,
108 State.SESSION_INITIALIZED_PRE_APPROVED,
109 State.TERMINATED_SUCCESS,
110 State.TERMINATED_APPLICATION_FAILURE,
111 State.TERMINATED_SECURITY_ERROR,
112 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
113 ));
114 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
115 State.SESSION_ACCEPTED,
116 State.TERMINATED_SUCCESS,
117 State.TERMINATED_DECLINED_OR_BUSY,
118 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
119 State.TERMINATED_CANCEL_OR_TIMEOUT,
120 State.TERMINATED_APPLICATION_FAILURE,
121 State.TERMINATED_SECURITY_ERROR
122 ));
123 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
124 State.SESSION_ACCEPTED,
125 State.TERMINATED_SUCCESS,
126 State.TERMINATED_DECLINED_OR_BUSY,
127 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
128 State.TERMINATED_CANCEL_OR_TIMEOUT,
129 State.TERMINATED_APPLICATION_FAILURE,
130 State.TERMINATED_SECURITY_ERROR
131 ));
132 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
133 State.TERMINATED_SUCCESS,
134 State.TERMINATED_DECLINED_OR_BUSY,
135 State.TERMINATED_CONNECTIVITY_ERROR,
136 State.TERMINATED_CANCEL_OR_TIMEOUT,
137 State.TERMINATED_APPLICATION_FAILURE,
138 State.TERMINATED_SECURITY_ERROR
139 ));
140 VALID_TRANSITIONS = transitionBuilder.build();
141 }
142
143 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
144 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>> pendingIceCandidates = new LinkedList<>();
145 private final OmemoVerification omemoVerification = new OmemoVerification();
146 private final Message message;
147 private State state = State.NULL;
148 private StateTransitionException stateTransitionException;
149 private Set<Media> proposedMedia;
150 private RtpContentMap initiatorRtpContentMap;
151 private RtpContentMap responderRtpContentMap;
152 private IceUdpTransportInfo.Setup peerDtlsSetup;
153 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
154 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
155 private ScheduledFuture<?> ringingTimeoutFuture;
156
157 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
158 super(jingleConnectionManager, id, initiator);
159 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
160 id.account,
161 id.with.asBareJid(),
162 false,
163 false
164 );
165 this.message = new Message(
166 conversation,
167 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
168 Message.TYPE_RTP_SESSION,
169 id.sessionId
170 );
171 }
172
173 private static State reasonToState(Reason reason) {
174 switch (reason) {
175 case SUCCESS:
176 return State.TERMINATED_SUCCESS;
177 case DECLINE:
178 case BUSY:
179 return State.TERMINATED_DECLINED_OR_BUSY;
180 case CANCEL:
181 case TIMEOUT:
182 return State.TERMINATED_CANCEL_OR_TIMEOUT;
183 case SECURITY_ERROR:
184 return State.TERMINATED_SECURITY_ERROR;
185 case FAILED_APPLICATION:
186 case UNSUPPORTED_TRANSPORTS:
187 case UNSUPPORTED_APPLICATIONS:
188 return State.TERMINATED_APPLICATION_FAILURE;
189 default:
190 return State.TERMINATED_CONNECTIVITY_ERROR;
191 }
192 }
193
194 @Override
195 synchronized void deliverPacket(final JinglePacket jinglePacket) {
196 switch (jinglePacket.getAction()) {
197 case SESSION_INITIATE:
198 receiveSessionInitiate(jinglePacket);
199 break;
200 case TRANSPORT_INFO:
201 receiveTransportInfo(jinglePacket);
202 break;
203 case SESSION_ACCEPT:
204 receiveSessionAccept(jinglePacket);
205 break;
206 case SESSION_TERMINATE:
207 receiveSessionTerminate(jinglePacket);
208 break;
209 default:
210 respondOk(jinglePacket);
211 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
212 break;
213 }
214 }
215
216 @Override
217 synchronized void notifyRebound() {
218 if (isTerminated()) {
219 return;
220 }
221 webRTCWrapper.close();
222 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
223 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
224 }
225 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
226 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
227 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
228 } else {
229 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
230 finish();
231 }
232 }
233
234 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
235 respondOk(jinglePacket);
236 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
237 final State previous = this.state;
238 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
239 if (TERMINATED.contains(previous)) {
240 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
241 return;
242 }
243 webRTCWrapper.close();
244 final State target = reasonToState(wrapper.reason);
245 transitionOrThrow(target);
246 writeLogMessage(target);
247 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
248 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
249 }
250 finish();
251 }
252
253 private void receiveTransportInfo(final JinglePacket jinglePacket) {
254 //Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to INITIALIZED only after transport-info has been received
255 if (isInState(State.NULL, State.PROCEED, State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
256 final RtpContentMap contentMap;
257 try {
258 contentMap = RtpContentMap.of(jinglePacket);
259 } catch (final IllegalArgumentException | NullPointerException e) {
260 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
261 respondOk(jinglePacket);
262 return;
263 }
264 receiveTransportInfo(jinglePacket, contentMap);
265 } else {
266 if (isTerminated()) {
267 respondOk(jinglePacket);
268 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated");
269 } else {
270 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
271 terminateWithOutOfOrder(jinglePacket);
272 }
273 }
274 }
275
276 private void receiveTransportInfo(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
277 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
278 if (this.state == State.SESSION_ACCEPTED) {
279 //zero candidates + modified credentials are an ICE restart offer
280 if (checkForIceRestart(jinglePacket, contentMap)) {
281 return;
282 }
283 respondOk(jinglePacket);
284 try {
285 processCandidates(candidates);
286 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
287 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
288 }
289 } else {
290 respondOk(jinglePacket);
291 pendingIceCandidates.addAll(candidates);
292 }
293 }
294
295 private boolean checkForIceRestart(final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
296 final RtpContentMap existing = getRemoteContentMap();
297 final IceUdpTransportInfo.Credentials existingCredentials;
298 final IceUdpTransportInfo.Credentials newCredentials;
299 try {
300 existingCredentials = existing.getCredentials();
301 newCredentials = rtpContentMap.getCredentials();
302 } catch (final IllegalStateException e) {
303 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
304 return false;
305 }
306 if (existingCredentials.equals(newCredentials)) {
307 return false;
308 }
309 //TODO an alternative approach is to check if we already got an iq result to our ICE-restart
310 // and if that's the case we are seeing an answer.
311 // This might be more spec compliant but also more error prone potentially
312 final boolean isOffer = rtpContentMap.emptyCandidates();
313 final RtpContentMap restartContentMap;
314 try {
315 if (isOffer) {
316 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
317 restartContentMap = existing.modifiedCredentials(newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
318 } else {
319 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
320 Log.d(Config.LOGTAG, "received confirmation of ICE restart" + newCredentials + " peer_setup=" + setup);
321 // DTLS setup attribute needs to be rewritten to reflect current peer state
322 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
323 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
324 }
325 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
326 return isOffer;
327 } else {
328 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
329 respondWithTieBreak(jinglePacket);
330 return true;
331 }
332 } catch (final Exception exception) {
333 respondOk(jinglePacket);
334 final Throwable rootCause = Throwables.getRootCause(exception);
335 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
336 //If this happens a termination is already in progress
337 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
338 return true;
339 }
340 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
341 webRTCWrapper.close();
342 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
343 return true;
344 }
345 }
346
347 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
348 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
349 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
350 throw new IllegalStateException("Invalid peer setup");
351 }
352 return peerSetup;
353 }
354
355 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
356 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
357 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
358 }
359 this.peerDtlsSetup = setup;
360 }
361
362 private boolean applyIceRestart(final JinglePacket jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer) throws ExecutionException, InterruptedException {
363 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap);
364 final org.webrtc.SessionDescription.Type type = isOffer ? org.webrtc.SessionDescription.Type.OFFER : org.webrtc.SessionDescription.Type.ANSWER;
365 org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(type, sessionDescription.toString());
366 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
367 if (isInitiator()) {
368 //We ignore the offer and respond with tie-break. This will clause the responder not to apply the content map
369 return false;
370 }
371 }
372 webRTCWrapper.setRemoteDescription(sdp).get();
373 setRemoteContentMap(restartContentMap);
374 if (isOffer) {
375 webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
376 final SessionDescription localSessionDescription = setLocalSessionDescription();
377 setLocalContentMap(RtpContentMap.of(localSessionDescription));
378 //We need to respond OK before sending any candidates
379 respondOk(jinglePacket);
380 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
381 } else {
382 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
383 }
384 return true;
385 }
386
387 private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
388 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
389 processCandidate(content);
390 }
391 }
392
393 private void processCandidate(final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
394 final RtpContentMap rtpContentMap = getRemoteContentMap();
395 final List<String> indices = toIdentificationTags(rtpContentMap);
396 final String sdpMid = content.getKey(); //aka content name
397 final IceUdpTransportInfo transport = content.getValue().transport;
398 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
399
400 //TODO check that credentials remained the same
401
402 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
403 final String sdp;
404 try {
405 sdp = candidate.toSdpAttribute(credentials.ufrag);
406 } catch (final IllegalArgumentException e) {
407 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
408 continue;
409 }
410 final int mLineIndex = indices.indexOf(sdpMid);
411 if (mLineIndex < 0) {
412 Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
413 }
414 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
415 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
416 this.webRTCWrapper.addIceCandidate(iceCandidate);
417 }
418 }
419
420 private RtpContentMap getRemoteContentMap() {
421 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
422 }
423
424 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
425 final Group originalGroup = rtpContentMap.group;
426 final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
427 if (identificationTags.size() == 0) {
428 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
429 }
430 return identificationTags;
431 }
432
433 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
434 final RtpContentMap receivedContentMap;
435 try {
436 receivedContentMap = RtpContentMap.of(jinglePacket);
437 } catch (final Exception e) {
438 return Futures.immediateFailedFuture(e);
439 }
440 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
441 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
442 return Futures.transform(future, omemoVerifiedPayload -> {
443 //TODO test if an exception here triggers a correct abort
444 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
445 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received verifiable DTLS fingerprint via " + omemoVerification);
446 return omemoVerifiedPayload.getPayload();
447 }, MoreExecutors.directExecutor());
448 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
449 return Futures.immediateFailedFuture(
450 new SecurityException("DTLS fingerprint was unexpectedly not verifiable")
451 );
452 } else {
453 return Futures.immediateFuture(receivedContentMap);
454 }
455 }
456
457 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
458 if (isInitiator()) {
459 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
460 if (isTerminated()) {
461 Log.d(Config.LOGTAG, String.format(
462 "%s: got a reason to terminate with out-of-order. but already in state %s",
463 id.account.getJid().asBareJid(),
464 getState()
465 ));
466 respondWithOutOfOrder(jinglePacket);
467 } else {
468 terminateWithOutOfOrder(jinglePacket);
469 }
470 return;
471 }
472 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
473 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
474 @Override
475 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
476 receiveSessionInitiate(jinglePacket, rtpContentMap);
477 }
478
479 @Override
480 public void onFailure(@NonNull final Throwable throwable) {
481 respondOk(jinglePacket);
482 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
483 }
484 }, MoreExecutors.directExecutor());
485 }
486
487 private void receiveSessionInitiate(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
488 try {
489 contentMap.requireContentDescriptions();
490 contentMap.requireDTLSFingerprint(true);
491 } catch (final RuntimeException e) {
492 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e));
493 respondOk(jinglePacket);
494 sendSessionTerminate(Reason.of(e), e.getMessage());
495 return;
496 }
497 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
498 final State target;
499 if (this.state == State.PROCEED) {
500 Preconditions.checkState(
501 proposedMedia != null && proposedMedia.size() > 0,
502 "proposed media must be set when processing pre-approved session-initiate"
503 );
504 if (!this.proposedMedia.equals(contentMap.getMedia())) {
505 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
506 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
507 this.proposedMedia,
508 contentMap.getMedia()
509 ));
510 return;
511 }
512 target = State.SESSION_INITIALIZED_PRE_APPROVED;
513 } else {
514 target = State.SESSION_INITIALIZED;
515 }
516 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
517 respondOk(jinglePacket);
518 pendingIceCandidates.addAll(contentMap.contents.entrySet());
519 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
520 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
521 sendSessionAccept();
522 } else {
523 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
524 startRinging();
525 }
526 } else {
527 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
528 terminateWithOutOfOrder(jinglePacket);
529 }
530 }
531
532 private void receiveSessionAccept(final JinglePacket jinglePacket) {
533 if (!isInitiator()) {
534 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
535 terminateWithOutOfOrder(jinglePacket);
536 return;
537 }
538 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
539 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
540 @Override
541 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
542 receiveSessionAccept(jinglePacket, rtpContentMap);
543 }
544
545 @Override
546 public void onFailure(@NonNull final Throwable throwable) {
547 respondOk(jinglePacket);
548 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", throwable);
549 webRTCWrapper.close();
550 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
551 }
552 }, MoreExecutors.directExecutor());
553 }
554
555 private void receiveSessionAccept(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
556 try {
557 contentMap.requireContentDescriptions();
558 contentMap.requireDTLSFingerprint();
559 } catch (final RuntimeException e) {
560 respondOk(jinglePacket);
561 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
562 webRTCWrapper.close();
563 sendSessionTerminate(Reason.of(e), e.getMessage());
564 return;
565 }
566 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
567 if (!initiatorMedia.equals(contentMap.getMedia())) {
568 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
569 "Your session-included included media %s but our session-initiate was %s",
570 this.proposedMedia,
571 contentMap.getMedia()
572 ));
573 return;
574 }
575 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
576 if (transition(State.SESSION_ACCEPTED)) {
577 respondOk(jinglePacket);
578 receiveSessionAccept(contentMap);
579 } else {
580 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
581 respondOk(jinglePacket);
582 }
583 }
584
585 private void receiveSessionAccept(final RtpContentMap contentMap) {
586 this.responderRtpContentMap = contentMap;
587 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
588 final SessionDescription sessionDescription;
589 try {
590 sessionDescription = SessionDescription.of(contentMap);
591 } catch (final IllegalArgumentException | NullPointerException e) {
592 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
593 webRTCWrapper.close();
594 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
595 return;
596 }
597 final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
598 org.webrtc.SessionDescription.Type.ANSWER,
599 sessionDescription.toString()
600 );
601 try {
602 this.webRTCWrapper.setRemoteDescription(answer).get();
603 } catch (final Exception e) {
604 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e));
605 webRTCWrapper.close();
606 sendSessionTerminate(Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
607 return;
608 }
609 processCandidates(contentMap.contents.entrySet());
610 }
611
612 private void sendSessionAccept() {
613 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
614 if (rtpContentMap == null) {
615 throw new IllegalStateException("initiator RTP Content Map has not been set");
616 }
617 final SessionDescription offer;
618 try {
619 offer = SessionDescription.of(rtpContentMap);
620 } catch (final IllegalArgumentException | NullPointerException e) {
621 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
622 webRTCWrapper.close();
623 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
624 return;
625 }
626 sendSessionAccept(rtpContentMap.getMedia(), offer);
627 }
628
629 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
630 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
631 }
632
633 private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
634 if (isTerminated()) {
635 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
636 return;
637 }
638 try {
639 setupWebRTC(media, iceServers);
640 } catch (final WebRTCWrapper.InitializationException e) {
641 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
642 webRTCWrapper.close();
643 sendSessionTerminate(Reason.FAILED_APPLICATION);
644 return;
645 }
646 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
647 org.webrtc.SessionDescription.Type.OFFER,
648 offer.toString()
649 );
650 try {
651 this.webRTCWrapper.setRemoteDescription(sdp).get();
652 addIceCandidatesFromBlackLog();
653 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
654 prepareSessionAccept(webRTCSessionDescription);
655 } catch (final Exception e) {
656 failureToAcceptSession(e);
657 }
658 }
659
660 private void failureToAcceptSession(final Throwable throwable) {
661 if (isTerminated()) {
662 return;
663 }
664 final Throwable rootCause = Throwables.getRootCause(throwable);
665 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
666 webRTCWrapper.close();
667 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
668 }
669
670 private void addIceCandidatesFromBlackLog() {
671 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
672 while ((foo = this.pendingIceCandidates.poll()) != null) {
673 processCandidate(foo);
674 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidate from back log");
675 }
676 }
677
678 private void prepareSessionAccept(final org.webrtc.SessionDescription webRTCSessionDescription) {
679 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
680 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
681 this.responderRtpContentMap = respondingRtpContentMap;
682 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
683 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
684 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = prepareOutgoingContentMap(respondingRtpContentMap);
685 Futures.addCallback(outgoingContentMapFuture,
686 new FutureCallback<RtpContentMap>() {
687 @Override
688 public void onSuccess(final RtpContentMap outgoingContentMap) {
689 sendSessionAccept(outgoingContentMap);
690 }
691
692 @Override
693 public void onFailure(@NonNull Throwable throwable) {
694 failureToAcceptSession(throwable);
695 }
696 },
697 MoreExecutors.directExecutor()
698 );
699 }
700
701 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
702 if (isTerminated()) {
703 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session accept was too slow. already terminated. nothing to do.");
704 return;
705 }
706 transitionOrThrow(State.SESSION_ACCEPTED);
707 final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
708 send(sessionAccept);
709 }
710
711 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(final RtpContentMap rtpContentMap) {
712 if (this.omemoVerification.hasDeviceId()) {
713 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
714 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
715 return Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
716 omemoVerification.setOrEnsureEqual(verifiedPayload);
717 return verifiedPayload.getPayload();
718 }, MoreExecutors.directExecutor());
719 } else {
720 return Futures.immediateFuture(rtpContentMap);
721 }
722 }
723
724 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
725 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
726 switch (message.getName()) {
727 case "propose":
728 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
729 break;
730 case "proceed":
731 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
732 break;
733 case "retract":
734 receiveRetract(from, serverMessageId, timestamp);
735 break;
736 case "reject":
737 receiveReject(from, serverMessageId, timestamp);
738 break;
739 case "accept":
740 receiveAccept(from, serverMessageId, timestamp);
741 break;
742 default:
743 break;
744 }
745 }
746
747 void deliverFailedProceed() {
748 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
749 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
750 webRTCWrapper.close();
751 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
752 this.finish();
753 }
754 }
755
756 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
757 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
758 if (originatedFromMyself) {
759 if (transition(State.ACCEPTED)) {
760 if (serverMsgId != null) {
761 this.message.setServerMsgId(serverMsgId);
762 }
763 this.message.setTime(timestamp);
764 this.message.setCarbon(true); //indicate that call was accepted on other device
765 this.writeLogMessageSuccess(0);
766 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
767 this.finish();
768 } else {
769 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
770 }
771 } else {
772 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
773 }
774 }
775
776 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
777 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
778 //reject from another one of my clients
779 if (originatedFromMyself) {
780 receiveRejectFromMyself(serverMsgId, timestamp);
781 } else if (isInitiator()) {
782 if (from.equals(id.with)) {
783 receiveRejectFromResponder();
784 } else {
785 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
786 }
787 } else {
788 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
789 }
790 }
791
792 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
793 if (transition(State.REJECTED)) {
794 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
795 this.finish();
796 if (serverMsgId != null) {
797 this.message.setServerMsgId(serverMsgId);
798 }
799 this.message.setTime(timestamp);
800 this.message.setCarbon(true); //indicate that call was rejected on other device
801 writeLogMessageMissed();
802 } else {
803 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
804 }
805 }
806
807 private void receiveRejectFromResponder() {
808 if (isInState(State.PROCEED)) {
809 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered");
810 closeTransitionLogFinish(State.REJECTED_RACED);
811 return;
812 }
813 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
814 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
815 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
816 return;
817 }
818 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state);
819 }
820
821 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
822 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
823 if (originatedFromMyself) {
824 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
825 } else if (transition(State.PROPOSED, () -> {
826 final Collection<RtpDescription> descriptions = Collections2.transform(
827 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
828 input -> (RtpDescription) input
829 );
830 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
831 Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
832 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
833 this.proposedMedia = Sets.newHashSet(media);
834 })) {
835 if (serverMsgId != null) {
836 this.message.setServerMsgId(serverMsgId);
837 }
838 this.message.setTime(timestamp);
839 startRinging();
840 } else {
841 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
842 }
843 }
844
845 private void startRinging() {
846 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
847 ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
848 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
849 }
850
851 private synchronized void ringingTimeout() {
852 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
853 switch (this.state) {
854 case PROPOSED:
855 message.markUnread();
856 rejectCallFromProposed();
857 break;
858 case SESSION_INITIALIZED:
859 message.markUnread();
860 rejectCallFromSessionInitiate();
861 break;
862 }
863 }
864
865 private void cancelRingingTimeout() {
866 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
867 if (future != null && !future.isCancelled()) {
868 future.cancel(false);
869 }
870 }
871
872 private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
873 final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
874 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
875 if (from.equals(id.with)) {
876 if (isInitiator()) {
877 if (transition(State.PROCEED)) {
878 if (serverMsgId != null) {
879 this.message.setServerMsgId(serverMsgId);
880 }
881 this.message.setTime(timestamp);
882 final Integer remoteDeviceId = proceed.getDeviceId();
883 if (isOmemoEnabled()) {
884 this.omemoVerification.setDeviceId(remoteDeviceId);
885 } else {
886 if (remoteDeviceId != null) {
887 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
888 }
889 this.omemoVerification.setDeviceId(null);
890 }
891 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
892 } else {
893 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
894 }
895 } else {
896 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
897 }
898 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
899 if (transition(State.ACCEPTED)) {
900 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
901 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
902 this.finish();
903 }
904 } else {
905 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
906 }
907 }
908
909 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
910 if (from.equals(id.with)) {
911 final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
912 if (transition(target)) {
913 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
914 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
915 if (serverMsgId != null) {
916 this.message.setServerMsgId(serverMsgId);
917 }
918 this.message.setTime(timestamp);
919 if (target == State.RETRACTED) {
920 this.message.markUnread();
921 }
922 writeLogMessageMissed();
923 finish();
924 } else {
925 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
926 }
927 } else {
928 //TODO parse retract from self
929 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
930 }
931 }
932
933 public void sendSessionInitiate() {
934 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
935 }
936
937 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
938 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
939 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
940 }
941
942 private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
943 if (isTerminated()) {
944 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
945 return;
946 }
947 try {
948 setupWebRTC(media, iceServers);
949 } catch (final WebRTCWrapper.InitializationException e) {
950 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
951 webRTCWrapper.close();
952 sendRetract(Reason.ofThrowable(e));
953 return;
954 }
955 try {
956 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
957 prepareSessionInitiate(webRTCSessionDescription, targetState);
958 } catch (final Exception e) {
959 //TODO sending the error text is worthwhile as well. Especially for FailureToSet exceptions
960 failureToInitiateSession(e, targetState);
961 }
962 }
963
964 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
965 if (isTerminated()) {
966 return;
967 }
968 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(throwable));
969 webRTCWrapper.close();
970 final Reason reason = Reason.ofThrowable(throwable);
971 if (isInState(targetState)) {
972 sendSessionTerminate(reason);
973 } else {
974 sendRetract(reason);
975 }
976 }
977
978 private void sendRetract(final Reason reason) {
979 //TODO embed reason into retract
980 sendJingleMessage("retract", id.with.asBareJid());
981 transitionOrThrow(reasonToState(reason));
982 this.finish();
983 }
984
985 private void prepareSessionInitiate(final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
986 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
987 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
988 this.initiatorRtpContentMap = rtpContentMap;
989 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
990 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = encryptSessionInitiate(rtpContentMap);
991 Futures.addCallback(outgoingContentMapFuture, new FutureCallback<RtpContentMap>() {
992 @Override
993 public void onSuccess(final RtpContentMap outgoingContentMap) {
994 sendSessionInitiate(outgoingContentMap, targetState);
995 }
996
997 @Override
998 public void onFailure(@NonNull final Throwable throwable) {
999 failureToInitiateSession(throwable, targetState);
1000 }
1001 }, MoreExecutors.directExecutor());
1002 }
1003
1004 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1005 if (isTerminated()) {
1006 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session was too slow. already terminated. nothing to do.");
1007 return;
1008 }
1009 this.transitionOrThrow(targetState);
1010 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1011 send(sessionInitiate);
1012 }
1013
1014 private ListenableFuture<RtpContentMap> encryptSessionInitiate(final RtpContentMap rtpContentMap) {
1015 if (this.omemoVerification.hasDeviceId()) {
1016 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
1017 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
1018 final ListenableFuture<RtpContentMap> future = Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
1019 omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint());
1020 return verifiedPayload.getPayload();
1021 }, MoreExecutors.directExecutor());
1022 if (Config.REQUIRE_RTP_VERIFICATION) {
1023 return future;
1024 }
1025 return Futures.catching(
1026 future,
1027 CryptoFailedException.class,
1028 e -> {
1029 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e);
1030 return rtpContentMap;
1031 },
1032 MoreExecutors.directExecutor()
1033 );
1034 } else {
1035 return Futures.immediateFuture(rtpContentMap);
1036 }
1037 }
1038
1039 private void sendSessionTerminate(final Reason reason) {
1040 sendSessionTerminate(reason, null);
1041 }
1042
1043 private void sendSessionTerminate(final Reason reason, final String text) {
1044 final State previous = this.state;
1045 final State target = reasonToState(reason);
1046 transitionOrThrow(target);
1047 if (previous != State.NULL) {
1048 writeLogMessage(target);
1049 }
1050 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1051 jinglePacket.setReason(reason, text);
1052 Log.d(Config.LOGTAG, jinglePacket.toString());
1053 send(jinglePacket);
1054 finish();
1055 }
1056
1057 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
1058 final RtpContentMap transportInfo;
1059 try {
1060 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1061 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1062 } catch (final Exception e) {
1063 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
1064 return;
1065 }
1066 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1067 send(jinglePacket);
1068 }
1069
1070 private void send(final JinglePacket jinglePacket) {
1071 jinglePacket.setTo(id.with);
1072 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1073 }
1074
1075 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1076 if (response.getType() == IqPacket.TYPE.ERROR) {
1077 handleIqErrorResponse(response);
1078 return;
1079 }
1080 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1081 handleIqTimeoutResponse(response);
1082 }
1083 }
1084
1085 private void handleIqErrorResponse(final IqPacket response) {
1086 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1087 final String errorCondition = response.getErrorCondition();
1088 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
1089 if (isTerminated()) {
1090 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
1091 return;
1092 }
1093 this.webRTCWrapper.close();
1094 final State target;
1095 if (Arrays.asList(
1096 "service-unavailable",
1097 "recipient-unavailable",
1098 "remote-server-not-found",
1099 "remote-server-timeout"
1100 ).contains(errorCondition)) {
1101 target = State.TERMINATED_CONNECTIVITY_ERROR;
1102 } else {
1103 target = State.TERMINATED_APPLICATION_FAILURE;
1104 }
1105 transitionOrThrow(target);
1106 this.finish();
1107 }
1108
1109 private void handleIqTimeoutResponse(final IqPacket response) {
1110 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1111 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
1112 if (isTerminated()) {
1113 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
1114 return;
1115 }
1116 this.webRTCWrapper.close();
1117 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
1118 this.finish();
1119 }
1120
1121 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1122 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1123 this.webRTCWrapper.close();
1124 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1125 respondWithOutOfOrder(jinglePacket);
1126 this.finish();
1127 }
1128
1129 private void respondWithTieBreak(final JinglePacket jinglePacket) {
1130 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
1131 }
1132
1133 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1134 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
1135 }
1136
1137 void respondWithJingleError(final IqPacket original, String jingleCondition, String condition, String conditionType) {
1138 jingleConnectionManager.respondWithJingleError(id.account, original, jingleCondition, condition, conditionType);
1139 }
1140
1141 private void respondOk(final JinglePacket jinglePacket) {
1142 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1143 }
1144
1145 public void throwStateTransitionException() {
1146 final StateTransitionException exception = this.stateTransitionException;
1147 if (exception != null) {
1148 throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception);
1149 }
1150 }
1151
1152 public RtpEndUserState getEndUserState() {
1153 switch (this.state) {
1154 case NULL:
1155 case PROPOSED:
1156 case SESSION_INITIALIZED:
1157 if (isInitiator()) {
1158 return RtpEndUserState.RINGING;
1159 } else {
1160 return RtpEndUserState.INCOMING_CALL;
1161 }
1162 case PROCEED:
1163 if (isInitiator()) {
1164 return RtpEndUserState.RINGING;
1165 } else {
1166 return RtpEndUserState.ACCEPTING_CALL;
1167 }
1168 case SESSION_INITIALIZED_PRE_APPROVED:
1169 if (isInitiator()) {
1170 return RtpEndUserState.RINGING;
1171 } else {
1172 return RtpEndUserState.CONNECTING;
1173 }
1174 case SESSION_ACCEPTED:
1175 return getPeerConnectionStateAsEndUserState();
1176 case REJECTED:
1177 case REJECTED_RACED:
1178 case TERMINATED_DECLINED_OR_BUSY:
1179 if (isInitiator()) {
1180 return RtpEndUserState.DECLINED_OR_BUSY;
1181 } else {
1182 return RtpEndUserState.ENDED;
1183 }
1184 case TERMINATED_SUCCESS:
1185 case ACCEPTED:
1186 case RETRACTED:
1187 case TERMINATED_CANCEL_OR_TIMEOUT:
1188 return RtpEndUserState.ENDED;
1189 case RETRACTED_RACED:
1190 if (isInitiator()) {
1191 return RtpEndUserState.ENDED;
1192 } else {
1193 return RtpEndUserState.RETRACTED;
1194 }
1195 case TERMINATED_CONNECTIVITY_ERROR:
1196 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1197 case TERMINATED_APPLICATION_FAILURE:
1198 return RtpEndUserState.APPLICATION_ERROR;
1199 case TERMINATED_SECURITY_ERROR:
1200 return RtpEndUserState.SECURITY_ERROR;
1201 }
1202 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
1203 }
1204
1205
1206 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
1207 final PeerConnection.PeerConnectionState state;
1208 try {
1209 state = webRTCWrapper.getState();
1210 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
1211 //We usually close the WebRTCWrapper *before* transitioning so we might still
1212 //be in SESSION_ACCEPTED even though the peerConnection has been torn down
1213 return RtpEndUserState.ENDING_CALL;
1214 }
1215 switch (state) {
1216 case CONNECTED:
1217 return RtpEndUserState.CONNECTED;
1218 case NEW:
1219 case CONNECTING:
1220 return RtpEndUserState.CONNECTING;
1221 case CLOSED:
1222 return RtpEndUserState.ENDING_CALL;
1223 default:
1224 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.RECONNECTING;
1225 }
1226 }
1227
1228 public Set<Media> getMedia() {
1229 final State current = getState();
1230 if (current == State.NULL) {
1231 if (isInitiator()) {
1232 return Preconditions.checkNotNull(
1233 this.proposedMedia,
1234 "RTP connection has not been initialized properly"
1235 );
1236 }
1237 throw new IllegalStateException("RTP connection has not been initialized yet");
1238 }
1239 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
1240 return Preconditions.checkNotNull(
1241 this.proposedMedia,
1242 "RTP connection has not been initialized properly"
1243 );
1244 }
1245 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
1246 if (initiatorContentMap != null) {
1247 return initiatorContentMap.getMedia();
1248 } else if (isTerminated()) {
1249 return Collections.emptySet(); //we might fail before we ever got a chance to set media
1250 } else {
1251 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
1252 }
1253 }
1254
1255
1256 public boolean isVerified() {
1257 final String fingerprint = this.omemoVerification.getFingerprint();
1258 if (fingerprint == null) {
1259 return false;
1260 }
1261 final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint);
1262 return status != null && status.isVerified();
1263 }
1264
1265 public synchronized void acceptCall() {
1266 switch (this.state) {
1267 case PROPOSED:
1268 cancelRingingTimeout();
1269 acceptCallFromProposed();
1270 break;
1271 case SESSION_INITIALIZED:
1272 cancelRingingTimeout();
1273 acceptCallFromSessionInitialized();
1274 break;
1275 case ACCEPTED:
1276 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind");
1277 break;
1278 case PROCEED:
1279 case SESSION_ACCEPTED:
1280 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI");
1281 break;
1282 default:
1283 throw new IllegalStateException("Can not accept call from " + this.state);
1284 }
1285 }
1286
1287
1288 public void notifyPhoneCall() {
1289 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1290 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1291 rejectCall();
1292 } else {
1293 endCall();
1294 }
1295 }
1296
1297 public synchronized void rejectCall() {
1298 if (isTerminated()) {
1299 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do");
1300 return;
1301 }
1302 switch (this.state) {
1303 case PROPOSED:
1304 rejectCallFromProposed();
1305 break;
1306 case SESSION_INITIALIZED:
1307 rejectCallFromSessionInitiate();
1308 break;
1309 default:
1310 throw new IllegalStateException("Can not reject call from " + this.state);
1311 }
1312 }
1313
1314 public synchronized void endCall() {
1315 if (isTerminated()) {
1316 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
1317 return;
1318 }
1319 if (isInState(State.PROPOSED) && !isInitiator()) {
1320 rejectCallFromProposed();
1321 return;
1322 }
1323 if (isInState(State.PROCEED)) {
1324 if (isInitiator()) {
1325 retractFromProceed();
1326 } else {
1327 rejectCallFromProceed();
1328 }
1329 return;
1330 }
1331 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1332 this.webRTCWrapper.close();
1333 sendSessionTerminate(Reason.CANCEL);
1334 return;
1335 }
1336 if (isInState(State.SESSION_INITIALIZED)) {
1337 rejectCallFromSessionInitiate();
1338 return;
1339 }
1340 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1341 this.webRTCWrapper.close();
1342 sendSessionTerminate(Reason.SUCCESS);
1343 return;
1344 }
1345 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
1346 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
1347 return;
1348 }
1349 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1350 }
1351
1352 private void retractFromProceed() {
1353 Log.d(Config.LOGTAG, "retract from proceed");
1354 this.sendJingleMessage("retract");
1355 closeTransitionLogFinish(State.RETRACTED_RACED);
1356 }
1357
1358 private void closeTransitionLogFinish(final State state) {
1359 this.webRTCWrapper.close();
1360 transitionOrThrow(state);
1361 writeLogMessage(state);
1362 finish();
1363 }
1364
1365 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
1366 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1367 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1368 if (media.contains(Media.VIDEO)) {
1369 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1370 } else {
1371 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1372 }
1373 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1374 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1375 }
1376
1377 private void acceptCallFromProposed() {
1378 transitionOrThrow(State.PROCEED);
1379 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1380 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1381 this.sendJingleMessage("proceed");
1382 }
1383
1384 private void rejectCallFromProposed() {
1385 transitionOrThrow(State.REJECTED);
1386 writeLogMessageMissed();
1387 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1388 this.sendJingleMessage("reject");
1389 finish();
1390 }
1391
1392 private void rejectCallFromProceed() {
1393 this.sendJingleMessage("reject");
1394 closeTransitionLogFinish(State.REJECTED_RACED);
1395 }
1396
1397 private void rejectCallFromSessionInitiate() {
1398 webRTCWrapper.close();
1399 sendSessionTerminate(Reason.DECLINE);
1400 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1401 }
1402
1403 private void sendJingleMessage(final String action) {
1404 sendJingleMessage(action, id.with);
1405 }
1406
1407 private void sendJingleMessage(final String action, final Jid to) {
1408 final MessagePacket messagePacket = new MessagePacket();
1409 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
1410 messagePacket.setTo(to);
1411 final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
1412 if ("proceed".equals(action)) {
1413 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1414 if (isOmemoEnabled()) {
1415 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1416 final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1417 device.setAttribute("id", deviceId);
1418 }
1419 }
1420 messagePacket.addChild("store", "urn:xmpp:hints");
1421 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1422 }
1423
1424 private boolean isOmemoEnabled() {
1425 final Conversational conversational = message.getConversation();
1426 if (conversational instanceof Conversation) {
1427 return ((Conversation) conversational).getNextEncryption() == Message.ENCRYPTION_AXOLOTL;
1428 }
1429 return false;
1430 }
1431
1432 private void acceptCallFromSessionInitialized() {
1433 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1434 sendSessionAccept();
1435 }
1436
1437 private synchronized boolean isInState(State... state) {
1438 return Arrays.asList(state).contains(this.state);
1439 }
1440
1441 private boolean transition(final State target) {
1442 return transition(target, null);
1443 }
1444
1445 private synchronized boolean transition(final State target, final Runnable runnable) {
1446 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1447 if (validTransitions != null && validTransitions.contains(target)) {
1448 this.state = target;
1449 this.stateTransitionException = new StateTransitionException(target);
1450 if (runnable != null) {
1451 runnable.run();
1452 }
1453 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1454 updateEndUserState();
1455 updateOngoingCallNotification();
1456 return true;
1457 } else {
1458 return false;
1459 }
1460 }
1461
1462 void transitionOrThrow(final State target) {
1463 if (!transition(target)) {
1464 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
1465 }
1466 }
1467
1468 @Override
1469 public void onIceCandidate(final IceCandidate iceCandidate) {
1470 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1471 final String ufrag = rtpContentMap.getCredentials().ufrag;
1472 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, ufrag);
1473 if (candidate == null) {
1474 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate.toString());
1475 return;
1476 }
1477 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
1478 sendTransportInfo(iceCandidate.sdpMid, candidate);
1479 }
1480
1481 @Override
1482 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
1483 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
1484 this.stateHistory.add(newState);
1485 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
1486 this.sessionDuration.start();
1487 updateOngoingCallNotification();
1488 } else if (this.sessionDuration.isRunning()) {
1489 this.sessionDuration.stop();
1490 updateOngoingCallNotification();
1491 }
1492
1493 final boolean neverConnected = !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
1494
1495 if (newState == PeerConnection.PeerConnectionState.FAILED) {
1496 if (neverConnected) {
1497 if (isTerminated()) {
1498 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
1499 return;
1500 }
1501 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
1502 return;
1503 } else {
1504 webRTCWrapper.restartIce();
1505 }
1506 }
1507 updateEndUserState();
1508 }
1509
1510 @Override
1511 public void onRenegotiationNeeded() {
1512 this.webRTCWrapper.execute(this::initiateIceRestart);
1513 }
1514
1515 private void initiateIceRestart() {
1516 //TODO discover new TURN/STUN credentials
1517 this.stateHistory.clear();
1518 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
1519 final SessionDescription sessionDescription;
1520 try {
1521 sessionDescription = setLocalSessionDescription();
1522 } catch (final Exception e) {
1523 final Throwable cause = Throwables.getRootCause(e);
1524 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
1525 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
1526 return;
1527 }
1528 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
1529 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
1530 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1531 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
1532 jinglePacket.setTo(id.with);
1533 xmppConnectionService.sendIqPacket(id.account, jinglePacket, (account, response) -> {
1534 if (response.getType() == IqPacket.TYPE.RESULT) {
1535 Log.d(Config.LOGTAG, "received success to our ice restart");
1536 setLocalContentMap(rtpContentMap);
1537 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1538 return;
1539 }
1540 if (response.getType() == IqPacket.TYPE.ERROR) {
1541 final Element error = response.findChild("error");
1542 if (error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS)) {
1543 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
1544 return;
1545 }
1546 handleIqErrorResponse(response);
1547 }
1548 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1549 handleIqTimeoutResponse(response);
1550 }
1551 });
1552 }
1553
1554 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
1555 if (isInitiator()) {
1556 this.initiatorRtpContentMap = rtpContentMap;
1557 } else {
1558 this.responderRtpContentMap = rtpContentMap;
1559 }
1560 }
1561
1562 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
1563 if (isInitiator()) {
1564 this.responderRtpContentMap = rtpContentMap;
1565 } else {
1566 this.initiatorRtpContentMap = rtpContentMap;
1567 }
1568 }
1569
1570 private SessionDescription setLocalSessionDescription() throws ExecutionException, InterruptedException {
1571 final org.webrtc.SessionDescription sessionDescription = this.webRTCWrapper.setLocalDescription().get();
1572 return SessionDescription.parse(sessionDescription.description);
1573 }
1574
1575 private void closeWebRTCSessionAfterFailedConnection() {
1576 this.webRTCWrapper.close();
1577 synchronized (this) {
1578 if (isTerminated()) {
1579 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did");
1580 return;
1581 }
1582 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
1583 }
1584 }
1585
1586 public boolean zeroDuration() {
1587 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
1588 }
1589
1590 public long getCallDuration() {
1591 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
1592 }
1593
1594 public AppRTCAudioManager getAudioManager() {
1595 return webRTCWrapper.getAudioManager();
1596 }
1597
1598 public boolean isMicrophoneEnabled() {
1599 return webRTCWrapper.isMicrophoneEnabled();
1600 }
1601
1602 public boolean setMicrophoneEnabled(final boolean enabled) {
1603 return webRTCWrapper.setMicrophoneEnabled(enabled);
1604 }
1605
1606 public boolean isVideoEnabled() {
1607 return webRTCWrapper.isVideoEnabled();
1608 }
1609
1610 public void setVideoEnabled(final boolean enabled) {
1611 webRTCWrapper.setVideoEnabled(enabled);
1612 }
1613
1614 public boolean isCameraSwitchable() {
1615 return webRTCWrapper.isCameraSwitchable();
1616 }
1617
1618 public boolean isFrontCamera() {
1619 return webRTCWrapper.isFrontCamera();
1620 }
1621
1622 public ListenableFuture<Boolean> switchCamera() {
1623 return webRTCWrapper.switchCamera();
1624 }
1625
1626 @Override
1627 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1628 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
1629 }
1630
1631 private void updateEndUserState() {
1632 final RtpEndUserState endUserState = getEndUserState();
1633 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
1634 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1635 }
1636
1637 private void updateOngoingCallNotification() {
1638 final State state = this.state;
1639 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
1640 final boolean reconnecting;
1641 if (state == State.SESSION_ACCEPTED) {
1642 reconnecting = getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
1643 } else {
1644 reconnecting = false;
1645 }
1646 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
1647 } else {
1648 xmppConnectionService.removeOngoingCall();
1649 }
1650 }
1651
1652 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
1653 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
1654 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
1655 request.setTo(id.account.getDomain());
1656 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1657 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
1658 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
1659 if (response.getType() == IqPacket.TYPE.RESULT) {
1660 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1661 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
1662 for (final Element child : children) {
1663 if ("service".equals(child.getName())) {
1664 final String type = child.getAttribute("type");
1665 final String host = child.getAttribute("host");
1666 final String sport = child.getAttribute("port");
1667 final Integer port = sport == null ? null : Ints.tryParse(sport);
1668 final String transport = child.getAttribute("transport");
1669 final String username = child.getAttribute("username");
1670 final String password = child.getAttribute("password");
1671 if (Strings.isNullOrEmpty(host) || port == null) {
1672 continue;
1673 }
1674 if (port < 0 || port > 65535) {
1675 continue;
1676 }
1677 if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) {
1678 if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) {
1679 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services");
1680 continue;
1681 }
1682 final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer
1683 .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport));
1684 iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK);
1685 if (username != null && password != null) {
1686 iceServerBuilder.setUsername(username);
1687 iceServerBuilder.setPassword(password);
1688 } else if (Arrays.asList("turn", "turns").contains(type)) {
1689 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
1690 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
1691 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
1692 continue;
1693 }
1694 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
1695 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
1696 listBuilder.add(iceServer);
1697 }
1698 }
1699 }
1700 }
1701 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
1702 if (iceServers.size() == 0) {
1703 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
1704 }
1705 onIceServersDiscovered.onIceServersDiscovered(iceServers);
1706 });
1707 } else {
1708 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
1709 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
1710 }
1711 }
1712
1713 private void finish() {
1714 if (isTerminated()) {
1715 this.cancelRingingTimeout();
1716 this.webRTCWrapper.verifyClosed();
1717 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
1718 this.jingleConnectionManager.finishConnectionOrThrow(this);
1719 } else {
1720 throw new IllegalStateException(String.format("Unable to call finish from %s", this.state));
1721 }
1722 }
1723
1724 private void writeLogMessage(final State state) {
1725 final long duration = getCallDuration();
1726 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
1727 writeLogMessageSuccess(duration);
1728 } else {
1729 writeLogMessageMissed();
1730 }
1731 }
1732
1733 private void writeLogMessageSuccess(final long duration) {
1734 this.message.setBody(new RtpSessionStatus(true, duration).toString());
1735 this.writeMessage();
1736 }
1737
1738 private void writeLogMessageMissed() {
1739 this.message.setBody(new RtpSessionStatus(false, 0).toString());
1740 this.writeMessage();
1741 }
1742
1743 private void writeMessage() {
1744 final Conversational conversational = message.getConversation();
1745 if (conversational instanceof Conversation) {
1746 ((Conversation) conversational).add(this.message);
1747 xmppConnectionService.createMessageAsync(message);
1748 xmppConnectionService.updateConversationUi();
1749 } else {
1750 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1751 }
1752 }
1753
1754 public State getState() {
1755 return this.state;
1756 }
1757
1758 boolean isTerminated() {
1759 return TERMINATED.contains(this.state);
1760 }
1761
1762 public Optional<VideoTrack> getLocalVideoTrack() {
1763 return webRTCWrapper.getLocalVideoTrack();
1764 }
1765
1766 public Optional<VideoTrack> getRemoteVideoTrack() {
1767 return webRTCWrapper.getRemoteVideoTrack();
1768 }
1769
1770 public EglBase.Context getEglBaseContext() {
1771 return webRTCWrapper.getEglBaseContext();
1772 }
1773
1774 void setProposedMedia(final Set<Media> media) {
1775 this.proposedMedia = media;
1776 }
1777
1778 public void fireStateUpdate() {
1779 final RtpEndUserState endUserState = getEndUserState();
1780 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1781 }
1782
1783 private interface OnIceServersDiscovered {
1784 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1785 }
1786
1787 private static class StateTransitionException extends Exception {
1788 private final State state;
1789
1790 private StateTransitionException(final State state) {
1791 this.state = state;
1792 }
1793 }
1794}