1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Optional;
9import com.google.common.base.Preconditions;
10import com.google.common.base.Stopwatch;
11import com.google.common.base.Strings;
12import com.google.common.base.Throwables;
13import com.google.common.collect.Collections2;
14import com.google.common.collect.ImmutableList;
15import com.google.common.collect.ImmutableMap;
16import com.google.common.collect.Sets;
17import com.google.common.primitives.Ints;
18import com.google.common.util.concurrent.FutureCallback;
19import com.google.common.util.concurrent.Futures;
20import com.google.common.util.concurrent.ListenableFuture;
21import com.google.common.util.concurrent.MoreExecutors;
22
23import org.webrtc.DtmfSender;
24import org.webrtc.EglBase;
25import org.webrtc.IceCandidate;
26import org.webrtc.PeerConnection;
27import org.webrtc.VideoTrack;
28
29import java.util.Arrays;
30import java.util.Collection;
31import java.util.Collections;
32import java.util.LinkedList;
33import java.util.List;
34import java.util.Map;
35import java.util.Queue;
36import java.util.Set;
37import java.util.concurrent.ExecutionException;
38import java.util.concurrent.ScheduledFuture;
39import java.util.concurrent.TimeUnit;
40
41import eu.siacs.conversations.Config;
42import eu.siacs.conversations.crypto.axolotl.AxolotlService;
43import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
44import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
45import eu.siacs.conversations.entities.Account;
46import eu.siacs.conversations.entities.Conversation;
47import eu.siacs.conversations.entities.Conversational;
48import eu.siacs.conversations.entities.Message;
49import eu.siacs.conversations.entities.RtpSessionStatus;
50import eu.siacs.conversations.services.AppRTCAudioManager;
51import eu.siacs.conversations.utils.IP;
52import eu.siacs.conversations.xml.Element;
53import eu.siacs.conversations.xml.Namespace;
54import eu.siacs.conversations.xmpp.Jid;
55import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
56import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
57import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
58import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
59import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
60import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
61import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
62import eu.siacs.conversations.xmpp.stanzas.IqPacket;
63import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
64
65public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
66
67 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
68 State.PROCEED,
69 State.SESSION_INITIALIZED_PRE_APPROVED,
70 State.SESSION_ACCEPTED
71 );
72 private static final long BUSY_TIME_OUT = 30;
73 private static final List<State> TERMINATED = Arrays.asList(
74 State.ACCEPTED,
75 State.REJECTED,
76 State.REJECTED_RACED,
77 State.RETRACTED,
78 State.RETRACTED_RACED,
79 State.TERMINATED_SUCCESS,
80 State.TERMINATED_DECLINED_OR_BUSY,
81 State.TERMINATED_CONNECTIVITY_ERROR,
82 State.TERMINATED_CANCEL_OR_TIMEOUT,
83 State.TERMINATED_APPLICATION_FAILURE,
84 State.TERMINATED_SECURITY_ERROR
85 );
86
87 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
88
89 static {
90 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
91 transitionBuilder.put(State.NULL, ImmutableList.of(
92 State.PROPOSED,
93 State.SESSION_INITIALIZED,
94 State.TERMINATED_APPLICATION_FAILURE,
95 State.TERMINATED_SECURITY_ERROR
96 ));
97 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
98 State.ACCEPTED,
99 State.PROCEED,
100 State.REJECTED,
101 State.RETRACTED,
102 State.TERMINATED_APPLICATION_FAILURE,
103 State.TERMINATED_SECURITY_ERROR,
104 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
105 ));
106 transitionBuilder.put(State.PROCEED, ImmutableList.of(
107 State.REJECTED_RACED,
108 State.RETRACTED_RACED,
109 State.SESSION_INITIALIZED_PRE_APPROVED,
110 State.TERMINATED_SUCCESS,
111 State.TERMINATED_APPLICATION_FAILURE,
112 State.TERMINATED_SECURITY_ERROR,
113 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
114 ));
115 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
116 State.SESSION_ACCEPTED,
117 State.TERMINATED_SUCCESS,
118 State.TERMINATED_DECLINED_OR_BUSY,
119 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
120 State.TERMINATED_CANCEL_OR_TIMEOUT,
121 State.TERMINATED_APPLICATION_FAILURE,
122 State.TERMINATED_SECURITY_ERROR
123 ));
124 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
125 State.SESSION_ACCEPTED,
126 State.TERMINATED_SUCCESS,
127 State.TERMINATED_DECLINED_OR_BUSY,
128 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
129 State.TERMINATED_CANCEL_OR_TIMEOUT,
130 State.TERMINATED_APPLICATION_FAILURE,
131 State.TERMINATED_SECURITY_ERROR
132 ));
133 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
134 State.TERMINATED_SUCCESS,
135 State.TERMINATED_DECLINED_OR_BUSY,
136 State.TERMINATED_CONNECTIVITY_ERROR,
137 State.TERMINATED_CANCEL_OR_TIMEOUT,
138 State.TERMINATED_APPLICATION_FAILURE,
139 State.TERMINATED_SECURITY_ERROR
140 ));
141 VALID_TRANSITIONS = transitionBuilder.build();
142 }
143
144 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
145 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>> pendingIceCandidates = new LinkedList<>();
146 private final OmemoVerification omemoVerification = new OmemoVerification();
147 private final Message message;
148 private State state = State.NULL;
149 private StateTransitionException stateTransitionException;
150 private Set<Media> proposedMedia;
151 private RtpContentMap initiatorRtpContentMap;
152 private RtpContentMap responderRtpContentMap;
153 private IceUdpTransportInfo.Setup peerDtlsSetup;
154 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
155 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
156 private ScheduledFuture<?> ringingTimeoutFuture;
157
158 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
159 super(jingleConnectionManager, id, initiator);
160 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
161 id.account,
162 id.with.asBareJid(),
163 false,
164 false
165 );
166 this.message = new Message(
167 conversation,
168 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
169 Message.TYPE_RTP_SESSION,
170 id.sessionId
171 );
172 }
173
174 private static State reasonToState(Reason reason) {
175 switch (reason) {
176 case SUCCESS:
177 return State.TERMINATED_SUCCESS;
178 case DECLINE:
179 case BUSY:
180 return State.TERMINATED_DECLINED_OR_BUSY;
181 case CANCEL:
182 case TIMEOUT:
183 return State.TERMINATED_CANCEL_OR_TIMEOUT;
184 case SECURITY_ERROR:
185 return State.TERMINATED_SECURITY_ERROR;
186 case FAILED_APPLICATION:
187 case UNSUPPORTED_TRANSPORTS:
188 case UNSUPPORTED_APPLICATIONS:
189 return State.TERMINATED_APPLICATION_FAILURE;
190 default:
191 return State.TERMINATED_CONNECTIVITY_ERROR;
192 }
193 }
194
195 @Override
196 synchronized void deliverPacket(final JinglePacket jinglePacket) {
197 switch (jinglePacket.getAction()) {
198 case SESSION_INITIATE:
199 receiveSessionInitiate(jinglePacket);
200 break;
201 case TRANSPORT_INFO:
202 receiveTransportInfo(jinglePacket);
203 break;
204 case SESSION_ACCEPT:
205 receiveSessionAccept(jinglePacket);
206 break;
207 case SESSION_TERMINATE:
208 receiveSessionTerminate(jinglePacket);
209 break;
210 default:
211 respondOk(jinglePacket);
212 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
213 break;
214 }
215 }
216
217 @Override
218 synchronized void notifyRebound() {
219 if (isTerminated()) {
220 return;
221 }
222 webRTCWrapper.close();
223 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
224 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
225 }
226 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
227 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
228 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
229 } else {
230 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
231 finish();
232 }
233 }
234
235 //TODO: remove - hack to test dtmfSending
236 public DtmfSender getDtmfSender() {
237 return webRTCWrapper.getDtmfSender();
238 }
239
240 //FIXME: possible implementation
241 public boolean applyDtmfTone(String tone) {
242 return webRTCWrapper.applyDtmfTone(tone);
243 }
244
245
246 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
247 respondOk(jinglePacket);
248 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
249 final State previous = this.state;
250 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
251 if (TERMINATED.contains(previous)) {
252 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
253 return;
254 }
255 webRTCWrapper.close();
256 final State target = reasonToState(wrapper.reason);
257 transitionOrThrow(target);
258 writeLogMessage(target);
259 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
260 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
261 }
262 finish();
263 }
264
265 private void receiveTransportInfo(final JinglePacket jinglePacket) {
266 //Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to INITIALIZED only after transport-info has been received
267 if (isInState(State.NULL, State.PROCEED, State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
268 final RtpContentMap contentMap;
269 try {
270 contentMap = RtpContentMap.of(jinglePacket);
271 } catch (final IllegalArgumentException | NullPointerException e) {
272 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
273 respondOk(jinglePacket);
274 return;
275 }
276 receiveTransportInfo(jinglePacket, contentMap);
277 } else {
278 if (isTerminated()) {
279 respondOk(jinglePacket);
280 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated");
281 } else {
282 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
283 terminateWithOutOfOrder(jinglePacket);
284 }
285 }
286 }
287
288 private void receiveTransportInfo(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
289 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
290 if (this.state == State.SESSION_ACCEPTED) {
291 //zero candidates + modified credentials are an ICE restart offer
292 if (checkForIceRestart(jinglePacket, contentMap)) {
293 return;
294 }
295 respondOk(jinglePacket);
296 try {
297 processCandidates(candidates);
298 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
299 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
300 }
301 } else {
302 respondOk(jinglePacket);
303 pendingIceCandidates.addAll(candidates);
304 }
305 }
306
307 private boolean checkForIceRestart(final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
308 final RtpContentMap existing = getRemoteContentMap();
309 final IceUdpTransportInfo.Credentials existingCredentials;
310 final IceUdpTransportInfo.Credentials newCredentials;
311 try {
312 existingCredentials = existing.getCredentials();
313 newCredentials = rtpContentMap.getCredentials();
314 } catch (final IllegalStateException e) {
315 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
316 return false;
317 }
318 if (existingCredentials.equals(newCredentials)) {
319 return false;
320 }
321 //TODO an alternative approach is to check if we already got an iq result to our ICE-restart
322 // and if that's the case we are seeing an answer.
323 // This might be more spec compliant but also more error prone potentially
324 final boolean isOffer = rtpContentMap.emptyCandidates();
325 final RtpContentMap restartContentMap;
326 try {
327 if (isOffer) {
328 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
329 restartContentMap = existing.modifiedCredentials(newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
330 } else {
331 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
332 Log.d(Config.LOGTAG, "received confirmation of ICE restart" + newCredentials + " peer_setup=" + setup);
333 // DTLS setup attribute needs to be rewritten to reflect current peer state
334 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
335 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
336 }
337 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
338 return isOffer;
339 } else {
340 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
341 respondWithTieBreak(jinglePacket);
342 return true;
343 }
344 } catch (final Exception exception) {
345 respondOk(jinglePacket);
346 final Throwable rootCause = Throwables.getRootCause(exception);
347 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
348 //If this happens a termination is already in progress
349 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
350 return true;
351 }
352 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
353 webRTCWrapper.close();
354 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
355 return true;
356 }
357 }
358
359 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
360 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
361 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
362 throw new IllegalStateException("Invalid peer setup");
363 }
364 return peerSetup;
365 }
366
367 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
368 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
369 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
370 }
371 this.peerDtlsSetup = setup;
372 }
373
374 private boolean applyIceRestart(final JinglePacket jinglePacket, final RtpContentMap restartContentMap, final boolean isOffer) throws ExecutionException, InterruptedException {
375 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap);
376 final org.webrtc.SessionDescription.Type type = isOffer ? org.webrtc.SessionDescription.Type.OFFER : org.webrtc.SessionDescription.Type.ANSWER;
377 org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(type, sessionDescription.toString());
378 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
379 if (isInitiator()) {
380 //We ignore the offer and respond with tie-break. This will clause the responder not to apply the content map
381 return false;
382 }
383 }
384 webRTCWrapper.setRemoteDescription(sdp).get();
385 setRemoteContentMap(restartContentMap);
386 if (isOffer) {
387 webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
388 final SessionDescription localSessionDescription = setLocalSessionDescription();
389 setLocalContentMap(RtpContentMap.of(localSessionDescription));
390 //We need to respond OK before sending any candidates
391 respondOk(jinglePacket);
392 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
393 } else {
394 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
395 }
396 return true;
397 }
398
399 private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
400 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
401 processCandidate(content);
402 }
403 }
404
405 private void processCandidate(final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
406 final RtpContentMap rtpContentMap = getRemoteContentMap();
407 final List<String> indices = toIdentificationTags(rtpContentMap);
408 final String sdpMid = content.getKey(); //aka content name
409 final IceUdpTransportInfo transport = content.getValue().transport;
410 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
411
412 //TODO check that credentials remained the same
413
414 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
415 final String sdp;
416 try {
417 sdp = candidate.toSdpAttribute(credentials.ufrag);
418 } catch (final IllegalArgumentException e) {
419 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
420 continue;
421 }
422 final int mLineIndex = indices.indexOf(sdpMid);
423 if (mLineIndex < 0) {
424 Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
425 }
426 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
427 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
428 this.webRTCWrapper.addIceCandidate(iceCandidate);
429 }
430 }
431
432 private RtpContentMap getRemoteContentMap() {
433 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
434 }
435
436 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
437 final Group originalGroup = rtpContentMap.group;
438 final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
439 if (identificationTags.size() == 0) {
440 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
441 }
442 return identificationTags;
443 }
444
445 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
446 final RtpContentMap receivedContentMap;
447 try {
448 receivedContentMap = RtpContentMap.of(jinglePacket);
449 } catch (final Exception e) {
450 return Futures.immediateFailedFuture(e);
451 }
452 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
453 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
454 return Futures.transform(future, omemoVerifiedPayload -> {
455 //TODO test if an exception here triggers a correct abort
456 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
457 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received verifiable DTLS fingerprint via " + omemoVerification);
458 return omemoVerifiedPayload.getPayload();
459 }, MoreExecutors.directExecutor());
460 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
461 return Futures.immediateFailedFuture(
462 new SecurityException("DTLS fingerprint was unexpectedly not verifiable")
463 );
464 } else {
465 return Futures.immediateFuture(receivedContentMap);
466 }
467 }
468
469 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
470 if (isInitiator()) {
471 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
472 if (isTerminated()) {
473 Log.d(Config.LOGTAG, String.format(
474 "%s: got a reason to terminate with out-of-order. but already in state %s",
475 id.account.getJid().asBareJid(),
476 getState()
477 ));
478 respondWithOutOfOrder(jinglePacket);
479 } else {
480 terminateWithOutOfOrder(jinglePacket);
481 }
482 return;
483 }
484 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
485 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
486 @Override
487 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
488 receiveSessionInitiate(jinglePacket, rtpContentMap);
489 }
490
491 @Override
492 public void onFailure(@NonNull final Throwable throwable) {
493 respondOk(jinglePacket);
494 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
495 }
496 }, MoreExecutors.directExecutor());
497 }
498
499 private void receiveSessionInitiate(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
500 try {
501 contentMap.requireContentDescriptions();
502 contentMap.requireDTLSFingerprint(true);
503 } catch (final RuntimeException e) {
504 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e));
505 respondOk(jinglePacket);
506 sendSessionTerminate(Reason.of(e), e.getMessage());
507 return;
508 }
509 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
510 final State target;
511 if (this.state == State.PROCEED) {
512 Preconditions.checkState(
513 proposedMedia != null && proposedMedia.size() > 0,
514 "proposed media must be set when processing pre-approved session-initiate"
515 );
516 if (!this.proposedMedia.equals(contentMap.getMedia())) {
517 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
518 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
519 this.proposedMedia,
520 contentMap.getMedia()
521 ));
522 return;
523 }
524 target = State.SESSION_INITIALIZED_PRE_APPROVED;
525 } else {
526 target = State.SESSION_INITIALIZED;
527 }
528 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
529 respondOk(jinglePacket);
530 pendingIceCandidates.addAll(contentMap.contents.entrySet());
531 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
532 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
533 sendSessionAccept();
534 } else {
535 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
536 startRinging();
537 }
538 } else {
539 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
540 terminateWithOutOfOrder(jinglePacket);
541 }
542 }
543
544 private void receiveSessionAccept(final JinglePacket jinglePacket) {
545 if (!isInitiator()) {
546 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
547 terminateWithOutOfOrder(jinglePacket);
548 return;
549 }
550 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
551 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
552 @Override
553 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
554 receiveSessionAccept(jinglePacket, rtpContentMap);
555 }
556
557 @Override
558 public void onFailure(@NonNull final Throwable throwable) {
559 respondOk(jinglePacket);
560 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", throwable);
561 webRTCWrapper.close();
562 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
563 }
564 }, MoreExecutors.directExecutor());
565 }
566
567 private void receiveSessionAccept(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
568 try {
569 contentMap.requireContentDescriptions();
570 contentMap.requireDTLSFingerprint();
571 } catch (final RuntimeException e) {
572 respondOk(jinglePacket);
573 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
574 webRTCWrapper.close();
575 sendSessionTerminate(Reason.of(e), e.getMessage());
576 return;
577 }
578 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
579 if (!initiatorMedia.equals(contentMap.getMedia())) {
580 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
581 "Your session-included included media %s but our session-initiate was %s",
582 this.proposedMedia,
583 contentMap.getMedia()
584 ));
585 return;
586 }
587 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
588 if (transition(State.SESSION_ACCEPTED)) {
589 respondOk(jinglePacket);
590 receiveSessionAccept(contentMap);
591 } else {
592 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
593 respondOk(jinglePacket);
594 }
595 }
596
597 private void receiveSessionAccept(final RtpContentMap contentMap) {
598 this.responderRtpContentMap = contentMap;
599 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
600 final SessionDescription sessionDescription;
601 try {
602 sessionDescription = SessionDescription.of(contentMap);
603 } catch (final IllegalArgumentException | NullPointerException e) {
604 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
605 webRTCWrapper.close();
606 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
607 return;
608 }
609 final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
610 org.webrtc.SessionDescription.Type.ANSWER,
611 sessionDescription.toString()
612 );
613 try {
614 this.webRTCWrapper.setRemoteDescription(answer).get();
615 } catch (final Exception e) {
616 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e));
617 webRTCWrapper.close();
618 sendSessionTerminate(Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
619 return;
620 }
621 processCandidates(contentMap.contents.entrySet());
622 }
623
624 private void sendSessionAccept() {
625 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
626 if (rtpContentMap == null) {
627 throw new IllegalStateException("initiator RTP Content Map has not been set");
628 }
629 final SessionDescription offer;
630 try {
631 offer = SessionDescription.of(rtpContentMap);
632 } catch (final IllegalArgumentException | NullPointerException e) {
633 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
634 webRTCWrapper.close();
635 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
636 return;
637 }
638 sendSessionAccept(rtpContentMap.getMedia(), offer);
639 }
640
641 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
642 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
643 }
644
645 private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
646 if (isTerminated()) {
647 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
648 return;
649 }
650 try {
651 setupWebRTC(media, iceServers);
652 } catch (final WebRTCWrapper.InitializationException e) {
653 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
654 webRTCWrapper.close();
655 sendSessionTerminate(Reason.FAILED_APPLICATION);
656 return;
657 }
658 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
659 org.webrtc.SessionDescription.Type.OFFER,
660 offer.toString()
661 );
662 try {
663 this.webRTCWrapper.setRemoteDescription(sdp).get();
664 addIceCandidatesFromBlackLog();
665 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
666 prepareSessionAccept(webRTCSessionDescription);
667 } catch (final Exception e) {
668 failureToAcceptSession(e);
669 }
670 }
671
672 private void failureToAcceptSession(final Throwable throwable) {
673 if (isTerminated()) {
674 return;
675 }
676 final Throwable rootCause = Throwables.getRootCause(throwable);
677 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
678 webRTCWrapper.close();
679 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
680 }
681
682 private void addIceCandidatesFromBlackLog() {
683 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
684 while ((foo = this.pendingIceCandidates.poll()) != null) {
685 processCandidate(foo);
686 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidate from back log");
687 }
688 }
689
690 private void prepareSessionAccept(final org.webrtc.SessionDescription webRTCSessionDescription) {
691 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
692 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
693 this.responderRtpContentMap = respondingRtpContentMap;
694 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
695 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
696 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = prepareOutgoingContentMap(respondingRtpContentMap);
697 Futures.addCallback(outgoingContentMapFuture,
698 new FutureCallback<RtpContentMap>() {
699 @Override
700 public void onSuccess(final RtpContentMap outgoingContentMap) {
701 sendSessionAccept(outgoingContentMap);
702 }
703
704 @Override
705 public void onFailure(@NonNull Throwable throwable) {
706 failureToAcceptSession(throwable);
707 }
708 },
709 MoreExecutors.directExecutor()
710 );
711 }
712
713 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
714 if (isTerminated()) {
715 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session accept was too slow. already terminated. nothing to do.");
716 return;
717 }
718 transitionOrThrow(State.SESSION_ACCEPTED);
719 final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
720 send(sessionAccept);
721 }
722
723 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(final RtpContentMap rtpContentMap) {
724 if (this.omemoVerification.hasDeviceId()) {
725 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
726 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
727 return Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
728 omemoVerification.setOrEnsureEqual(verifiedPayload);
729 return verifiedPayload.getPayload();
730 }, MoreExecutors.directExecutor());
731 } else {
732 return Futures.immediateFuture(rtpContentMap);
733 }
734 }
735
736 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
737 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
738 switch (message.getName()) {
739 case "propose":
740 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
741 break;
742 case "proceed":
743 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
744 break;
745 case "retract":
746 receiveRetract(from, serverMessageId, timestamp);
747 break;
748 case "reject":
749 receiveReject(from, serverMessageId, timestamp);
750 break;
751 case "accept":
752 receiveAccept(from, serverMessageId, timestamp);
753 break;
754 default:
755 break;
756 }
757 }
758
759 void deliverFailedProceed() {
760 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
761 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
762 webRTCWrapper.close();
763 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
764 this.finish();
765 }
766 }
767
768 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
769 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
770 if (originatedFromMyself) {
771 if (transition(State.ACCEPTED)) {
772 if (serverMsgId != null) {
773 this.message.setServerMsgId(serverMsgId);
774 }
775 this.message.setTime(timestamp);
776 this.message.setCarbon(true); //indicate that call was accepted on other device
777 this.writeLogMessageSuccess(0);
778 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
779 this.finish();
780 } else {
781 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
782 }
783 } else {
784 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
785 }
786 }
787
788 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
789 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
790 //reject from another one of my clients
791 if (originatedFromMyself) {
792 receiveRejectFromMyself(serverMsgId, timestamp);
793 } else if (isInitiator()) {
794 if (from.equals(id.with)) {
795 receiveRejectFromResponder();
796 } else {
797 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
798 }
799 } else {
800 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
801 }
802 }
803
804 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
805 if (transition(State.REJECTED)) {
806 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
807 this.finish();
808 if (serverMsgId != null) {
809 this.message.setServerMsgId(serverMsgId);
810 }
811 this.message.setTime(timestamp);
812 this.message.setCarbon(true); //indicate that call was rejected on other device
813 writeLogMessageMissed();
814 } else {
815 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
816 }
817 }
818
819 private void receiveRejectFromResponder() {
820 if (isInState(State.PROCEED)) {
821 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered");
822 closeTransitionLogFinish(State.REJECTED_RACED);
823 return;
824 }
825 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
826 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
827 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
828 return;
829 }
830 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state);
831 }
832
833 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
834 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
835 if (originatedFromMyself) {
836 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
837 } else if (transition(State.PROPOSED, () -> {
838 final Collection<RtpDescription> descriptions = Collections2.transform(
839 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
840 input -> (RtpDescription) input
841 );
842 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
843 Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
844 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
845 this.proposedMedia = Sets.newHashSet(media);
846 })) {
847 if (serverMsgId != null) {
848 this.message.setServerMsgId(serverMsgId);
849 }
850 this.message.setTime(timestamp);
851 startRinging();
852 } else {
853 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
854 }
855 }
856
857 private void startRinging() {
858 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
859 ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
860 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
861 }
862
863 private synchronized void ringingTimeout() {
864 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
865 switch (this.state) {
866 case PROPOSED:
867 message.markUnread();
868 rejectCallFromProposed();
869 break;
870 case SESSION_INITIALIZED:
871 message.markUnread();
872 rejectCallFromSessionInitiate();
873 break;
874 }
875 }
876
877 private void cancelRingingTimeout() {
878 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
879 if (future != null && !future.isCancelled()) {
880 future.cancel(false);
881 }
882 }
883
884 private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
885 final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
886 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
887 if (from.equals(id.with)) {
888 if (isInitiator()) {
889 if (transition(State.PROCEED)) {
890 if (serverMsgId != null) {
891 this.message.setServerMsgId(serverMsgId);
892 }
893 this.message.setTime(timestamp);
894 final Integer remoteDeviceId = proceed.getDeviceId();
895 if (isOmemoEnabled()) {
896 this.omemoVerification.setDeviceId(remoteDeviceId);
897 } else {
898 if (remoteDeviceId != null) {
899 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
900 }
901 this.omemoVerification.setDeviceId(null);
902 }
903 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
904 } else {
905 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
906 }
907 } else {
908 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
909 }
910 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
911 if (transition(State.ACCEPTED)) {
912 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
913 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
914 this.finish();
915 }
916 } else {
917 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
918 }
919 }
920
921 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
922 if (from.equals(id.with)) {
923 final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
924 if (transition(target)) {
925 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
926 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
927 if (serverMsgId != null) {
928 this.message.setServerMsgId(serverMsgId);
929 }
930 this.message.setTime(timestamp);
931 if (target == State.RETRACTED) {
932 this.message.markUnread();
933 }
934 writeLogMessageMissed();
935 finish();
936 } else {
937 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
938 }
939 } else {
940 //TODO parse retract from self
941 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
942 }
943 }
944
945 public void sendSessionInitiate() {
946 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
947 }
948
949 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
950 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
951 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
952 }
953
954 private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
955 if (isTerminated()) {
956 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
957 return;
958 }
959 try {
960 setupWebRTC(media, iceServers);
961 } catch (final WebRTCWrapper.InitializationException e) {
962 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
963 webRTCWrapper.close();
964 sendRetract(Reason.ofThrowable(e));
965 return;
966 }
967 try {
968 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
969 prepareSessionInitiate(webRTCSessionDescription, targetState);
970 } catch (final Exception e) {
971 //TODO sending the error text is worthwhile as well. Especially for FailureToSet exceptions
972 failureToInitiateSession(e, targetState);
973 }
974 }
975
976 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
977 if (isTerminated()) {
978 return;
979 }
980 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(throwable));
981 webRTCWrapper.close();
982 final Reason reason = Reason.ofThrowable(throwable);
983 if (isInState(targetState)) {
984 sendSessionTerminate(reason);
985 } else {
986 sendRetract(reason);
987 }
988 }
989
990 private void sendRetract(final Reason reason) {
991 //TODO embed reason into retract
992 sendJingleMessage("retract", id.with.asBareJid());
993 transitionOrThrow(reasonToState(reason));
994 this.finish();
995 }
996
997 private void prepareSessionInitiate(final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
998 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
999 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
1000 this.initiatorRtpContentMap = rtpContentMap;
1001 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1002 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = encryptSessionInitiate(rtpContentMap);
1003 Futures.addCallback(outgoingContentMapFuture, new FutureCallback<RtpContentMap>() {
1004 @Override
1005 public void onSuccess(final RtpContentMap outgoingContentMap) {
1006 sendSessionInitiate(outgoingContentMap, targetState);
1007 }
1008
1009 @Override
1010 public void onFailure(@NonNull final Throwable throwable) {
1011 failureToInitiateSession(throwable, targetState);
1012 }
1013 }, MoreExecutors.directExecutor());
1014 }
1015
1016 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1017 if (isTerminated()) {
1018 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session was too slow. already terminated. nothing to do.");
1019 return;
1020 }
1021 this.transitionOrThrow(targetState);
1022 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1023 send(sessionInitiate);
1024 }
1025
1026 private ListenableFuture<RtpContentMap> encryptSessionInitiate(final RtpContentMap rtpContentMap) {
1027 if (this.omemoVerification.hasDeviceId()) {
1028 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
1029 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
1030 final ListenableFuture<RtpContentMap> future = Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
1031 omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint());
1032 return verifiedPayload.getPayload();
1033 }, MoreExecutors.directExecutor());
1034 if (Config.REQUIRE_RTP_VERIFICATION) {
1035 return future;
1036 }
1037 return Futures.catching(
1038 future,
1039 CryptoFailedException.class,
1040 e -> {
1041 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e);
1042 return rtpContentMap;
1043 },
1044 MoreExecutors.directExecutor()
1045 );
1046 } else {
1047 return Futures.immediateFuture(rtpContentMap);
1048 }
1049 }
1050
1051 private void sendSessionTerminate(final Reason reason) {
1052 sendSessionTerminate(reason, null);
1053 }
1054
1055 private void sendSessionTerminate(final Reason reason, final String text) {
1056 final State previous = this.state;
1057 final State target = reasonToState(reason);
1058 transitionOrThrow(target);
1059 if (previous != State.NULL) {
1060 writeLogMessage(target);
1061 }
1062 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1063 jinglePacket.setReason(reason, text);
1064 Log.d(Config.LOGTAG, jinglePacket.toString());
1065 send(jinglePacket);
1066 finish();
1067 }
1068
1069 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
1070 final RtpContentMap transportInfo;
1071 try {
1072 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1073 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1074 } catch (final Exception e) {
1075 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
1076 return;
1077 }
1078 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1079 send(jinglePacket);
1080 }
1081
1082 private void send(final JinglePacket jinglePacket) {
1083 jinglePacket.setTo(id.with);
1084 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1085 }
1086
1087 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1088 if (response.getType() == IqPacket.TYPE.ERROR) {
1089 handleIqErrorResponse(response);
1090 return;
1091 }
1092 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1093 handleIqTimeoutResponse(response);
1094 }
1095 }
1096
1097 private void handleIqErrorResponse(final IqPacket response) {
1098 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1099 final String errorCondition = response.getErrorCondition();
1100 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
1101 if (isTerminated()) {
1102 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
1103 return;
1104 }
1105 this.webRTCWrapper.close();
1106 final State target;
1107 if (Arrays.asList(
1108 "service-unavailable",
1109 "recipient-unavailable",
1110 "remote-server-not-found",
1111 "remote-server-timeout"
1112 ).contains(errorCondition)) {
1113 target = State.TERMINATED_CONNECTIVITY_ERROR;
1114 } else {
1115 target = State.TERMINATED_APPLICATION_FAILURE;
1116 }
1117 transitionOrThrow(target);
1118 this.finish();
1119 }
1120
1121 private void handleIqTimeoutResponse(final IqPacket response) {
1122 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1123 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
1124 if (isTerminated()) {
1125 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
1126 return;
1127 }
1128 this.webRTCWrapper.close();
1129 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
1130 this.finish();
1131 }
1132
1133 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1134 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1135 this.webRTCWrapper.close();
1136 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1137 respondWithOutOfOrder(jinglePacket);
1138 this.finish();
1139 }
1140
1141 private void respondWithTieBreak(final JinglePacket jinglePacket) {
1142 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
1143 }
1144
1145 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1146 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
1147 }
1148
1149 void respondWithJingleError(final IqPacket original, String jingleCondition, String condition, String conditionType) {
1150 jingleConnectionManager.respondWithJingleError(id.account, original, jingleCondition, condition, conditionType);
1151 }
1152
1153 private void respondOk(final JinglePacket jinglePacket) {
1154 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1155 }
1156
1157 public void throwStateTransitionException() {
1158 final StateTransitionException exception = this.stateTransitionException;
1159 if (exception != null) {
1160 throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception);
1161 }
1162 }
1163
1164 public RtpEndUserState getEndUserState() {
1165 switch (this.state) {
1166 case NULL:
1167 case PROPOSED:
1168 case SESSION_INITIALIZED:
1169 if (isInitiator()) {
1170 return RtpEndUserState.RINGING;
1171 } else {
1172 return RtpEndUserState.INCOMING_CALL;
1173 }
1174 case PROCEED:
1175 if (isInitiator()) {
1176 return RtpEndUserState.RINGING;
1177 } else {
1178 return RtpEndUserState.ACCEPTING_CALL;
1179 }
1180 case SESSION_INITIALIZED_PRE_APPROVED:
1181 if (isInitiator()) {
1182 return RtpEndUserState.RINGING;
1183 } else {
1184 return RtpEndUserState.CONNECTING;
1185 }
1186 case SESSION_ACCEPTED:
1187 return getPeerConnectionStateAsEndUserState();
1188 case REJECTED:
1189 case REJECTED_RACED:
1190 case TERMINATED_DECLINED_OR_BUSY:
1191 if (isInitiator()) {
1192 return RtpEndUserState.DECLINED_OR_BUSY;
1193 } else {
1194 return RtpEndUserState.ENDED;
1195 }
1196 case TERMINATED_SUCCESS:
1197 case ACCEPTED:
1198 case RETRACTED:
1199 case TERMINATED_CANCEL_OR_TIMEOUT:
1200 return RtpEndUserState.ENDED;
1201 case RETRACTED_RACED:
1202 if (isInitiator()) {
1203 return RtpEndUserState.ENDED;
1204 } else {
1205 return RtpEndUserState.RETRACTED;
1206 }
1207 case TERMINATED_CONNECTIVITY_ERROR:
1208 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1209 case TERMINATED_APPLICATION_FAILURE:
1210 return RtpEndUserState.APPLICATION_ERROR;
1211 case TERMINATED_SECURITY_ERROR:
1212 return RtpEndUserState.SECURITY_ERROR;
1213 }
1214 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
1215 }
1216
1217
1218 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
1219 final PeerConnection.PeerConnectionState state;
1220 try {
1221 state = webRTCWrapper.getState();
1222 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
1223 //We usually close the WebRTCWrapper *before* transitioning so we might still
1224 //be in SESSION_ACCEPTED even though the peerConnection has been torn down
1225 return RtpEndUserState.ENDING_CALL;
1226 }
1227 switch (state) {
1228 case CONNECTED:
1229 return RtpEndUserState.CONNECTED;
1230 case NEW:
1231 case CONNECTING:
1232 return RtpEndUserState.CONNECTING;
1233 case CLOSED:
1234 return RtpEndUserState.ENDING_CALL;
1235 default:
1236 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.RECONNECTING;
1237 }
1238 }
1239
1240 public Set<Media> getMedia() {
1241 final State current = getState();
1242 if (current == State.NULL) {
1243 if (isInitiator()) {
1244 return Preconditions.checkNotNull(
1245 this.proposedMedia,
1246 "RTP connection has not been initialized properly"
1247 );
1248 }
1249 throw new IllegalStateException("RTP connection has not been initialized yet");
1250 }
1251 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
1252 return Preconditions.checkNotNull(
1253 this.proposedMedia,
1254 "RTP connection has not been initialized properly"
1255 );
1256 }
1257 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
1258 if (initiatorContentMap != null) {
1259 return initiatorContentMap.getMedia();
1260 } else if (isTerminated()) {
1261 return Collections.emptySet(); //we might fail before we ever got a chance to set media
1262 } else {
1263 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
1264 }
1265 }
1266
1267
1268 public boolean isVerified() {
1269 final String fingerprint = this.omemoVerification.getFingerprint();
1270 if (fingerprint == null) {
1271 return false;
1272 }
1273 final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint);
1274 return status != null && status.isVerified();
1275 }
1276
1277 public synchronized void acceptCall() {
1278 switch (this.state) {
1279 case PROPOSED:
1280 cancelRingingTimeout();
1281 acceptCallFromProposed();
1282 break;
1283 case SESSION_INITIALIZED:
1284 cancelRingingTimeout();
1285 acceptCallFromSessionInitialized();
1286 break;
1287 case ACCEPTED:
1288 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind");
1289 break;
1290 case PROCEED:
1291 case SESSION_ACCEPTED:
1292 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI");
1293 break;
1294 default:
1295 throw new IllegalStateException("Can not accept call from " + this.state);
1296 }
1297 }
1298
1299
1300 public void notifyPhoneCall() {
1301 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1302 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1303 rejectCall();
1304 } else {
1305 endCall();
1306 }
1307 }
1308
1309 public synchronized void rejectCall() {
1310 if (isTerminated()) {
1311 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do");
1312 return;
1313 }
1314 switch (this.state) {
1315 case PROPOSED:
1316 rejectCallFromProposed();
1317 break;
1318 case SESSION_INITIALIZED:
1319 rejectCallFromSessionInitiate();
1320 break;
1321 default:
1322 throw new IllegalStateException("Can not reject call from " + this.state);
1323 }
1324 }
1325
1326 public synchronized void endCall() {
1327 if (isTerminated()) {
1328 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
1329 return;
1330 }
1331 if (isInState(State.PROPOSED) && !isInitiator()) {
1332 rejectCallFromProposed();
1333 return;
1334 }
1335 if (isInState(State.PROCEED)) {
1336 if (isInitiator()) {
1337 retractFromProceed();
1338 } else {
1339 rejectCallFromProceed();
1340 }
1341 return;
1342 }
1343 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1344 this.webRTCWrapper.close();
1345 sendSessionTerminate(Reason.CANCEL);
1346 return;
1347 }
1348 if (isInState(State.SESSION_INITIALIZED)) {
1349 rejectCallFromSessionInitiate();
1350 return;
1351 }
1352 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1353 this.webRTCWrapper.close();
1354 sendSessionTerminate(Reason.SUCCESS);
1355 return;
1356 }
1357 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
1358 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
1359 return;
1360 }
1361 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1362 }
1363
1364 private void retractFromProceed() {
1365 Log.d(Config.LOGTAG, "retract from proceed");
1366 this.sendJingleMessage("retract");
1367 closeTransitionLogFinish(State.RETRACTED_RACED);
1368 }
1369
1370 private void closeTransitionLogFinish(final State state) {
1371 this.webRTCWrapper.close();
1372 transitionOrThrow(state);
1373 writeLogMessage(state);
1374 finish();
1375 }
1376
1377 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
1378 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1379 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1380 if (media.contains(Media.VIDEO)) {
1381 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1382 } else {
1383 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1384 }
1385 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1386 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1387 }
1388
1389 private void acceptCallFromProposed() {
1390 transitionOrThrow(State.PROCEED);
1391 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1392 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1393 this.sendJingleMessage("proceed");
1394 }
1395
1396 private void rejectCallFromProposed() {
1397 transitionOrThrow(State.REJECTED);
1398 writeLogMessageMissed();
1399 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1400 this.sendJingleMessage("reject");
1401 finish();
1402 }
1403
1404 private void rejectCallFromProceed() {
1405 this.sendJingleMessage("reject");
1406 closeTransitionLogFinish(State.REJECTED_RACED);
1407 }
1408
1409 private void rejectCallFromSessionInitiate() {
1410 webRTCWrapper.close();
1411 sendSessionTerminate(Reason.DECLINE);
1412 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1413 }
1414
1415 private void sendJingleMessage(final String action) {
1416 sendJingleMessage(action, id.with);
1417 }
1418
1419 private void sendJingleMessage(final String action, final Jid to) {
1420 final MessagePacket messagePacket = new MessagePacket();
1421 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
1422 messagePacket.setTo(to);
1423 final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
1424 if ("proceed".equals(action)) {
1425 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1426 if (isOmemoEnabled()) {
1427 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1428 final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1429 device.setAttribute("id", deviceId);
1430 }
1431 }
1432 messagePacket.addChild("store", "urn:xmpp:hints");
1433 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1434 }
1435
1436 private boolean isOmemoEnabled() {
1437 final Conversational conversational = message.getConversation();
1438 if (conversational instanceof Conversation) {
1439 return ((Conversation) conversational).getNextEncryption() == Message.ENCRYPTION_AXOLOTL;
1440 }
1441 return false;
1442 }
1443
1444 private void acceptCallFromSessionInitialized() {
1445 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1446 sendSessionAccept();
1447 }
1448
1449 private synchronized boolean isInState(State... state) {
1450 return Arrays.asList(state).contains(this.state);
1451 }
1452
1453 private boolean transition(final State target) {
1454 return transition(target, null);
1455 }
1456
1457 private synchronized boolean transition(final State target, final Runnable runnable) {
1458 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1459 if (validTransitions != null && validTransitions.contains(target)) {
1460 this.state = target;
1461 this.stateTransitionException = new StateTransitionException(target);
1462 if (runnable != null) {
1463 runnable.run();
1464 }
1465 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1466 updateEndUserState();
1467 updateOngoingCallNotification();
1468 return true;
1469 } else {
1470 return false;
1471 }
1472 }
1473
1474 void transitionOrThrow(final State target) {
1475 if (!transition(target)) {
1476 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
1477 }
1478 }
1479
1480 @Override
1481 public void onIceCandidate(final IceCandidate iceCandidate) {
1482 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1483 final String ufrag = rtpContentMap.getCredentials().ufrag;
1484 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, ufrag);
1485 if (candidate == null) {
1486 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate.toString());
1487 return;
1488 }
1489 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
1490 sendTransportInfo(iceCandidate.sdpMid, candidate);
1491 }
1492
1493 @Override
1494 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
1495 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
1496 this.stateHistory.add(newState);
1497 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
1498 this.sessionDuration.start();
1499 updateOngoingCallNotification();
1500 } else if (this.sessionDuration.isRunning()) {
1501 this.sessionDuration.stop();
1502 updateOngoingCallNotification();
1503 }
1504
1505 final boolean neverConnected = !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
1506
1507 if (newState == PeerConnection.PeerConnectionState.FAILED) {
1508 if (neverConnected) {
1509 if (isTerminated()) {
1510 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
1511 return;
1512 }
1513 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
1514 return;
1515 } else {
1516 webRTCWrapper.restartIce();
1517 }
1518 }
1519 updateEndUserState();
1520 }
1521
1522 @Override
1523 public void onRenegotiationNeeded() {
1524 this.webRTCWrapper.execute(this::initiateIceRestart);
1525 }
1526
1527 private void initiateIceRestart() {
1528 //TODO discover new TURN/STUN credentials
1529 this.stateHistory.clear();
1530 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
1531 final SessionDescription sessionDescription;
1532 try {
1533 sessionDescription = setLocalSessionDescription();
1534 } catch (final Exception e) {
1535 final Throwable cause = Throwables.getRootCause(e);
1536 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
1537 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
1538 return;
1539 }
1540 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
1541 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
1542 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1543 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
1544 jinglePacket.setTo(id.with);
1545 xmppConnectionService.sendIqPacket(id.account, jinglePacket, (account, response) -> {
1546 if (response.getType() == IqPacket.TYPE.RESULT) {
1547 Log.d(Config.LOGTAG, "received success to our ice restart");
1548 setLocalContentMap(rtpContentMap);
1549 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1550 return;
1551 }
1552 if (response.getType() == IqPacket.TYPE.ERROR) {
1553 final Element error = response.findChild("error");
1554 if (error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS)) {
1555 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
1556 return;
1557 }
1558 handleIqErrorResponse(response);
1559 }
1560 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1561 handleIqTimeoutResponse(response);
1562 }
1563 });
1564 }
1565
1566 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
1567 if (isInitiator()) {
1568 this.initiatorRtpContentMap = rtpContentMap;
1569 } else {
1570 this.responderRtpContentMap = rtpContentMap;
1571 }
1572 }
1573
1574 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
1575 if (isInitiator()) {
1576 this.responderRtpContentMap = rtpContentMap;
1577 } else {
1578 this.initiatorRtpContentMap = rtpContentMap;
1579 }
1580 }
1581
1582 private SessionDescription setLocalSessionDescription() throws ExecutionException, InterruptedException {
1583 final org.webrtc.SessionDescription sessionDescription = this.webRTCWrapper.setLocalDescription().get();
1584 return SessionDescription.parse(sessionDescription.description);
1585 }
1586
1587 private void closeWebRTCSessionAfterFailedConnection() {
1588 this.webRTCWrapper.close();
1589 synchronized (this) {
1590 if (isTerminated()) {
1591 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did");
1592 return;
1593 }
1594 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
1595 }
1596 }
1597
1598 public boolean zeroDuration() {
1599 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
1600 }
1601
1602 public long getCallDuration() {
1603 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
1604 }
1605
1606 public AppRTCAudioManager getAudioManager() {
1607 return webRTCWrapper.getAudioManager();
1608 }
1609
1610 public boolean isMicrophoneEnabled() {
1611 return webRTCWrapper.isMicrophoneEnabled();
1612 }
1613
1614 public boolean setMicrophoneEnabled(final boolean enabled) {
1615 return webRTCWrapper.setMicrophoneEnabled(enabled);
1616 }
1617
1618 public boolean isVideoEnabled() {
1619 return webRTCWrapper.isVideoEnabled();
1620 }
1621
1622 public void setVideoEnabled(final boolean enabled) {
1623 webRTCWrapper.setVideoEnabled(enabled);
1624 }
1625
1626 public boolean isCameraSwitchable() {
1627 return webRTCWrapper.isCameraSwitchable();
1628 }
1629
1630 public boolean isFrontCamera() {
1631 return webRTCWrapper.isFrontCamera();
1632 }
1633
1634 public ListenableFuture<Boolean> switchCamera() {
1635 return webRTCWrapper.switchCamera();
1636 }
1637
1638 @Override
1639 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1640 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
1641 }
1642
1643 private void updateEndUserState() {
1644 final RtpEndUserState endUserState = getEndUserState();
1645 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
1646 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1647 }
1648
1649 private void updateOngoingCallNotification() {
1650 final State state = this.state;
1651 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
1652 final boolean reconnecting;
1653 if (state == State.SESSION_ACCEPTED) {
1654 reconnecting = getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
1655 } else {
1656 reconnecting = false;
1657 }
1658 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
1659 } else {
1660 xmppConnectionService.removeOngoingCall();
1661 }
1662 }
1663
1664 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
1665 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
1666 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
1667 request.setTo(id.account.getDomain());
1668 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1669 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
1670 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
1671 if (response.getType() == IqPacket.TYPE.RESULT) {
1672 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1673 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
1674 for (final Element child : children) {
1675 if ("service".equals(child.getName())) {
1676 final String type = child.getAttribute("type");
1677 final String host = child.getAttribute("host");
1678 final String sport = child.getAttribute("port");
1679 final Integer port = sport == null ? null : Ints.tryParse(sport);
1680 final String transport = child.getAttribute("transport");
1681 final String username = child.getAttribute("username");
1682 final String password = child.getAttribute("password");
1683 if (Strings.isNullOrEmpty(host) || port == null) {
1684 continue;
1685 }
1686 if (port < 0 || port > 65535) {
1687 continue;
1688 }
1689 if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) {
1690 if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) {
1691 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services");
1692 continue;
1693 }
1694 final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer
1695 .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport));
1696 iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK);
1697 if (username != null && password != null) {
1698 iceServerBuilder.setUsername(username);
1699 iceServerBuilder.setPassword(password);
1700 } else if (Arrays.asList("turn", "turns").contains(type)) {
1701 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
1702 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
1703 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
1704 continue;
1705 }
1706 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
1707 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
1708 listBuilder.add(iceServer);
1709 }
1710 }
1711 }
1712 }
1713 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
1714 if (iceServers.size() == 0) {
1715 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
1716 }
1717 onIceServersDiscovered.onIceServersDiscovered(iceServers);
1718 });
1719 } else {
1720 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
1721 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
1722 }
1723 }
1724
1725 private void finish() {
1726 if (isTerminated()) {
1727 this.cancelRingingTimeout();
1728 this.webRTCWrapper.verifyClosed();
1729 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
1730 this.jingleConnectionManager.finishConnectionOrThrow(this);
1731 } else {
1732 throw new IllegalStateException(String.format("Unable to call finish from %s", this.state));
1733 }
1734 }
1735
1736 private void writeLogMessage(final State state) {
1737 final long duration = getCallDuration();
1738 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
1739 writeLogMessageSuccess(duration);
1740 } else {
1741 writeLogMessageMissed();
1742 }
1743 }
1744
1745 private void writeLogMessageSuccess(final long duration) {
1746 this.message.setBody(new RtpSessionStatus(true, duration).toString());
1747 this.writeMessage();
1748 }
1749
1750 private void writeLogMessageMissed() {
1751 this.message.setBody(new RtpSessionStatus(false, 0).toString());
1752 this.writeMessage();
1753 }
1754
1755 private void writeMessage() {
1756 final Conversational conversational = message.getConversation();
1757 if (conversational instanceof Conversation) {
1758 ((Conversation) conversational).add(this.message);
1759 xmppConnectionService.createMessageAsync(message);
1760 xmppConnectionService.updateConversationUi();
1761 } else {
1762 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1763 }
1764 }
1765
1766 public State getState() {
1767 return this.state;
1768 }
1769
1770 boolean isTerminated() {
1771 return TERMINATED.contains(this.state);
1772 }
1773
1774 public Optional<VideoTrack> getLocalVideoTrack() {
1775 return webRTCWrapper.getLocalVideoTrack();
1776 }
1777
1778 public Optional<VideoTrack> getRemoteVideoTrack() {
1779 return webRTCWrapper.getRemoteVideoTrack();
1780 }
1781
1782 public EglBase.Context getEglBaseContext() {
1783 return webRTCWrapper.getEglBaseContext();
1784 }
1785
1786 void setProposedMedia(final Set<Media> media) {
1787 this.proposedMedia = media;
1788 }
1789
1790 public void fireStateUpdate() {
1791 final RtpEndUserState endUserState = getEndUserState();
1792 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1793 }
1794
1795 private interface OnIceServersDiscovered {
1796 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1797 }
1798
1799 private static class StateTransitionException extends Exception {
1800 private final State state;
1801
1802 private StateTransitionException(final State state) {
1803 this.state = state;
1804 }
1805 }
1806}