1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import androidx.annotation.NonNull;
6import androidx.annotation.Nullable;
7
8import com.google.common.base.Optional;
9import com.google.common.base.Preconditions;
10import com.google.common.base.Stopwatch;
11import com.google.common.base.Strings;
12import com.google.common.base.Throwables;
13import com.google.common.collect.Collections2;
14import com.google.common.collect.ImmutableList;
15import com.google.common.collect.ImmutableMap;
16import com.google.common.collect.Sets;
17import com.google.common.primitives.Ints;
18import com.google.common.util.concurrent.FutureCallback;
19import com.google.common.util.concurrent.Futures;
20import com.google.common.util.concurrent.ListenableFuture;
21import com.google.common.util.concurrent.MoreExecutors;
22
23import org.webrtc.EglBase;
24import org.webrtc.IceCandidate;
25import org.webrtc.PeerConnection;
26import org.webrtc.VideoTrack;
27
28import java.util.ArrayDeque;
29import java.util.Arrays;
30import java.util.Collection;
31import java.util.Collections;
32import java.util.LinkedList;
33import java.util.List;
34import java.util.Map;
35import java.util.Queue;
36import java.util.Set;
37import java.util.concurrent.ScheduledFuture;
38import java.util.concurrent.TimeUnit;
39
40import eu.siacs.conversations.Config;
41import eu.siacs.conversations.crypto.axolotl.AxolotlService;
42import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
43import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
44import eu.siacs.conversations.entities.Account;
45import eu.siacs.conversations.entities.Conversation;
46import eu.siacs.conversations.entities.Conversational;
47import eu.siacs.conversations.entities.Message;
48import eu.siacs.conversations.entities.RtpSessionStatus;
49import eu.siacs.conversations.services.AppRTCAudioManager;
50import eu.siacs.conversations.utils.IP;
51import eu.siacs.conversations.xml.Element;
52import eu.siacs.conversations.xml.Namespace;
53import eu.siacs.conversations.xmpp.Jid;
54import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
55import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
56import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
57import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
58import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
59import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
60import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
61import eu.siacs.conversations.xmpp.stanzas.IqPacket;
62import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
63
64public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
65
66 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
67 State.PROCEED,
68 State.SESSION_INITIALIZED_PRE_APPROVED,
69 State.SESSION_ACCEPTED
70 );
71 private static final long BUSY_TIME_OUT = 30;
72 private static final List<State> TERMINATED = Arrays.asList(
73 State.ACCEPTED,
74 State.REJECTED,
75 State.REJECTED_RACED,
76 State.RETRACTED,
77 State.RETRACTED_RACED,
78 State.TERMINATED_SUCCESS,
79 State.TERMINATED_DECLINED_OR_BUSY,
80 State.TERMINATED_CONNECTIVITY_ERROR,
81 State.TERMINATED_CANCEL_OR_TIMEOUT,
82 State.TERMINATED_APPLICATION_FAILURE,
83 State.TERMINATED_SECURITY_ERROR
84 );
85
86 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
87
88 static {
89 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
90 transitionBuilder.put(State.NULL, ImmutableList.of(
91 State.PROPOSED,
92 State.SESSION_INITIALIZED,
93 State.TERMINATED_APPLICATION_FAILURE,
94 State.TERMINATED_SECURITY_ERROR
95 ));
96 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
97 State.ACCEPTED,
98 State.PROCEED,
99 State.REJECTED,
100 State.RETRACTED,
101 State.TERMINATED_APPLICATION_FAILURE,
102 State.TERMINATED_SECURITY_ERROR,
103 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
104 ));
105 transitionBuilder.put(State.PROCEED, ImmutableList.of(
106 State.REJECTED_RACED,
107 State.RETRACTED_RACED,
108 State.SESSION_INITIALIZED_PRE_APPROVED,
109 State.TERMINATED_SUCCESS,
110 State.TERMINATED_APPLICATION_FAILURE,
111 State.TERMINATED_SECURITY_ERROR,
112 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
113 ));
114 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
115 State.SESSION_ACCEPTED,
116 State.TERMINATED_SUCCESS,
117 State.TERMINATED_DECLINED_OR_BUSY,
118 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
119 State.TERMINATED_CANCEL_OR_TIMEOUT,
120 State.TERMINATED_APPLICATION_FAILURE,
121 State.TERMINATED_SECURITY_ERROR
122 ));
123 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
124 State.SESSION_ACCEPTED,
125 State.TERMINATED_SUCCESS,
126 State.TERMINATED_DECLINED_OR_BUSY,
127 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
128 State.TERMINATED_CANCEL_OR_TIMEOUT,
129 State.TERMINATED_APPLICATION_FAILURE,
130 State.TERMINATED_SECURITY_ERROR
131 ));
132 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
133 State.TERMINATED_SUCCESS,
134 State.TERMINATED_DECLINED_OR_BUSY,
135 State.TERMINATED_CONNECTIVITY_ERROR,
136 State.TERMINATED_CANCEL_OR_TIMEOUT,
137 State.TERMINATED_APPLICATION_FAILURE,
138 State.TERMINATED_SECURITY_ERROR
139 ));
140 VALID_TRANSITIONS = transitionBuilder.build();
141 }
142
143 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
144 private final ArrayDeque<Set<Map.Entry<String, RtpContentMap.DescriptionTransport>>> pendingIceCandidates = new ArrayDeque<>();
145 private final OmemoVerification omemoVerification = new OmemoVerification();
146 private final Message message;
147 private State state = State.NULL;
148 private StateTransitionException stateTransitionException;
149 private Set<Media> proposedMedia;
150 private RtpContentMap initiatorRtpContentMap;
151 private RtpContentMap responderRtpContentMap;
152 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
153 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
154 private ScheduledFuture<?> ringingTimeoutFuture;
155
156 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
157 super(jingleConnectionManager, id, initiator);
158 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
159 id.account,
160 id.with.asBareJid(),
161 false,
162 false
163 );
164 this.message = new Message(
165 conversation,
166 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
167 Message.TYPE_RTP_SESSION,
168 id.sessionId
169 );
170 }
171
172 private static State reasonToState(Reason reason) {
173 switch (reason) {
174 case SUCCESS:
175 return State.TERMINATED_SUCCESS;
176 case DECLINE:
177 case BUSY:
178 return State.TERMINATED_DECLINED_OR_BUSY;
179 case CANCEL:
180 case TIMEOUT:
181 return State.TERMINATED_CANCEL_OR_TIMEOUT;
182 case SECURITY_ERROR:
183 return State.TERMINATED_SECURITY_ERROR;
184 case FAILED_APPLICATION:
185 case UNSUPPORTED_TRANSPORTS:
186 case UNSUPPORTED_APPLICATIONS:
187 return State.TERMINATED_APPLICATION_FAILURE;
188 default:
189 return State.TERMINATED_CONNECTIVITY_ERROR;
190 }
191 }
192
193 @Override
194 synchronized void deliverPacket(final JinglePacket jinglePacket) {
195 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
196 switch (jinglePacket.getAction()) {
197 case SESSION_INITIATE:
198 receiveSessionInitiate(jinglePacket);
199 break;
200 case TRANSPORT_INFO:
201 receiveTransportInfo(jinglePacket);
202 break;
203 case SESSION_ACCEPT:
204 receiveSessionAccept(jinglePacket);
205 break;
206 case SESSION_TERMINATE:
207 receiveSessionTerminate(jinglePacket);
208 break;
209 default:
210 respondOk(jinglePacket);
211 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
212 break;
213 }
214 }
215
216 @Override
217 synchronized void notifyRebound() {
218 if (isTerminated()) {
219 return;
220 }
221 webRTCWrapper.close();
222 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
223 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
224 }
225 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
226 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
227 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
228 } else {
229 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
230 finish();
231 }
232 }
233
234 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
235 respondOk(jinglePacket);
236 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
237 final State previous = this.state;
238 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
239 if (TERMINATED.contains(previous)) {
240 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
241 return;
242 }
243 webRTCWrapper.close();
244 final State target = reasonToState(wrapper.reason);
245 transitionOrThrow(target);
246 writeLogMessage(target);
247 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
248 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
249 }
250 finish();
251 }
252
253 private void receiveTransportInfo(final JinglePacket jinglePacket) {
254 //Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to INITIALIZED only after transport-info has been received
255 if (isInState(State.NULL, State.PROCEED, State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
256 respondOk(jinglePacket);
257 final RtpContentMap contentMap;
258 try {
259 contentMap = RtpContentMap.of(jinglePacket);
260 } catch (IllegalArgumentException | NullPointerException e) {
261 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
262 return;
263 }
264 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
265 if (this.state == State.SESSION_ACCEPTED) {
266 try {
267 processCandidates(candidates);
268 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
269 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
270 }
271 } else {
272 pendingIceCandidates.push(candidates);
273 }
274 } else {
275 if (isTerminated()) {
276 respondOk(jinglePacket);
277 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated");
278 } else {
279 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
280 terminateWithOutOfOrder(jinglePacket);
281 }
282 }
283 }
284
285 private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
286 final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
287 final Group originalGroup = rtpContentMap.group;
288 final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
289 if (identificationTags.size() == 0) {
290 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
291 }
292 processCandidates(identificationTags, contents);
293 }
294
295 private void processCandidates(final List<String> indices, final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
296 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
297 final String ufrag = content.getValue().transport.getAttribute("ufrag");
298 for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
299 final String sdp;
300 try {
301 sdp = candidate.toSdpAttribute(ufrag);
302 } catch (IllegalArgumentException e) {
303 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
304 continue;
305 }
306 final String sdpMid = content.getKey();
307 final int mLineIndex = indices.indexOf(sdpMid);
308 if (mLineIndex < 0) {
309 Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
310 }
311 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
312 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
313 this.webRTCWrapper.addIceCandidate(iceCandidate);
314 }
315 }
316 }
317
318 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
319 final RtpContentMap receivedContentMap;
320 try {
321 receivedContentMap = RtpContentMap.of(jinglePacket);
322 } catch (final Exception e) {
323 return Futures.immediateFailedFuture(e);
324 }
325 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
326 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
327 return Futures.transform(future, omemoVerifiedPayload -> {
328 //TODO test if an exception here triggers a correct abort
329 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
330 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received verifiable DTLS fingerprint via " + omemoVerification);
331 return omemoVerifiedPayload.getPayload();
332 }, MoreExecutors.directExecutor());
333 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
334 return Futures.immediateFailedFuture(
335 new SecurityException("DTLS fingerprint was unexpectedly not verifiable")
336 );
337 } else {
338 return Futures.immediateFuture(receivedContentMap);
339 }
340 }
341
342 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
343 if (isInitiator()) {
344 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
345 if (isTerminated()) {
346 Log.d(Config.LOGTAG, String.format(
347 "%s: got a reason to terminate with out-of-order. but already in state %s",
348 id.account.getJid().asBareJid(),
349 getState()
350 ));
351 respondWithOutOfOrder(jinglePacket);
352 } else {
353 terminateWithOutOfOrder(jinglePacket);
354 }
355 return;
356 }
357 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
358 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
359 @Override
360 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
361 receiveSessionInitiate(jinglePacket, rtpContentMap);
362 }
363
364 @Override
365 public void onFailure(@NonNull final Throwable throwable) {
366 respondOk(jinglePacket);
367 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
368 }
369 }, MoreExecutors.directExecutor());
370 }
371
372 private void receiveSessionInitiate(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
373 try {
374 contentMap.requireContentDescriptions();
375 contentMap.requireDTLSFingerprint();
376 } catch (final RuntimeException e) {
377 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e));
378 respondOk(jinglePacket);
379 sendSessionTerminate(Reason.of(e), e.getMessage());
380 return;
381 }
382 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
383 final State target;
384 if (this.state == State.PROCEED) {
385 Preconditions.checkState(
386 proposedMedia != null && proposedMedia.size() > 0,
387 "proposed media must be set when processing pre-approved session-initiate"
388 );
389 if (!this.proposedMedia.equals(contentMap.getMedia())) {
390 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
391 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
392 this.proposedMedia,
393 contentMap.getMedia()
394 ));
395 return;
396 }
397 target = State.SESSION_INITIALIZED_PRE_APPROVED;
398 } else {
399 target = State.SESSION_INITIALIZED;
400 }
401 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
402 respondOk(jinglePacket);
403
404 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
405 if (candidates.size() > 0) {
406 pendingIceCandidates.push(candidates);
407 }
408 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
409 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
410 sendSessionAccept();
411 } else {
412 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
413 startRinging();
414 }
415 } else {
416 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
417 terminateWithOutOfOrder(jinglePacket);
418 }
419 }
420
421 private void receiveSessionAccept(final JinglePacket jinglePacket) {
422 if (!isInitiator()) {
423 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
424 terminateWithOutOfOrder(jinglePacket);
425 return;
426 }
427 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
428 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
429 @Override
430 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
431 receiveSessionAccept(jinglePacket, rtpContentMap);
432 }
433
434 @Override
435 public void onFailure(@NonNull final Throwable throwable) {
436 respondOk(jinglePacket);
437 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", throwable);
438 webRTCWrapper.close();
439 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
440 }
441 }, MoreExecutors.directExecutor());
442 }
443
444 private void receiveSessionAccept(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
445 try {
446 contentMap.requireContentDescriptions();
447 contentMap.requireDTLSFingerprint();
448 } catch (final RuntimeException e) {
449 respondOk(jinglePacket);
450 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
451 webRTCWrapper.close();
452 sendSessionTerminate(Reason.of(e), e.getMessage());
453 return;
454 }
455 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
456 if (!initiatorMedia.equals(contentMap.getMedia())) {
457 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
458 "Your session-included included media %s but our session-initiate was %s",
459 this.proposedMedia,
460 contentMap.getMedia()
461 ));
462 return;
463 }
464 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
465 if (transition(State.SESSION_ACCEPTED)) {
466 respondOk(jinglePacket);
467 receiveSessionAccept(contentMap);
468 } else {
469 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
470 respondOk(jinglePacket);
471 }
472 }
473
474 private void receiveSessionAccept(final RtpContentMap contentMap) {
475 this.responderRtpContentMap = contentMap;
476 final SessionDescription sessionDescription;
477 try {
478 sessionDescription = SessionDescription.of(contentMap);
479 } catch (final IllegalArgumentException | NullPointerException e) {
480 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
481 webRTCWrapper.close();
482 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
483 return;
484 }
485 final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
486 org.webrtc.SessionDescription.Type.ANSWER,
487 sessionDescription.toString()
488 );
489 try {
490 this.webRTCWrapper.setRemoteDescription(answer).get();
491 } catch (final Exception e) {
492 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e));
493 webRTCWrapper.close();
494 sendSessionTerminate(Reason.FAILED_APPLICATION);
495 return;
496 }
497 final List<String> identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags();
498 processCandidates(identificationTags, contentMap.contents.entrySet());
499 }
500
501 private void sendSessionAccept() {
502 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
503 if (rtpContentMap == null) {
504 throw new IllegalStateException("initiator RTP Content Map has not been set");
505 }
506 final SessionDescription offer;
507 try {
508 offer = SessionDescription.of(rtpContentMap);
509 } catch (final IllegalArgumentException | NullPointerException e) {
510 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
511 webRTCWrapper.close();
512 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
513 return;
514 }
515 sendSessionAccept(rtpContentMap.getMedia(), offer);
516 }
517
518 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
519 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
520 }
521
522 private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
523 if (isTerminated()) {
524 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
525 return;
526 }
527 try {
528 setupWebRTC(media, iceServers);
529 } catch (final WebRTCWrapper.InitializationException e) {
530 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
531 webRTCWrapper.close();
532 sendSessionTerminate(Reason.FAILED_APPLICATION);
533 return;
534 }
535 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
536 org.webrtc.SessionDescription.Type.OFFER,
537 offer.toString()
538 );
539 try {
540 this.webRTCWrapper.setRemoteDescription(sdp).get();
541 addIceCandidatesFromBlackLog();
542 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
543 prepareSessionAccept(webRTCSessionDescription);
544 } catch (final Exception e) {
545 //TODO sending the error text is worthwhile as well. Especially for FailureToSet exceptions
546 failureToAcceptSession(e);
547 }
548 }
549
550 private void failureToAcceptSession(final Throwable throwable) {
551 if (isTerminated()) {
552 return;
553 }
554 Log.d(Config.LOGTAG, "unable to send session accept", Throwables.getRootCause(throwable));
555 webRTCWrapper.close();
556 sendSessionTerminate(Reason.ofThrowable(throwable));
557 }
558
559 private void addIceCandidatesFromBlackLog() {
560 while (!this.pendingIceCandidates.isEmpty()) {
561 processCandidates(this.pendingIceCandidates.poll());
562 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log");
563 }
564 }
565
566 private void prepareSessionAccept(final org.webrtc.SessionDescription webRTCSessionDescription) {
567 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
568 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
569 this.responderRtpContentMap = respondingRtpContentMap;
570 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = prepareOutgoingContentMap(respondingRtpContentMap);
571 Futures.addCallback(outgoingContentMapFuture,
572 new FutureCallback<RtpContentMap>() {
573 @Override
574 public void onSuccess(final RtpContentMap outgoingContentMap) {
575 sendSessionAccept(outgoingContentMap);
576 }
577
578 @Override
579 public void onFailure(@NonNull Throwable throwable) {
580 failureToAcceptSession(throwable);
581 }
582 },
583 MoreExecutors.directExecutor()
584 );
585 }
586
587 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
588 if (isTerminated()) {
589 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session accept was too slow. already terminated. nothing to do.");
590 return;
591 }
592 transitionOrThrow(State.SESSION_ACCEPTED);
593 final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
594 send(sessionAccept);
595 }
596
597 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(final RtpContentMap rtpContentMap) {
598 if (this.omemoVerification.hasDeviceId()) {
599 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
600 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
601 return Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
602 omemoVerification.setOrEnsureEqual(verifiedPayload);
603 return verifiedPayload.getPayload();
604 }, MoreExecutors.directExecutor());
605 } else {
606 return Futures.immediateFuture(rtpContentMap);
607 }
608 }
609
610 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
611 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
612 switch (message.getName()) {
613 case "propose":
614 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
615 break;
616 case "proceed":
617 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
618 break;
619 case "retract":
620 receiveRetract(from, serverMessageId, timestamp);
621 break;
622 case "reject":
623 receiveReject(from, serverMessageId, timestamp);
624 break;
625 case "accept":
626 receiveAccept(from, serverMessageId, timestamp);
627 break;
628 default:
629 break;
630 }
631 }
632
633 void deliverFailedProceed() {
634 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
635 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
636 webRTCWrapper.close();
637 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
638 this.finish();
639 }
640 }
641
642 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
643 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
644 if (originatedFromMyself) {
645 if (transition(State.ACCEPTED)) {
646 if (serverMsgId != null) {
647 this.message.setServerMsgId(serverMsgId);
648 }
649 this.message.setTime(timestamp);
650 this.message.setCarbon(true); //indicate that call was accepted on other device
651 this.writeLogMessageSuccess(0);
652 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
653 this.finish();
654 } else {
655 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
656 }
657 } else {
658 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
659 }
660 }
661
662 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
663 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
664 //reject from another one of my clients
665 if (originatedFromMyself) {
666 receiveRejectFromMyself(serverMsgId, timestamp);
667 } else if (isInitiator()) {
668 if (from.equals(id.with)) {
669 receiveRejectFromResponder();
670 } else {
671 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
672 }
673 } else {
674 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
675 }
676 }
677
678 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
679 if (transition(State.REJECTED)) {
680 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
681 this.finish();
682 if (serverMsgId != null) {
683 this.message.setServerMsgId(serverMsgId);
684 }
685 this.message.setTime(timestamp);
686 this.message.setCarbon(true); //indicate that call was rejected on other device
687 writeLogMessageMissed();
688 } else {
689 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
690 }
691 }
692
693 private void receiveRejectFromResponder() {
694 if (isInState(State.PROCEED)) {
695 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered");
696 closeTransitionLogFinish(State.REJECTED_RACED);
697 return;
698 }
699 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
700 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
701 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
702 return;
703 }
704 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state);
705 }
706
707 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
708 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
709 if (originatedFromMyself) {
710 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
711 } else if (transition(State.PROPOSED, () -> {
712 final Collection<RtpDescription> descriptions = Collections2.transform(
713 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
714 input -> (RtpDescription) input
715 );
716 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
717 Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
718 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
719 this.proposedMedia = Sets.newHashSet(media);
720 })) {
721 if (serverMsgId != null) {
722 this.message.setServerMsgId(serverMsgId);
723 }
724 this.message.setTime(timestamp);
725 startRinging();
726 } else {
727 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
728 }
729 }
730
731 private void startRinging() {
732 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
733 ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
734 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
735 }
736
737 private synchronized void ringingTimeout() {
738 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
739 switch (this.state) {
740 case PROPOSED:
741 message.markUnread();
742 rejectCallFromProposed();
743 break;
744 case SESSION_INITIALIZED:
745 message.markUnread();
746 rejectCallFromSessionInitiate();
747 break;
748 }
749 }
750
751 private void cancelRingingTimeout() {
752 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
753 if (future != null && !future.isCancelled()) {
754 future.cancel(false);
755 }
756 }
757
758 private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
759 final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
760 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
761 if (from.equals(id.with)) {
762 if (isInitiator()) {
763 if (transition(State.PROCEED)) {
764 if (serverMsgId != null) {
765 this.message.setServerMsgId(serverMsgId);
766 }
767 this.message.setTime(timestamp);
768 final Integer remoteDeviceId = proceed.getDeviceId();
769 if (isOmemoEnabled()) {
770 this.omemoVerification.setDeviceId(remoteDeviceId);
771 } else {
772 if (remoteDeviceId != null) {
773 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
774 }
775 this.omemoVerification.setDeviceId(null);
776 }
777 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
778 } else {
779 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
780 }
781 } else {
782 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
783 }
784 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
785 if (transition(State.ACCEPTED)) {
786 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
787 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
788 this.finish();
789 }
790 } else {
791 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
792 }
793 }
794
795 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
796 if (from.equals(id.with)) {
797 final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
798 if (transition(target)) {
799 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
800 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
801 if (serverMsgId != null) {
802 this.message.setServerMsgId(serverMsgId);
803 }
804 this.message.setTime(timestamp);
805 if (target == State.RETRACTED) {
806 this.message.markUnread();
807 }
808 writeLogMessageMissed();
809 finish();
810 } else {
811 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
812 }
813 } else {
814 //TODO parse retract from self
815 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
816 }
817 }
818
819 public void sendSessionInitiate() {
820 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
821 }
822
823 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
824 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
825 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
826 }
827
828 private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
829 if (isTerminated()) {
830 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
831 return;
832 }
833 try {
834 setupWebRTC(media, iceServers);
835 } catch (final WebRTCWrapper.InitializationException e) {
836 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
837 webRTCWrapper.close();
838 sendRetract(Reason.ofThrowable(e));
839 return;
840 }
841 try {
842 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.setLocalDescription().get();
843 prepareSessionInitiate(webRTCSessionDescription, targetState);
844 } catch (final Exception e) {
845 //TODO sending the error text is worthwhile as well. Especially for FailureToSet exceptions
846 failureToInitiateSession(e, targetState);
847 }
848 }
849
850 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
851 if (isTerminated()) {
852 return;
853 }
854 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(throwable));
855 webRTCWrapper.close();
856 final Reason reason = Reason.ofThrowable(throwable);
857 if (isInState(targetState)) {
858 sendSessionTerminate(reason);
859 } else {
860 sendRetract(reason);
861 }
862 }
863
864 private void sendRetract(final Reason reason) {
865 //TODO embed reason into retract
866 sendJingleMessage("retract", id.with.asBareJid());
867 transitionOrThrow(reasonToState(reason));
868 this.finish();
869 }
870
871 private void prepareSessionInitiate(final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
872 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
873 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
874 this.initiatorRtpContentMap = rtpContentMap;
875 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = encryptSessionInitiate(rtpContentMap);
876 Futures.addCallback(outgoingContentMapFuture, new FutureCallback<RtpContentMap>() {
877 @Override
878 public void onSuccess(final RtpContentMap outgoingContentMap) {
879 sendSessionInitiate(outgoingContentMap, targetState);
880 }
881
882 @Override
883 public void onFailure(@NonNull final Throwable throwable) {
884 failureToInitiateSession(throwable, targetState);
885 }
886 }, MoreExecutors.directExecutor());
887 }
888
889 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
890 if (isTerminated()) {
891 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session was too slow. already terminated. nothing to do.");
892 return;
893 }
894 this.transitionOrThrow(targetState);
895 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
896 send(sessionInitiate);
897 }
898
899 private ListenableFuture<RtpContentMap> encryptSessionInitiate(final RtpContentMap rtpContentMap) {
900 if (this.omemoVerification.hasDeviceId()) {
901 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
902 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
903 final ListenableFuture<RtpContentMap> future = Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
904 omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint());
905 return verifiedPayload.getPayload();
906 }, MoreExecutors.directExecutor());
907 if (Config.REQUIRE_RTP_VERIFICATION) {
908 return future;
909 }
910 return Futures.catching(
911 future,
912 CryptoFailedException.class,
913 e -> {
914 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e);
915 return rtpContentMap;
916 },
917 MoreExecutors.directExecutor()
918 );
919 } else {
920 return Futures.immediateFuture(rtpContentMap);
921 }
922 }
923
924 private void sendSessionTerminate(final Reason reason) {
925 sendSessionTerminate(reason, null);
926 }
927
928 private void sendSessionTerminate(final Reason reason, final String text) {
929 final State previous = this.state;
930 final State target = reasonToState(reason);
931 transitionOrThrow(target);
932 if (previous != State.NULL) {
933 writeLogMessage(target);
934 }
935 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
936 jinglePacket.setReason(reason, text);
937 Log.d(Config.LOGTAG, jinglePacket.toString());
938 send(jinglePacket);
939 finish();
940 }
941
942 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
943 final RtpContentMap transportInfo;
944 try {
945 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
946 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
947 } catch (final Exception e) {
948 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
949 return;
950 }
951 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
952 send(jinglePacket);
953 }
954
955 private void send(final JinglePacket jinglePacket) {
956 jinglePacket.setTo(id.with);
957 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
958 }
959
960 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
961 if (response.getType() == IqPacket.TYPE.ERROR) {
962 final String errorCondition = response.getErrorCondition();
963 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
964 if (isTerminated()) {
965 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
966 return;
967 }
968 this.webRTCWrapper.close();
969 final State target;
970 if (Arrays.asList(
971 "service-unavailable",
972 "recipient-unavailable",
973 "remote-server-not-found",
974 "remote-server-timeout"
975 ).contains(errorCondition)) {
976 target = State.TERMINATED_CONNECTIVITY_ERROR;
977 } else {
978 target = State.TERMINATED_APPLICATION_FAILURE;
979 }
980 transitionOrThrow(target);
981 this.finish();
982 } else if (response.getType() == IqPacket.TYPE.TIMEOUT) {
983 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
984 if (isTerminated()) {
985 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
986 return;
987 }
988 this.webRTCWrapper.close();
989 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
990 this.finish();
991 }
992 }
993
994 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
995 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
996 this.webRTCWrapper.close();
997 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
998 respondWithOutOfOrder(jinglePacket);
999 this.finish();
1000 }
1001
1002 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1003 jingleConnectionManager.respondWithJingleError(id.account, jinglePacket, "out-of-order", "unexpected-request", "wait");
1004 }
1005
1006 private void respondOk(final JinglePacket jinglePacket) {
1007 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1008 }
1009
1010 public void throwStateTransitionException() {
1011 final StateTransitionException exception = this.stateTransitionException;
1012 if (exception != null) {
1013 throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception);
1014 }
1015 }
1016
1017 public RtpEndUserState getEndUserState() {
1018 switch (this.state) {
1019 case NULL:
1020 case PROPOSED:
1021 case SESSION_INITIALIZED:
1022 if (isInitiator()) {
1023 return RtpEndUserState.RINGING;
1024 } else {
1025 return RtpEndUserState.INCOMING_CALL;
1026 }
1027 case PROCEED:
1028 if (isInitiator()) {
1029 return RtpEndUserState.RINGING;
1030 } else {
1031 return RtpEndUserState.ACCEPTING_CALL;
1032 }
1033 case SESSION_INITIALIZED_PRE_APPROVED:
1034 if (isInitiator()) {
1035 return RtpEndUserState.RINGING;
1036 } else {
1037 return RtpEndUserState.CONNECTING;
1038 }
1039 case SESSION_ACCEPTED:
1040 return getPeerConnectionStateAsEndUserState();
1041 case REJECTED:
1042 case REJECTED_RACED:
1043 case TERMINATED_DECLINED_OR_BUSY:
1044 if (isInitiator()) {
1045 return RtpEndUserState.DECLINED_OR_BUSY;
1046 } else {
1047 return RtpEndUserState.ENDED;
1048 }
1049 case TERMINATED_SUCCESS:
1050 case ACCEPTED:
1051 case RETRACTED:
1052 case TERMINATED_CANCEL_OR_TIMEOUT:
1053 return RtpEndUserState.ENDED;
1054 case RETRACTED_RACED:
1055 if (isInitiator()) {
1056 return RtpEndUserState.ENDED;
1057 } else {
1058 return RtpEndUserState.RETRACTED;
1059 }
1060 case TERMINATED_CONNECTIVITY_ERROR:
1061 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1062 case TERMINATED_APPLICATION_FAILURE:
1063 return RtpEndUserState.APPLICATION_ERROR;
1064 case TERMINATED_SECURITY_ERROR:
1065 return RtpEndUserState.SECURITY_ERROR;
1066 }
1067 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
1068 }
1069
1070
1071 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
1072 final PeerConnection.PeerConnectionState state;
1073 try {
1074 state = webRTCWrapper.getState();
1075 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
1076 //We usually close the WebRTCWrapper *before* transitioning so we might still
1077 //be in SESSION_ACCEPTED even though the peerConnection has been torn down
1078 return RtpEndUserState.ENDING_CALL;
1079 }
1080 switch (state) {
1081 case CONNECTED:
1082 return RtpEndUserState.CONNECTED;
1083 case NEW:
1084 case CONNECTING:
1085 return RtpEndUserState.CONNECTING;
1086 case CLOSED:
1087 return RtpEndUserState.ENDING_CALL;
1088 default:
1089 return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.RECONNECTING;
1090 }
1091 }
1092
1093 public Set<Media> getMedia() {
1094 final State current = getState();
1095 if (current == State.NULL) {
1096 if (isInitiator()) {
1097 return Preconditions.checkNotNull(
1098 this.proposedMedia,
1099 "RTP connection has not been initialized properly"
1100 );
1101 }
1102 throw new IllegalStateException("RTP connection has not been initialized yet");
1103 }
1104 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
1105 return Preconditions.checkNotNull(
1106 this.proposedMedia,
1107 "RTP connection has not been initialized properly"
1108 );
1109 }
1110 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
1111 if (initiatorContentMap != null) {
1112 return initiatorContentMap.getMedia();
1113 } else if (isTerminated()) {
1114 return Collections.emptySet(); //we might fail before we ever got a chance to set media
1115 } else {
1116 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
1117 }
1118 }
1119
1120
1121 public boolean isVerified() {
1122 final String fingerprint = this.omemoVerification.getFingerprint();
1123 if (fingerprint == null) {
1124 return false;
1125 }
1126 final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint);
1127 return status != null && status.isVerified();
1128 }
1129
1130 public synchronized void acceptCall() {
1131 switch (this.state) {
1132 case PROPOSED:
1133 cancelRingingTimeout();
1134 acceptCallFromProposed();
1135 break;
1136 case SESSION_INITIALIZED:
1137 cancelRingingTimeout();
1138 acceptCallFromSessionInitialized();
1139 break;
1140 case ACCEPTED:
1141 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind");
1142 break;
1143 case PROCEED:
1144 case SESSION_ACCEPTED:
1145 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI");
1146 break;
1147 default:
1148 throw new IllegalStateException("Can not accept call from " + this.state);
1149 }
1150 }
1151
1152
1153 public void notifyPhoneCall() {
1154 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1155 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1156 rejectCall();
1157 } else {
1158 endCall();
1159 }
1160 }
1161
1162 public synchronized void rejectCall() {
1163 if (isTerminated()) {
1164 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do");
1165 return;
1166 }
1167 switch (this.state) {
1168 case PROPOSED:
1169 rejectCallFromProposed();
1170 break;
1171 case SESSION_INITIALIZED:
1172 rejectCallFromSessionInitiate();
1173 break;
1174 default:
1175 throw new IllegalStateException("Can not reject call from " + this.state);
1176 }
1177 }
1178
1179 public synchronized void endCall() {
1180 if (isTerminated()) {
1181 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
1182 return;
1183 }
1184 if (isInState(State.PROPOSED) && !isInitiator()) {
1185 rejectCallFromProposed();
1186 return;
1187 }
1188 if (isInState(State.PROCEED)) {
1189 if (isInitiator()) {
1190 retractFromProceed();
1191 } else {
1192 rejectCallFromProceed();
1193 }
1194 return;
1195 }
1196 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1197 this.webRTCWrapper.close();
1198 sendSessionTerminate(Reason.CANCEL);
1199 return;
1200 }
1201 if (isInState(State.SESSION_INITIALIZED)) {
1202 rejectCallFromSessionInitiate();
1203 return;
1204 }
1205 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1206 this.webRTCWrapper.close();
1207 sendSessionTerminate(Reason.SUCCESS);
1208 return;
1209 }
1210 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
1211 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
1212 return;
1213 }
1214 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1215 }
1216
1217 private void retractFromProceed() {
1218 Log.d(Config.LOGTAG, "retract from proceed");
1219 this.sendJingleMessage("retract");
1220 closeTransitionLogFinish(State.RETRACTED_RACED);
1221 }
1222
1223 private void closeTransitionLogFinish(final State state) {
1224 this.webRTCWrapper.close();
1225 transitionOrThrow(state);
1226 writeLogMessage(state);
1227 finish();
1228 }
1229
1230 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
1231 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1232 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1233 if (media.contains(Media.VIDEO)) {
1234 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1235 } else {
1236 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1237 }
1238 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1239 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1240 }
1241
1242 private void acceptCallFromProposed() {
1243 transitionOrThrow(State.PROCEED);
1244 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1245 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1246 this.sendJingleMessage("proceed");
1247 }
1248
1249 private void rejectCallFromProposed() {
1250 transitionOrThrow(State.REJECTED);
1251 writeLogMessageMissed();
1252 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1253 this.sendJingleMessage("reject");
1254 finish();
1255 }
1256
1257 private void rejectCallFromProceed() {
1258 this.sendJingleMessage("reject");
1259 closeTransitionLogFinish(State.REJECTED_RACED);
1260 }
1261
1262 private void rejectCallFromSessionInitiate() {
1263 webRTCWrapper.close();
1264 sendSessionTerminate(Reason.DECLINE);
1265 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1266 }
1267
1268 private void sendJingleMessage(final String action) {
1269 sendJingleMessage(action, id.with);
1270 }
1271
1272 private void sendJingleMessage(final String action, final Jid to) {
1273 final MessagePacket messagePacket = new MessagePacket();
1274 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
1275 messagePacket.setTo(to);
1276 final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
1277 if ("proceed".equals(action)) {
1278 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1279 if (isOmemoEnabled()) {
1280 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1281 final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1282 device.setAttribute("id", deviceId);
1283 }
1284 }
1285 messagePacket.addChild("store", "urn:xmpp:hints");
1286 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1287 }
1288
1289 private boolean isOmemoEnabled() {
1290 final Conversational conversational = message.getConversation();
1291 if (conversational instanceof Conversation) {
1292 return ((Conversation) conversational).getNextEncryption() == Message.ENCRYPTION_AXOLOTL;
1293 }
1294 return false;
1295 }
1296
1297 private void acceptCallFromSessionInitialized() {
1298 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1299 sendSessionAccept();
1300 }
1301
1302 private synchronized boolean isInState(State... state) {
1303 return Arrays.asList(state).contains(this.state);
1304 }
1305
1306 private boolean transition(final State target) {
1307 return transition(target, null);
1308 }
1309
1310 private synchronized boolean transition(final State target, final Runnable runnable) {
1311 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1312 if (validTransitions != null && validTransitions.contains(target)) {
1313 this.state = target;
1314 this.stateTransitionException = new StateTransitionException(target);
1315 if (runnable != null) {
1316 runnable.run();
1317 }
1318 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1319 updateEndUserState();
1320 updateOngoingCallNotification();
1321 return true;
1322 } else {
1323 return false;
1324 }
1325 }
1326
1327 void transitionOrThrow(final State target) {
1328 if (!transition(target)) {
1329 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
1330 }
1331 }
1332
1333 @Override
1334 public void onIceCandidate(final IceCandidate iceCandidate) {
1335 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
1336 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
1337 sendTransportInfo(iceCandidate.sdpMid, candidate);
1338 }
1339
1340 @Override
1341 public void onConnectionChange(final PeerConnection.PeerConnectionState oldState, final PeerConnection.PeerConnectionState newState) {
1342 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed: " + oldState + "->" + newState);
1343 this.stateHistory.add(newState);
1344 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
1345 this.sessionDuration.start();
1346 } else if (this.sessionDuration.isRunning()) {
1347 this.sessionDuration.stop();
1348 }
1349
1350 final boolean neverConnected = !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
1351 final boolean failedOrDisconnected = Arrays.asList(PeerConnection.PeerConnectionState.FAILED, PeerConnection.PeerConnectionState.DISCONNECTED).contains(newState);
1352
1353
1354 if (neverConnected && failedOrDisconnected) {
1355 if (isTerminated()) {
1356 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
1357 return;
1358 }
1359 new Thread(this::closeWebRTCSessionAfterFailedConnection).start();
1360 } else if (newState == PeerConnection.PeerConnectionState.FAILED) {
1361 Log.d(Config.LOGTAG, "attempting to restart ICE");
1362 webRTCWrapper.restartIce();
1363 }
1364 updateEndUserState();
1365 }
1366
1367 @Override
1368 public void onRenegotiationNeeded() {
1369 Log.d(Config.LOGTAG, "onRenegotiationNeeded()");
1370 }
1371
1372 private void closeWebRTCSessionAfterFailedConnection() {
1373 this.webRTCWrapper.close();
1374 synchronized (this) {
1375 if (isTerminated()) {
1376 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did");
1377 return;
1378 }
1379 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
1380 }
1381 }
1382
1383 public boolean zeroDuration() {
1384 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
1385 }
1386
1387 public long getCallDuration() {
1388 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
1389 }
1390
1391 public AppRTCAudioManager getAudioManager() {
1392 return webRTCWrapper.getAudioManager();
1393 }
1394
1395 public boolean isMicrophoneEnabled() {
1396 return webRTCWrapper.isMicrophoneEnabled();
1397 }
1398
1399 public boolean setMicrophoneEnabled(final boolean enabled) {
1400 return webRTCWrapper.setMicrophoneEnabled(enabled);
1401 }
1402
1403 public boolean isVideoEnabled() {
1404 return webRTCWrapper.isVideoEnabled();
1405 }
1406
1407 public void setVideoEnabled(final boolean enabled) {
1408 webRTCWrapper.setVideoEnabled(enabled);
1409 }
1410
1411 public boolean isCameraSwitchable() {
1412 return webRTCWrapper.isCameraSwitchable();
1413 }
1414
1415 public boolean isFrontCamera() {
1416 return webRTCWrapper.isFrontCamera();
1417 }
1418
1419 public ListenableFuture<Boolean> switchCamera() {
1420 return webRTCWrapper.switchCamera();
1421 }
1422
1423 @Override
1424 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1425 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
1426 }
1427
1428 private void updateEndUserState() {
1429 final RtpEndUserState endUserState = getEndUserState();
1430 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
1431 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1432 }
1433
1434 private void updateOngoingCallNotification() {
1435 if (STATES_SHOWING_ONGOING_CALL.contains(this.state)) {
1436 xmppConnectionService.setOngoingCall(id, getMedia());
1437 } else {
1438 xmppConnectionService.removeOngoingCall();
1439 }
1440 }
1441
1442 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
1443 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
1444 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
1445 request.setTo(id.account.getDomain());
1446 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1447 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
1448 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
1449 if (response.getType() == IqPacket.TYPE.RESULT) {
1450 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1451 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
1452 for (final Element child : children) {
1453 if ("service".equals(child.getName())) {
1454 final String type = child.getAttribute("type");
1455 final String host = child.getAttribute("host");
1456 final String sport = child.getAttribute("port");
1457 final Integer port = sport == null ? null : Ints.tryParse(sport);
1458 final String transport = child.getAttribute("transport");
1459 final String username = child.getAttribute("username");
1460 final String password = child.getAttribute("password");
1461 if (Strings.isNullOrEmpty(host) || port == null) {
1462 continue;
1463 }
1464 if (port < 0 || port > 65535) {
1465 continue;
1466 }
1467 if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) {
1468 if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) {
1469 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services");
1470 continue;
1471 }
1472 final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer
1473 .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport));
1474 iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK);
1475 if (username != null && password != null) {
1476 iceServerBuilder.setUsername(username);
1477 iceServerBuilder.setPassword(password);
1478 } else if (Arrays.asList("turn", "turns").contains(type)) {
1479 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
1480 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
1481 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
1482 continue;
1483 }
1484 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
1485 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
1486 listBuilder.add(iceServer);
1487 }
1488 }
1489 }
1490 }
1491 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
1492 if (iceServers.size() == 0) {
1493 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
1494 }
1495 onIceServersDiscovered.onIceServersDiscovered(iceServers);
1496 });
1497 } else {
1498 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
1499 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
1500 }
1501 }
1502
1503 private void finish() {
1504 if (isTerminated()) {
1505 this.cancelRingingTimeout();
1506 this.webRTCWrapper.verifyClosed();
1507 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
1508 this.jingleConnectionManager.finishConnectionOrThrow(this);
1509 } else {
1510 throw new IllegalStateException(String.format("Unable to call finish from %s", this.state));
1511 }
1512 }
1513
1514 private void writeLogMessage(final State state) {
1515 final long duration = getCallDuration();
1516 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
1517 writeLogMessageSuccess(duration);
1518 } else {
1519 writeLogMessageMissed();
1520 }
1521 }
1522
1523 private void writeLogMessageSuccess(final long duration) {
1524 this.message.setBody(new RtpSessionStatus(true, duration).toString());
1525 this.writeMessage();
1526 }
1527
1528 private void writeLogMessageMissed() {
1529 this.message.setBody(new RtpSessionStatus(false, 0).toString());
1530 this.writeMessage();
1531 }
1532
1533 private void writeMessage() {
1534 final Conversational conversational = message.getConversation();
1535 if (conversational instanceof Conversation) {
1536 ((Conversation) conversational).add(this.message);
1537 xmppConnectionService.createMessageAsync(message);
1538 xmppConnectionService.updateConversationUi();
1539 } else {
1540 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1541 }
1542 }
1543
1544 public State getState() {
1545 return this.state;
1546 }
1547
1548 boolean isTerminated() {
1549 return TERMINATED.contains(this.state);
1550 }
1551
1552 public Optional<VideoTrack> getLocalVideoTrack() {
1553 return webRTCWrapper.getLocalVideoTrack();
1554 }
1555
1556 public Optional<VideoTrack> getRemoteVideoTrack() {
1557 return webRTCWrapper.getRemoteVideoTrack();
1558 }
1559
1560
1561 public EglBase.Context getEglBaseContext() {
1562 return webRTCWrapper.getEglBaseContext();
1563 }
1564
1565 void setProposedMedia(final Set<Media> media) {
1566 this.proposedMedia = media;
1567 }
1568
1569 public void fireStateUpdate() {
1570 final RtpEndUserState endUserState = getEndUserState();
1571 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1572 }
1573
1574 private interface OnIceServersDiscovered {
1575 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1576 }
1577
1578 private static class StateTransitionException extends Exception {
1579 private final State state;
1580
1581 private StateTransitionException(final State state) {
1582 this.state = state;
1583 }
1584 }
1585}