1package eu.siacs.conversations.xmpp.jingle;
2
3import android.os.SystemClock;
4import android.util.Log;
5
6import androidx.annotation.NonNull;
7import androidx.annotation.Nullable;
8
9import com.google.common.base.Optional;
10import com.google.common.base.Preconditions;
11import com.google.common.base.Strings;
12import com.google.common.base.Throwables;
13import com.google.common.collect.Collections2;
14import com.google.common.collect.ImmutableList;
15import com.google.common.collect.ImmutableMap;
16import com.google.common.collect.Sets;
17import com.google.common.primitives.Ints;
18import com.google.common.util.concurrent.FutureCallback;
19import com.google.common.util.concurrent.Futures;
20import com.google.common.util.concurrent.ListenableFuture;
21import com.google.common.util.concurrent.MoreExecutors;
22
23import org.webrtc.DtmfSender;
24import org.webrtc.EglBase;
25import org.webrtc.IceCandidate;
26import org.webrtc.PeerConnection;
27import org.webrtc.VideoTrack;
28
29import java.util.ArrayDeque;
30import java.util.Arrays;
31import java.util.Collection;
32import java.util.Collections;
33import java.util.List;
34import java.util.Map;
35import java.util.Set;
36import java.util.concurrent.ScheduledFuture;
37import java.util.concurrent.TimeUnit;
38
39import eu.siacs.conversations.Config;
40import eu.siacs.conversations.crypto.axolotl.AxolotlService;
41import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
42import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
43import eu.siacs.conversations.entities.Account;
44import eu.siacs.conversations.entities.Conversation;
45import eu.siacs.conversations.entities.Conversational;
46import eu.siacs.conversations.entities.Message;
47import eu.siacs.conversations.entities.RtpSessionStatus;
48import eu.siacs.conversations.services.AppRTCAudioManager;
49import eu.siacs.conversations.utils.IP;
50import eu.siacs.conversations.xml.Element;
51import eu.siacs.conversations.xml.Namespace;
52import eu.siacs.conversations.xmpp.Jid;
53import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
54import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
55import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
56import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
57import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
58import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
59import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
60import eu.siacs.conversations.xmpp.stanzas.IqPacket;
61import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
62
63public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
64
65 public static final List<State> STATES_SHOWING_ONGOING_CALL = Arrays.asList(
66 State.PROCEED,
67 State.SESSION_INITIALIZED_PRE_APPROVED,
68 State.SESSION_ACCEPTED
69 );
70 private static final long BUSY_TIME_OUT = 30;
71 private static final List<State> TERMINATED = Arrays.asList(
72 State.ACCEPTED,
73 State.REJECTED,
74 State.REJECTED_RACED,
75 State.RETRACTED,
76 State.RETRACTED_RACED,
77 State.TERMINATED_SUCCESS,
78 State.TERMINATED_DECLINED_OR_BUSY,
79 State.TERMINATED_CONNECTIVITY_ERROR,
80 State.TERMINATED_CANCEL_OR_TIMEOUT,
81 State.TERMINATED_APPLICATION_FAILURE,
82 State.TERMINATED_SECURITY_ERROR
83 );
84
85 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
86
87 static {
88 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
89 transitionBuilder.put(State.NULL, ImmutableList.of(
90 State.PROPOSED,
91 State.SESSION_INITIALIZED,
92 State.TERMINATED_APPLICATION_FAILURE,
93 State.TERMINATED_SECURITY_ERROR
94 ));
95 transitionBuilder.put(State.PROPOSED, ImmutableList.of(
96 State.ACCEPTED,
97 State.PROCEED,
98 State.REJECTED,
99 State.RETRACTED,
100 State.TERMINATED_APPLICATION_FAILURE,
101 State.TERMINATED_SECURITY_ERROR,
102 State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds
103 ));
104 transitionBuilder.put(State.PROCEED, ImmutableList.of(
105 State.REJECTED_RACED,
106 State.RETRACTED_RACED,
107 State.SESSION_INITIALIZED_PRE_APPROVED,
108 State.TERMINATED_SUCCESS,
109 State.TERMINATED_APPLICATION_FAILURE,
110 State.TERMINATED_SECURITY_ERROR,
111 State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message
112 ));
113 transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(
114 State.SESSION_ACCEPTED,
115 State.TERMINATED_SUCCESS,
116 State.TERMINATED_DECLINED_OR_BUSY,
117 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
118 State.TERMINATED_CANCEL_OR_TIMEOUT,
119 State.TERMINATED_APPLICATION_FAILURE,
120 State.TERMINATED_SECURITY_ERROR
121 ));
122 transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of(
123 State.SESSION_ACCEPTED,
124 State.TERMINATED_SUCCESS,
125 State.TERMINATED_DECLINED_OR_BUSY,
126 State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts
127 State.TERMINATED_CANCEL_OR_TIMEOUT,
128 State.TERMINATED_APPLICATION_FAILURE,
129 State.TERMINATED_SECURITY_ERROR
130 ));
131 transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of(
132 State.TERMINATED_SUCCESS,
133 State.TERMINATED_DECLINED_OR_BUSY,
134 State.TERMINATED_CONNECTIVITY_ERROR,
135 State.TERMINATED_CANCEL_OR_TIMEOUT,
136 State.TERMINATED_APPLICATION_FAILURE,
137 State.TERMINATED_SECURITY_ERROR
138 ));
139 VALID_TRANSITIONS = transitionBuilder.build();
140 }
141
142 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
143 private final ArrayDeque<Set<Map.Entry<String, RtpContentMap.DescriptionTransport>>> pendingIceCandidates = new ArrayDeque<>();
144 private final OmemoVerification omemoVerification = new OmemoVerification();
145 private final Message message;
146 private State state = State.NULL;
147 private StateTransitionException stateTransitionException;
148 private Set<Media> proposedMedia;
149 private RtpContentMap initiatorRtpContentMap;
150 private RtpContentMap responderRtpContentMap;
151 private long rtpConnectionStarted = 0; //time of 'connected'
152 private long rtpConnectionEnded = 0;
153 private ScheduledFuture<?> ringingTimeoutFuture;
154
155 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
156 super(jingleConnectionManager, id, initiator);
157 final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(
158 id.account,
159 id.with.asBareJid(),
160 false,
161 false
162 );
163 this.message = new Message(
164 conversation,
165 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
166 Message.TYPE_RTP_SESSION,
167 id.sessionId
168 );
169 }
170
171 private static State reasonToState(Reason reason) {
172 switch (reason) {
173 case SUCCESS:
174 return State.TERMINATED_SUCCESS;
175 case DECLINE:
176 case BUSY:
177 return State.TERMINATED_DECLINED_OR_BUSY;
178 case CANCEL:
179 case TIMEOUT:
180 return State.TERMINATED_CANCEL_OR_TIMEOUT;
181 case SECURITY_ERROR:
182 return State.TERMINATED_SECURITY_ERROR;
183 case FAILED_APPLICATION:
184 case UNSUPPORTED_TRANSPORTS:
185 case UNSUPPORTED_APPLICATIONS:
186 return State.TERMINATED_APPLICATION_FAILURE;
187 default:
188 return State.TERMINATED_CONNECTIVITY_ERROR;
189 }
190 }
191
192 @Override
193 synchronized void deliverPacket(final JinglePacket jinglePacket) {
194 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
195 switch (jinglePacket.getAction()) {
196 case SESSION_INITIATE:
197 receiveSessionInitiate(jinglePacket);
198 break;
199 case TRANSPORT_INFO:
200 receiveTransportInfo(jinglePacket);
201 break;
202 case SESSION_ACCEPT:
203 receiveSessionAccept(jinglePacket);
204 break;
205 case SESSION_TERMINATE:
206 receiveSessionTerminate(jinglePacket);
207 break;
208 default:
209 respondOk(jinglePacket);
210 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
211 break;
212 }
213 }
214
215 @Override
216 synchronized void notifyRebound() {
217 if (isTerminated()) {
218 return;
219 }
220 webRTCWrapper.close();
221 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
222 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
223 }
224 if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
225 //we might have already changed resources (full jid) at this point; so this might not even reach the other party
226 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
227 } else {
228 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
229 finish();
230 }
231 }
232
233 //FIXME: possible implementation
234 public boolean applyDtmfTone(String tone) {
235 return webRTCWrapper.applyDtmfTone(tone);
236 }
237
238
239 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
240 respondOk(jinglePacket);
241 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
242 final State previous = this.state;
243 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous);
244 if (TERMINATED.contains(previous)) {
245 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous);
246 return;
247 }
248 webRTCWrapper.close();
249 final State target = reasonToState(wrapper.reason);
250 transitionOrThrow(target);
251 writeLogMessage(target);
252 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
253 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
254 }
255 finish();
256 }
257
258 private void receiveTransportInfo(final JinglePacket jinglePacket) {
259 //Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to INITIALIZED only after transport-info has been received
260 if (isInState(State.NULL, State.PROCEED, State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
261 respondOk(jinglePacket);
262 final RtpContentMap contentMap;
263 try {
264 contentMap = RtpContentMap.of(jinglePacket);
265 } catch (IllegalArgumentException | NullPointerException e) {
266 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
267 return;
268 }
269 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
270 if (this.state == State.SESSION_ACCEPTED) {
271 try {
272 processCandidates(candidates);
273 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
274 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
275 }
276 } else {
277 pendingIceCandidates.push(candidates);
278 }
279 } else {
280 if (isTerminated()) {
281 respondOk(jinglePacket);
282 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated");
283 } else {
284 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
285 terminateWithOutOfOrder(jinglePacket);
286 }
287 }
288 }
289
290 private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
291 final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
292 final Group originalGroup = rtpContentMap.group;
293 final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
294 if (identificationTags.size() == 0) {
295 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
296 }
297 processCandidates(identificationTags, contents);
298 }
299
300 private void processCandidates(final List<String> indices, final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
301 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
302 final String ufrag = content.getValue().transport.getAttribute("ufrag");
303 for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
304 final String sdp;
305 try {
306 sdp = candidate.toSdpAttribute(ufrag);
307 } catch (IllegalArgumentException e) {
308 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
309 continue;
310 }
311 final String sdpMid = content.getKey();
312 final int mLineIndex = indices.indexOf(sdpMid);
313 if (mLineIndex < 0) {
314 Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
315 }
316 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
317 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
318 this.webRTCWrapper.addIceCandidate(iceCandidate);
319 }
320 }
321 }
322
323 private ListenableFuture<RtpContentMap> receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
324 final RtpContentMap receivedContentMap = RtpContentMap.of(jinglePacket);
325 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
326 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
327 return Futures.transform(future, omemoVerifiedPayload -> {
328 //TODO test if an exception here triggers a correct abort
329 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
330 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received verifiable DTLS fingerprint via " + omemoVerification);
331 return omemoVerifiedPayload.getPayload();
332 }, MoreExecutors.directExecutor());
333 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
334 return Futures.immediateFailedFuture(
335 new SecurityException("DTLS fingerprint was unexpectedly not verifiable")
336 );
337 } else {
338 return Futures.immediateFuture(receivedContentMap);
339 }
340 }
341
342 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
343 if (isInitiator()) {
344 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
345 if (isTerminated()) {
346 Log.d(Config.LOGTAG, String.format(
347 "%s: got a reason to terminate with out-of-order. but already in state %s",
348 id.account.getJid().asBareJid(),
349 getState()
350 ));
351 respondWithOutOfOrder(jinglePacket);
352 } else {
353 terminateWithOutOfOrder(jinglePacket);
354 }
355 return;
356 }
357 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
358 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
359 @Override
360 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
361 receiveSessionInitiate(jinglePacket, rtpContentMap);
362 }
363
364 @Override
365 public void onFailure(@NonNull final Throwable throwable) {
366 respondOk(jinglePacket);
367 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
368 }
369 }, MoreExecutors.directExecutor());
370 }
371
372 private void receiveSessionInitiate(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
373 try {
374 contentMap.requireContentDescriptions();
375 contentMap.requireDTLSFingerprint();
376 } catch (final RuntimeException e) {
377 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e));
378 respondOk(jinglePacket);
379 sendSessionTerminate(Reason.of(e), e.getMessage());
380 return;
381 }
382 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
383 final State target;
384 if (this.state == State.PROCEED) {
385 Preconditions.checkState(
386 proposedMedia != null && proposedMedia.size() > 0,
387 "proposed media must be set when processing pre-approved session-initiate"
388 );
389 if (!this.proposedMedia.equals(contentMap.getMedia())) {
390 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
391 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
392 this.proposedMedia,
393 contentMap.getMedia()
394 ));
395 return;
396 }
397 target = State.SESSION_INITIALIZED_PRE_APPROVED;
398 } else {
399 target = State.SESSION_INITIALIZED;
400 }
401 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
402 respondOk(jinglePacket);
403
404 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
405 if (candidates.size() > 0) {
406 pendingIceCandidates.push(candidates);
407 }
408 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
409 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
410 sendSessionAccept();
411 } else {
412 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing");
413 startRinging();
414 }
415 } else {
416 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
417 terminateWithOutOfOrder(jinglePacket);
418 }
419 }
420
421 private void receiveSessionAccept(final JinglePacket jinglePacket) {
422 if (!isInitiator()) {
423 Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid()));
424 terminateWithOutOfOrder(jinglePacket);
425 return;
426 }
427 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
428 Futures.addCallback(future, new FutureCallback<RtpContentMap>() {
429 @Override
430 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
431 receiveSessionAccept(jinglePacket, rtpContentMap);
432 }
433
434 @Override
435 public void onFailure(@NonNull final Throwable throwable) {
436 respondOk(jinglePacket);
437 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", throwable);
438 webRTCWrapper.close();
439 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
440 }
441 }, MoreExecutors.directExecutor());
442 }
443
444 private void receiveSessionAccept(final JinglePacket jinglePacket, final RtpContentMap contentMap) {
445 try {
446 contentMap.requireContentDescriptions();
447 contentMap.requireDTLSFingerprint();
448 } catch (final RuntimeException e) {
449 respondOk(jinglePacket);
450 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e);
451 webRTCWrapper.close();
452 sendSessionTerminate(Reason.of(e), e.getMessage());
453 return;
454 }
455 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
456 if (!initiatorMedia.equals(contentMap.getMedia())) {
457 sendSessionTerminate(Reason.SECURITY_ERROR, String.format(
458 "Your session-included included media %s but our session-initiate was %s",
459 this.proposedMedia,
460 contentMap.getMedia()
461 ));
462 return;
463 }
464 Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents");
465 if (transition(State.SESSION_ACCEPTED)) {
466 respondOk(jinglePacket);
467 receiveSessionAccept(contentMap);
468 } else {
469 Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state));
470 respondOk(jinglePacket);
471 }
472 }
473
474 private void receiveSessionAccept(final RtpContentMap contentMap) {
475 this.responderRtpContentMap = contentMap;
476 final SessionDescription sessionDescription;
477 try {
478 sessionDescription = SessionDescription.of(contentMap);
479 } catch (final IllegalArgumentException | NullPointerException e) {
480 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e);
481 webRTCWrapper.close();
482 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
483 return;
484 }
485 final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription(
486 org.webrtc.SessionDescription.Type.ANSWER,
487 sessionDescription.toString()
488 );
489 try {
490 this.webRTCWrapper.setRemoteDescription(answer).get();
491 } catch (final Exception e) {
492 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e));
493 webRTCWrapper.close();
494 sendSessionTerminate(Reason.FAILED_APPLICATION);
495 return;
496 }
497 final List<String> identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags();
498 processCandidates(identificationTags, contentMap.contents.entrySet());
499 }
500
501 private void sendSessionAccept() {
502 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
503 if (rtpContentMap == null) {
504 throw new IllegalStateException("initiator RTP Content Map has not been set");
505 }
506 final SessionDescription offer;
507 try {
508 offer = SessionDescription.of(rtpContentMap);
509 } catch (final IllegalArgumentException | NullPointerException e) {
510 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e);
511 webRTCWrapper.close();
512 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
513 return;
514 }
515 sendSessionAccept(rtpContentMap.getMedia(), offer);
516 }
517
518 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
519 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
520 }
521
522 private synchronized void sendSessionAccept(final Set<Media> media, final SessionDescription offer, final List<PeerConnection.IceServer> iceServers) {
523 if (isTerminated()) {
524 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
525 return;
526 }
527 try {
528 setupWebRTC(media, iceServers);
529 } catch (final WebRTCWrapper.InitializationException e) {
530 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
531 webRTCWrapper.close();
532 sendSessionTerminate(Reason.FAILED_APPLICATION);
533 return;
534 }
535 final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(
536 org.webrtc.SessionDescription.Type.OFFER,
537 offer.toString()
538 );
539 try {
540 this.webRTCWrapper.setRemoteDescription(sdp).get();
541 addIceCandidatesFromBlackLog();
542 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get();
543 prepareSessionAccept(webRTCSessionDescription);
544 } catch (final Exception e) {
545 failureToAcceptSession(e);
546 }
547 }
548
549 private void failureToAcceptSession(final Throwable throwable) {
550 if (isTerminated()) {
551 return;
552 }
553 Log.d(Config.LOGTAG, "unable to send session accept", Throwables.getRootCause(throwable));
554 webRTCWrapper.close();
555 sendSessionTerminate(Reason.ofThrowable(throwable));
556 }
557
558 private void addIceCandidatesFromBlackLog() {
559 while (!this.pendingIceCandidates.isEmpty()) {
560 processCandidates(this.pendingIceCandidates.poll());
561 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log");
562 }
563 }
564
565 private void prepareSessionAccept(final org.webrtc.SessionDescription webRTCSessionDescription) {
566 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
567 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
568 this.responderRtpContentMap = respondingRtpContentMap;
569 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = prepareOutgoingContentMap(respondingRtpContentMap);
570 Futures.addCallback(outgoingContentMapFuture,
571 new FutureCallback<RtpContentMap>() {
572 @Override
573 public void onSuccess(final RtpContentMap outgoingContentMap) {
574 sendSessionAccept(outgoingContentMap, webRTCSessionDescription);
575 }
576
577 @Override
578 public void onFailure(@NonNull Throwable throwable) {
579 failureToAcceptSession(throwable);
580 }
581 },
582 MoreExecutors.directExecutor()
583 );
584 }
585
586 private void sendSessionAccept(final RtpContentMap rtpContentMap, final org.webrtc.SessionDescription webRTCSessionDescription) {
587 if (isTerminated()) {
588 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session accept was too slow. already terminated. nothing to do.");
589 return;
590 }
591 transitionOrThrow(State.SESSION_ACCEPTED);
592 final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
593 send(sessionAccept);
594 try {
595 webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
596 } catch (Exception e) {
597 failureToAcceptSession(e);
598 }
599 }
600
601 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(final RtpContentMap rtpContentMap) {
602 if (this.omemoVerification.hasDeviceId()) {
603 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
604 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
605 return Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
606 omemoVerification.setOrEnsureEqual(verifiedPayload);
607 return verifiedPayload.getPayload();
608 }, MoreExecutors.directExecutor());
609 } else {
610 return Futures.immediateFuture(rtpContentMap);
611 }
612 }
613
614 synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) {
615 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
616 switch (message.getName()) {
617 case "propose":
618 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
619 break;
620 case "proceed":
621 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
622 break;
623 case "retract":
624 receiveRetract(from, serverMessageId, timestamp);
625 break;
626 case "reject":
627 receiveReject(from, serverMessageId, timestamp);
628 break;
629 case "accept":
630 receiveAccept(from, serverMessageId, timestamp);
631 break;
632 default:
633 break;
634 }
635 }
636
637 void deliverFailedProceed() {
638 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message");
639 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
640 webRTCWrapper.close();
641 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error");
642 this.finish();
643 }
644 }
645
646 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
647 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
648 if (originatedFromMyself) {
649 if (transition(State.ACCEPTED)) {
650 if (serverMsgId != null) {
651 this.message.setServerMsgId(serverMsgId);
652 }
653 this.message.setTime(timestamp);
654 this.message.setCarbon(true); //indicate that call was accepted on other device
655 this.writeLogMessageSuccess(0);
656 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
657 this.finish();
658 } else {
659 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state);
660 }
661 } else {
662 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
663 }
664 }
665
666 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
667 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
668 //reject from another one of my clients
669 if (originatedFromMyself) {
670 receiveRejectFromMyself(serverMsgId, timestamp);
671 } else if (isInitiator()) {
672 if (from.equals(id.with)) {
673 receiveRejectFromResponder();
674 } else {
675 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
676 }
677 } else {
678 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with);
679 }
680 }
681
682 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
683 if (transition(State.REJECTED)) {
684 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
685 this.finish();
686 if (serverMsgId != null) {
687 this.message.setServerMsgId(serverMsgId);
688 }
689 this.message.setTime(timestamp);
690 this.message.setCarbon(true); //indicate that call was rejected on other device
691 writeLogMessageMissed();
692 } else {
693 Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state);
694 }
695 }
696
697 private void receiveRejectFromResponder() {
698 if (isInState(State.PROCEED)) {
699 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered");
700 closeTransitionLogFinish(State.REJECTED_RACED);
701 return;
702 }
703 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
704 Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
705 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
706 return;
707 }
708 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state);
709 }
710
711 private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
712 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
713 if (originatedFromMyself) {
714 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
715 } else if (transition(State.PROPOSED, () -> {
716 final Collection<RtpDescription> descriptions = Collections2.transform(
717 Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription),
718 input -> (RtpDescription) input
719 );
720 final Collection<Media> media = Collections2.transform(descriptions, RtpDescription::getMedia);
721 Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media");
722 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media);
723 this.proposedMedia = Sets.newHashSet(media);
724 })) {
725 if (serverMsgId != null) {
726 this.message.setServerMsgId(serverMsgId);
727 }
728 this.message.setTime(timestamp);
729 startRinging();
730 } else {
731 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
732 }
733 }
734
735 private void startRinging() {
736 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing");
737 ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
738 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
739 }
740
741 private synchronized void ringingTimeout() {
742 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
743 switch (this.state) {
744 case PROPOSED:
745 message.markUnread();
746 rejectCallFromProposed();
747 break;
748 case SESSION_INITIALIZED:
749 message.markUnread();
750 rejectCallFromSessionInitiate();
751 break;
752 }
753 }
754
755 private void cancelRingingTimeout() {
756 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
757 if (future != null && !future.isCancelled()) {
758 future.cancel(false);
759 }
760 }
761
762 private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
763 final Set<Media> media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed");
764 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
765 if (from.equals(id.with)) {
766 if (isInitiator()) {
767 if (transition(State.PROCEED)) {
768 if (serverMsgId != null) {
769 this.message.setServerMsgId(serverMsgId);
770 }
771 this.message.setTime(timestamp);
772 final Integer remoteDeviceId = proceed.getDeviceId();
773 if (isOmemoEnabled()) {
774 this.omemoVerification.setDeviceId(remoteDeviceId);
775 } else {
776 if (remoteDeviceId != null) {
777 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
778 }
779 this.omemoVerification.setDeviceId(null);
780 }
781 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
782 } else {
783 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
784 }
785 } else {
786 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
787 }
788 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
789 if (transition(State.ACCEPTED)) {
790 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced");
791 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
792 this.finish();
793 }
794 } else {
795 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
796 }
797 }
798
799 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
800 if (from.equals(id.with)) {
801 final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
802 if (transition(target)) {
803 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
804 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")");
805 if (serverMsgId != null) {
806 this.message.setServerMsgId(serverMsgId);
807 }
808 this.message.setTime(timestamp);
809 if (target == State.RETRACTED) {
810 this.message.markUnread();
811 }
812 writeLogMessageMissed();
813 finish();
814 } else {
815 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
816 }
817 } else {
818 //TODO parse retract from self
819 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring");
820 }
821 }
822
823 public void sendSessionInitiate() {
824 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
825 }
826
827 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
828 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
829 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
830 }
831
832 private synchronized void sendSessionInitiate(final Set<Media> media, final State targetState, final List<PeerConnection.IceServer> iceServers) {
833 if (isTerminated()) {
834 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do.");
835 return;
836 }
837 try {
838 setupWebRTC(media, iceServers);
839 } catch (final WebRTCWrapper.InitializationException e) {
840 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
841 webRTCWrapper.close();
842 sendRetract(Reason.ofThrowable(e));
843 return;
844 }
845 try {
846 org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get();
847 prepareSessionInitiate(webRTCSessionDescription, targetState);
848 } catch (final Exception e) {
849 failureToInitiateSession(e, targetState);
850 }
851 }
852
853 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
854 if (isTerminated()) {
855 return;
856 }
857 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(throwable));
858 webRTCWrapper.close();
859 final Reason reason = Reason.ofThrowable(throwable);
860 if (isInState(targetState)) {
861 sendSessionTerminate(reason);
862 } else {
863 sendRetract(reason);
864 }
865 }
866
867 private void sendRetract(final Reason reason) {
868 //TODO embed reason into retract
869 sendJingleMessage("retract", id.with.asBareJid());
870 transitionOrThrow(reasonToState(reason));
871 this.finish();
872 }
873
874 private void prepareSessionInitiate(final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
875 final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
876 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
877 this.initiatorRtpContentMap = rtpContentMap;
878 final ListenableFuture<RtpContentMap> outgoingContentMapFuture = encryptSessionInitiate(rtpContentMap);
879 Futures.addCallback(outgoingContentMapFuture, new FutureCallback<RtpContentMap>() {
880 @Override
881 public void onSuccess(final RtpContentMap outgoingContentMap) {
882 sendSessionInitiate(outgoingContentMap, webRTCSessionDescription, targetState);
883 }
884
885 @Override
886 public void onFailure(@NonNull final Throwable throwable) {
887 failureToInitiateSession(throwable, targetState);
888 }
889 }, MoreExecutors.directExecutor());
890 }
891
892 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
893 if (isTerminated()) {
894 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": preparing session was too slow. already terminated. nothing to do.");
895 return;
896 }
897 this.transitionOrThrow(targetState);
898 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
899 send(sessionInitiate);
900 try {
901 this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
902 } catch (Exception e) {
903 failureToInitiateSession(e, targetState);
904 }
905 }
906
907 private ListenableFuture<RtpContentMap> encryptSessionInitiate(final RtpContentMap rtpContentMap) {
908 if (this.omemoVerification.hasDeviceId()) {
909 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>> verifiedPayloadFuture = id.account.getAxolotlService()
910 .encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId());
911 final ListenableFuture<RtpContentMap> future = Futures.transform(verifiedPayloadFuture, verifiedPayload -> {
912 omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint());
913 return verifiedPayload.getPayload();
914 }, MoreExecutors.directExecutor());
915 if (Config.REQUIRE_RTP_VERIFICATION) {
916 return future;
917 }
918 return Futures.catching(
919 future,
920 CryptoFailedException.class,
921 e -> {
922 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e);
923 return rtpContentMap;
924 },
925 MoreExecutors.directExecutor()
926 );
927 } else {
928 return Futures.immediateFuture(rtpContentMap);
929 }
930 }
931
932 private void sendSessionTerminate(final Reason reason) {
933 sendSessionTerminate(reason, null);
934 }
935
936 private void sendSessionTerminate(final Reason reason, final String text) {
937 final State previous = this.state;
938 final State target = reasonToState(reason);
939 transitionOrThrow(target);
940 if (previous != State.NULL) {
941 writeLogMessage(target);
942 }
943 final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
944 jinglePacket.setReason(reason, text);
945 Log.d(Config.LOGTAG, jinglePacket.toString());
946 send(jinglePacket);
947 finish();
948 }
949
950 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
951 final RtpContentMap transportInfo;
952 try {
953 final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
954 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
955 } catch (final Exception e) {
956 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
957 return;
958 }
959 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
960 send(jinglePacket);
961 }
962
963 private void send(final JinglePacket jinglePacket) {
964 jinglePacket.setTo(id.with);
965 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
966 }
967
968 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
969 if (response.getType() == IqPacket.TYPE.ERROR) {
970 final String errorCondition = response.getErrorCondition();
971 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition);
972 if (isTerminated()) {
973 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
974 return;
975 }
976 this.webRTCWrapper.close();
977 final State target;
978 if (Arrays.asList(
979 "service-unavailable",
980 "recipient-unavailable",
981 "remote-server-not-found",
982 "remote-server-timeout"
983 ).contains(errorCondition)) {
984 target = State.TERMINATED_CONNECTIVITY_ERROR;
985 } else {
986 target = State.TERMINATED_APPLICATION_FAILURE;
987 }
988 transitionOrThrow(target);
989 this.finish();
990 } else if (response.getType() == IqPacket.TYPE.TIMEOUT) {
991 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error");
992 if (isTerminated()) {
993 Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated");
994 return;
995 }
996 this.webRTCWrapper.close();
997 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
998 this.finish();
999 }
1000 }
1001
1002 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1003 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1004 this.webRTCWrapper.close();
1005 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1006 respondWithOutOfOrder(jinglePacket);
1007 this.finish();
1008 }
1009
1010 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1011 jingleConnectionManager.respondWithJingleError(id.account, jinglePacket, "out-of-order", "unexpected-request", "wait");
1012 }
1013
1014 private void respondOk(final JinglePacket jinglePacket) {
1015 xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1016 }
1017
1018 public void throwStateTransitionException() {
1019 final StateTransitionException exception = this.stateTransitionException;
1020 if (exception != null) {
1021 throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception);
1022 }
1023 }
1024
1025 public RtpEndUserState getEndUserState() {
1026 switch (this.state) {
1027 case NULL:
1028 case PROPOSED:
1029 case SESSION_INITIALIZED:
1030 if (isInitiator()) {
1031 return RtpEndUserState.RINGING;
1032 } else {
1033 return RtpEndUserState.INCOMING_CALL;
1034 }
1035 case PROCEED:
1036 if (isInitiator()) {
1037 return RtpEndUserState.RINGING;
1038 } else {
1039 return RtpEndUserState.ACCEPTING_CALL;
1040 }
1041 case SESSION_INITIALIZED_PRE_APPROVED:
1042 if (isInitiator()) {
1043 return RtpEndUserState.RINGING;
1044 } else {
1045 return RtpEndUserState.CONNECTING;
1046 }
1047 case SESSION_ACCEPTED:
1048 final PeerConnection.PeerConnectionState state;
1049 try {
1050 state = webRTCWrapper.getState();
1051 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
1052 //We usually close the WebRTCWrapper *before* transitioning so we might still
1053 //be in SESSION_ACCEPTED even though the peerConnection has been torn down
1054 return RtpEndUserState.ENDING_CALL;
1055 }
1056 if (state == PeerConnection.PeerConnectionState.CONNECTED) {
1057 return RtpEndUserState.CONNECTED;
1058 } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) {
1059 return RtpEndUserState.CONNECTING;
1060 } else if (state == PeerConnection.PeerConnectionState.CLOSED) {
1061 return RtpEndUserState.ENDING_CALL;
1062 } else {
1063 return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1064 }
1065 case REJECTED:
1066 case REJECTED_RACED:
1067 case TERMINATED_DECLINED_OR_BUSY:
1068 if (isInitiator()) {
1069 return RtpEndUserState.DECLINED_OR_BUSY;
1070 } else {
1071 return RtpEndUserState.ENDED;
1072 }
1073 case TERMINATED_SUCCESS:
1074 case ACCEPTED:
1075 case RETRACTED:
1076 case TERMINATED_CANCEL_OR_TIMEOUT:
1077 return RtpEndUserState.ENDED;
1078 case RETRACTED_RACED:
1079 if (isInitiator()) {
1080 return RtpEndUserState.ENDED;
1081 } else {
1082 return RtpEndUserState.RETRACTED;
1083 }
1084 case TERMINATED_CONNECTIVITY_ERROR:
1085 return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
1086 case TERMINATED_APPLICATION_FAILURE:
1087 return RtpEndUserState.APPLICATION_ERROR;
1088 case TERMINATED_SECURITY_ERROR:
1089 return RtpEndUserState.SECURITY_ERROR;
1090 }
1091 throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state));
1092 }
1093
1094 public Set<Media> getMedia() {
1095 final State current = getState();
1096 if (current == State.NULL) {
1097 if (isInitiator()) {
1098 return Preconditions.checkNotNull(
1099 this.proposedMedia,
1100 "RTP connection has not been initialized properly"
1101 );
1102 }
1103 throw new IllegalStateException("RTP connection has not been initialized yet");
1104 }
1105 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
1106 return Preconditions.checkNotNull(
1107 this.proposedMedia,
1108 "RTP connection has not been initialized properly"
1109 );
1110 }
1111 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
1112 if (initiatorContentMap != null) {
1113 return initiatorContentMap.getMedia();
1114 } else if (isTerminated()) {
1115 return Collections.emptySet(); //we might fail before we ever got a chance to set media
1116 } else {
1117 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
1118 }
1119 }
1120
1121
1122 public boolean isVerified() {
1123 final String fingerprint = this.omemoVerification.getFingerprint();
1124 if (fingerprint == null) {
1125 return false;
1126 }
1127 final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint);
1128 return status != null && status.isVerified();
1129 }
1130
1131 public synchronized void acceptCall() {
1132 switch (this.state) {
1133 case PROPOSED:
1134 cancelRingingTimeout();
1135 acceptCallFromProposed();
1136 break;
1137 case SESSION_INITIALIZED:
1138 cancelRingingTimeout();
1139 acceptCallFromSessionInitialized();
1140 break;
1141 case ACCEPTED:
1142 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind");
1143 break;
1144 case PROCEED:
1145 case SESSION_ACCEPTED:
1146 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI");
1147 break;
1148 default:
1149 throw new IllegalStateException("Can not accept call from " + this.state);
1150 }
1151 }
1152
1153
1154 public void notifyPhoneCall() {
1155 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
1156 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
1157 rejectCall();
1158 } else {
1159 endCall();
1160 }
1161 }
1162
1163 public synchronized void rejectCall() {
1164 if (isTerminated()) {
1165 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do");
1166 return;
1167 }
1168 switch (this.state) {
1169 case PROPOSED:
1170 rejectCallFromProposed();
1171 break;
1172 case SESSION_INITIALIZED:
1173 rejectCallFromSessionInitiate();
1174 break;
1175 default:
1176 throw new IllegalStateException("Can not reject call from " + this.state);
1177 }
1178 }
1179
1180 public synchronized void endCall() {
1181 if (isTerminated()) {
1182 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do");
1183 return;
1184 }
1185 if (isInState(State.PROPOSED) && !isInitiator()) {
1186 rejectCallFromProposed();
1187 return;
1188 }
1189 if (isInState(State.PROCEED)) {
1190 if (isInitiator()) {
1191 retractFromProceed();
1192 } else {
1193 rejectCallFromProceed();
1194 }
1195 return;
1196 }
1197 if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
1198 this.webRTCWrapper.close();
1199 sendSessionTerminate(Reason.CANCEL);
1200 return;
1201 }
1202 if (isInState(State.SESSION_INITIALIZED)) {
1203 rejectCallFromSessionInitiate();
1204 return;
1205 }
1206 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
1207 this.webRTCWrapper.close();
1208 sendSessionTerminate(Reason.SUCCESS);
1209 return;
1210 }
1211 if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) {
1212 Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state);
1213 return;
1214 }
1215 throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
1216 }
1217
1218 private void retractFromProceed() {
1219 Log.d(Config.LOGTAG, "retract from proceed");
1220 this.sendJingleMessage("retract");
1221 closeTransitionLogFinish(State.RETRACTED_RACED);
1222 }
1223
1224 private void closeTransitionLogFinish(final State state) {
1225 this.webRTCWrapper.close();
1226 transitionOrThrow(state);
1227 writeLogMessage(state);
1228 finish();
1229 }
1230
1231 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
1232 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
1233 final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
1234 if (media.contains(Media.VIDEO)) {
1235 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
1236 } else {
1237 speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
1238 }
1239 this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
1240 this.webRTCWrapper.initializePeerConnection(media, iceServers);
1241 }
1242
1243 private void acceptCallFromProposed() {
1244 transitionOrThrow(State.PROCEED);
1245 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1246 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
1247 this.sendJingleMessage("proceed");
1248 }
1249
1250 private void rejectCallFromProposed() {
1251 transitionOrThrow(State.REJECTED);
1252 writeLogMessageMissed();
1253 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1254 this.sendJingleMessage("reject");
1255 finish();
1256 }
1257
1258 private void rejectCallFromProceed() {
1259 this.sendJingleMessage("reject");
1260 closeTransitionLogFinish(State.REJECTED_RACED);
1261 }
1262
1263 private void rejectCallFromSessionInitiate() {
1264 webRTCWrapper.close();
1265 sendSessionTerminate(Reason.DECLINE);
1266 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1267 }
1268
1269 private void sendJingleMessage(final String action) {
1270 sendJingleMessage(action, id.with);
1271 }
1272
1273 private void sendJingleMessage(final String action, final Jid to) {
1274 final MessagePacket messagePacket = new MessagePacket();
1275 messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those
1276 messagePacket.setTo(to);
1277 final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
1278 if ("proceed".equals(action)) {
1279 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
1280 if (isOmemoEnabled()) {
1281 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
1282 final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
1283 device.setAttribute("id", deviceId);
1284 }
1285 }
1286 messagePacket.addChild("store", "urn:xmpp:hints");
1287 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
1288 }
1289
1290 private boolean isOmemoEnabled() {
1291 final Conversational conversational = message.getConversation();
1292 if (conversational instanceof Conversation) {
1293 return ((Conversation) conversational).getNextEncryption() == Message.ENCRYPTION_AXOLOTL;
1294 }
1295 return false;
1296 }
1297
1298 private void acceptCallFromSessionInitialized() {
1299 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1300 sendSessionAccept();
1301 }
1302
1303 private synchronized boolean isInState(State... state) {
1304 return Arrays.asList(state).contains(this.state);
1305 }
1306
1307 private boolean transition(final State target) {
1308 return transition(target, null);
1309 }
1310
1311 private synchronized boolean transition(final State target, final Runnable runnable) {
1312 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
1313 if (validTransitions != null && validTransitions.contains(target)) {
1314 this.state = target;
1315 this.stateTransitionException = new StateTransitionException(target);
1316 if (runnable != null) {
1317 runnable.run();
1318 }
1319 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
1320 updateEndUserState();
1321 updateOngoingCallNotification();
1322 return true;
1323 } else {
1324 return false;
1325 }
1326 }
1327
1328 void transitionOrThrow(final State target) {
1329 if (!transition(target)) {
1330 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
1331 }
1332 }
1333
1334 @Override
1335 public void onIceCandidate(final IceCandidate iceCandidate) {
1336 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
1337 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
1338 sendTransportInfo(iceCandidate.sdpMid, candidate);
1339 }
1340
1341 @Override
1342 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
1343 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
1344 if (newState == PeerConnection.PeerConnectionState.CONNECTED && this.rtpConnectionStarted == 0) {
1345 this.rtpConnectionStarted = SystemClock.elapsedRealtime();
1346 }
1347 if (newState == PeerConnection.PeerConnectionState.CLOSED && this.rtpConnectionEnded == 0) {
1348 this.rtpConnectionEnded = SystemClock.elapsedRealtime();
1349 }
1350 //TODO 'DISCONNECTED' might be an opportunity to renew the offer and send a transport-replace
1351 //TODO exact syntax is yet to be determined but transport-replace sounds like the most reasonable
1352 //as there is no content-replace
1353 if (Arrays.asList(PeerConnection.PeerConnectionState.FAILED, PeerConnection.PeerConnectionState.DISCONNECTED).contains(newState)) {
1354 if (isTerminated()) {
1355 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state);
1356 return;
1357 }
1358 new Thread(this::closeWebRTCSessionAfterFailedConnection).start();
1359 } else {
1360 updateEndUserState();
1361 }
1362 }
1363
1364 private void closeWebRTCSessionAfterFailedConnection() {
1365 this.webRTCWrapper.close();
1366 synchronized (this) {
1367 if (isTerminated()) {
1368 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did");
1369 return;
1370 }
1371 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
1372 }
1373 }
1374
1375 public long getRtpConnectionStarted() {
1376 return this.rtpConnectionStarted;
1377 }
1378
1379 public long getRtpConnectionEnded() {
1380 return this.rtpConnectionEnded;
1381 }
1382
1383 public AppRTCAudioManager getAudioManager() {
1384 return webRTCWrapper.getAudioManager();
1385 }
1386
1387 public boolean isMicrophoneEnabled() {
1388 return webRTCWrapper.isMicrophoneEnabled();
1389 }
1390
1391 public boolean setMicrophoneEnabled(final boolean enabled) {
1392 return webRTCWrapper.setMicrophoneEnabled(enabled);
1393 }
1394
1395 public boolean isVideoEnabled() {
1396 return webRTCWrapper.isVideoEnabled();
1397 }
1398
1399 public void setVideoEnabled(final boolean enabled) {
1400 webRTCWrapper.setVideoEnabled(enabled);
1401 }
1402
1403 public boolean isCameraSwitchable() {
1404 return webRTCWrapper.isCameraSwitchable();
1405 }
1406
1407 public boolean isFrontCamera() {
1408 return webRTCWrapper.isFrontCamera();
1409 }
1410
1411 public ListenableFuture<Boolean> switchCamera() {
1412 return webRTCWrapper.switchCamera();
1413 }
1414
1415 @Override
1416 public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
1417 xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices);
1418 }
1419
1420 private void updateEndUserState() {
1421 final RtpEndUserState endUserState = getEndUserState();
1422 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
1423 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1424 }
1425
1426 private void updateOngoingCallNotification() {
1427 if (STATES_SHOWING_ONGOING_CALL.contains(this.state)) {
1428 xmppConnectionService.setOngoingCall(id, getMedia());
1429 } else {
1430 xmppConnectionService.removeOngoingCall();
1431 }
1432 }
1433
1434 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
1435 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
1436 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
1437 request.setTo(id.account.getDomain());
1438 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1439 xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> {
1440 ImmutableList.Builder<PeerConnection.IceServer> listBuilder = new ImmutableList.Builder<>();
1441 if (response.getType() == IqPacket.TYPE.RESULT) {
1442 final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
1443 final List<Element> children = services == null ? Collections.emptyList() : services.getChildren();
1444 for (final Element child : children) {
1445 if ("service".equals(child.getName())) {
1446 final String type = child.getAttribute("type");
1447 final String host = child.getAttribute("host");
1448 final String sport = child.getAttribute("port");
1449 final Integer port = sport == null ? null : Ints.tryParse(sport);
1450 final String transport = child.getAttribute("transport");
1451 final String username = child.getAttribute("username");
1452 final String password = child.getAttribute("password");
1453 if (Strings.isNullOrEmpty(host) || port == null) {
1454 continue;
1455 }
1456 if (port < 0 || port > 65535) {
1457 continue;
1458 }
1459 if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) {
1460 if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) {
1461 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services");
1462 continue;
1463 }
1464 final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer
1465 .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport));
1466 iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK);
1467 if (username != null && password != null) {
1468 iceServerBuilder.setUsername(username);
1469 iceServerBuilder.setPassword(password);
1470 } else if (Arrays.asList("turn", "turns").contains(type)) {
1471 //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder)
1472 //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
1473 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password");
1474 continue;
1475 }
1476 final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer();
1477 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer);
1478 listBuilder.add(iceServer);
1479 }
1480 }
1481 }
1482 }
1483 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
1484 if (iceServers.size() == 0) {
1485 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response);
1486 }
1487 onIceServersDiscovered.onIceServersDiscovered(iceServers);
1488 });
1489 } else {
1490 Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery");
1491 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
1492 }
1493 }
1494
1495 private void finish() {
1496 if (isTerminated()) {
1497 this.cancelRingingTimeout();
1498 this.webRTCWrapper.verifyClosed();
1499 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
1500 this.jingleConnectionManager.finishConnectionOrThrow(this);
1501 } else {
1502 throw new IllegalStateException(String.format("Unable to call finish from %s", this.state));
1503 }
1504 }
1505
1506 private void writeLogMessage(final State state) {
1507 final long started = this.rtpConnectionStarted;
1508 long duration = started <= 0 ? 0 : SystemClock.elapsedRealtime() - started;
1509 if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
1510 writeLogMessageSuccess(duration);
1511 } else {
1512 writeLogMessageMissed();
1513 }
1514 }
1515
1516 private void writeLogMessageSuccess(final long duration) {
1517 this.message.setBody(new RtpSessionStatus(true, duration).toString());
1518 this.writeMessage();
1519 }
1520
1521 private void writeLogMessageMissed() {
1522 this.message.setBody(new RtpSessionStatus(false, 0).toString());
1523 this.writeMessage();
1524 }
1525
1526 private void writeMessage() {
1527 final Conversational conversational = message.getConversation();
1528 if (conversational instanceof Conversation) {
1529 ((Conversation) conversational).add(this.message);
1530 xmppConnectionService.createMessageAsync(message);
1531 xmppConnectionService.updateConversationUi();
1532 } else {
1533 throw new IllegalStateException("Somehow the conversation in a message was a stub");
1534 }
1535 }
1536
1537 public State getState() {
1538 return this.state;
1539 }
1540
1541 boolean isTerminated() {
1542 return TERMINATED.contains(this.state);
1543 }
1544
1545 public Optional<VideoTrack> getLocalVideoTrack() {
1546 return webRTCWrapper.getLocalVideoTrack();
1547 }
1548
1549 public Optional<VideoTrack> getRemoteVideoTrack() {
1550 return webRTCWrapper.getRemoteVideoTrack();
1551 }
1552
1553
1554 public EglBase.Context getEglBaseContext() {
1555 return webRTCWrapper.getEglBaseContext();
1556 }
1557
1558 void setProposedMedia(final Set<Media> media) {
1559 this.proposedMedia = media;
1560 }
1561
1562 public void fireStateUpdate() {
1563 final RtpEndUserState endUserState = getEndUserState();
1564 xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState);
1565 }
1566
1567 private interface OnIceServersDiscovered {
1568 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
1569 }
1570
1571 private static class StateTransitionException extends Exception {
1572 private final State state;
1573
1574 private StateTransitionException(final State state) {
1575 this.state = state;
1576 }
1577 }
1578}