1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4import android.os.Environment;
5
6import androidx.annotation.NonNull;
7import androidx.annotation.Nullable;
8
9import com.google.common.base.Joiner;
10import com.google.common.base.Optional;
11import com.google.common.base.Preconditions;
12import com.google.common.base.Stopwatch;
13import com.google.common.base.Strings;
14import com.google.common.base.Throwables;
15import com.google.common.collect.Collections2;
16import com.google.common.collect.ImmutableList;
17import com.google.common.collect.ImmutableMap;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Multimap;
23import com.google.common.collect.Sets;
24import com.google.common.primitives.Ints;
25import com.google.common.util.concurrent.FutureCallback;
26import com.google.common.util.concurrent.Futures;
27import com.google.common.util.concurrent.ListenableFuture;
28import com.google.common.util.concurrent.MoreExecutors;
29
30import org.webrtc.DtmfSender;
31import org.webrtc.EglBase;
32import org.webrtc.IceCandidate;
33import org.webrtc.PeerConnection;
34import org.webrtc.VideoTrack;
35
36import java.io.File;
37import java.io.IOException;
38import java.util.Arrays;
39import java.util.Collection;
40import java.util.Collections;
41import java.util.LinkedList;
42import java.util.List;
43import java.util.Map;
44import java.util.Queue;
45import java.util.Set;
46import java.util.concurrent.ExecutionException;
47import java.util.concurrent.ScheduledFuture;
48import java.util.concurrent.TimeUnit;
49
50import eu.siacs.conversations.BuildConfig;
51import eu.siacs.conversations.Config;
52import eu.siacs.conversations.crypto.axolotl.AxolotlService;
53import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
54import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
55import eu.siacs.conversations.entities.Account;
56import eu.siacs.conversations.entities.Contact;
57import eu.siacs.conversations.entities.Conversation;
58import eu.siacs.conversations.entities.Conversational;
59import eu.siacs.conversations.entities.Message;
60import eu.siacs.conversations.entities.Presence;
61import eu.siacs.conversations.entities.RtpSessionStatus;
62import eu.siacs.conversations.entities.ServiceDiscoveryResult;
63import eu.siacs.conversations.services.AppRTCAudioManager;
64import eu.siacs.conversations.utils.IP;
65import eu.siacs.conversations.xml.Element;
66import eu.siacs.conversations.xml.Namespace;
67import eu.siacs.conversations.xmpp.Jid;
68import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
69import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
70import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
71import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
72import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
73import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
74import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
75import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
76import eu.siacs.conversations.xmpp.stanzas.IqPacket;
77import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
78
79public class JingleRtpConnection extends AbstractJingleConnection
80 implements WebRTCWrapper.EventCallback {
81
82 public static final List<State> STATES_SHOWING_ONGOING_CALL =
83 Arrays.asList(
84 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
85 private static final long BUSY_TIME_OUT = 30;
86 private static final List<State> TERMINATED =
87 Arrays.asList(
88 State.ACCEPTED,
89 State.REJECTED,
90 State.REJECTED_RACED,
91 State.RETRACTED,
92 State.RETRACTED_RACED,
93 State.TERMINATED_SUCCESS,
94 State.TERMINATED_DECLINED_OR_BUSY,
95 State.TERMINATED_CONNECTIVITY_ERROR,
96 State.TERMINATED_CANCEL_OR_TIMEOUT,
97 State.TERMINATED_APPLICATION_FAILURE,
98 State.TERMINATED_SECURITY_ERROR);
99
100 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
101
102 static {
103 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
104 new ImmutableMap.Builder<>();
105 transitionBuilder.put(
106 State.NULL,
107 ImmutableList.of(
108 State.PROPOSED,
109 State.SESSION_INITIALIZED,
110 State.TERMINATED_APPLICATION_FAILURE,
111 State.TERMINATED_SECURITY_ERROR));
112 transitionBuilder.put(
113 State.PROPOSED,
114 ImmutableList.of(
115 State.ACCEPTED,
116 State.PROCEED,
117 State.REJECTED,
118 State.RETRACTED,
119 State.TERMINATED_APPLICATION_FAILURE,
120 State.TERMINATED_SECURITY_ERROR,
121 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
122 // rebinds
123 ));
124 transitionBuilder.put(
125 State.PROCEED,
126 ImmutableList.of(
127 State.REJECTED_RACED,
128 State.RETRACTED_RACED,
129 State.SESSION_INITIALIZED_PRE_APPROVED,
130 State.TERMINATED_SUCCESS,
131 State.TERMINATED_APPLICATION_FAILURE,
132 State.TERMINATED_SECURITY_ERROR,
133 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
134 // bounces of the proceed message
135 ));
136 transitionBuilder.put(
137 State.SESSION_INITIALIZED,
138 ImmutableList.of(
139 State.SESSION_ACCEPTED,
140 State.TERMINATED_SUCCESS,
141 State.TERMINATED_DECLINED_OR_BUSY,
142 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
143 // and IQ timeouts
144 State.TERMINATED_CANCEL_OR_TIMEOUT,
145 State.TERMINATED_APPLICATION_FAILURE,
146 State.TERMINATED_SECURITY_ERROR));
147 transitionBuilder.put(
148 State.SESSION_INITIALIZED_PRE_APPROVED,
149 ImmutableList.of(
150 State.SESSION_ACCEPTED,
151 State.TERMINATED_SUCCESS,
152 State.TERMINATED_DECLINED_OR_BUSY,
153 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
154 // and IQ timeouts
155 State.TERMINATED_CANCEL_OR_TIMEOUT,
156 State.TERMINATED_APPLICATION_FAILURE,
157 State.TERMINATED_SECURITY_ERROR));
158 transitionBuilder.put(
159 State.SESSION_ACCEPTED,
160 ImmutableList.of(
161 State.TERMINATED_SUCCESS,
162 State.TERMINATED_DECLINED_OR_BUSY,
163 State.TERMINATED_CONNECTIVITY_ERROR,
164 State.TERMINATED_CANCEL_OR_TIMEOUT,
165 State.TERMINATED_APPLICATION_FAILURE,
166 State.TERMINATED_SECURITY_ERROR));
167 VALID_TRANSITIONS = transitionBuilder.build();
168 }
169
170 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
171 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
172 pendingIceCandidates = new LinkedList<>();
173 private final OmemoVerification omemoVerification = new OmemoVerification();
174 private final Message message;
175 private State state = State.NULL;
176 private Set<Media> proposedMedia;
177 private RtpContentMap initiatorRtpContentMap;
178 private RtpContentMap responderRtpContentMap;
179 private RtpContentMap incomingContentAdd;
180 private RtpContentMap outgoingContentAdd;
181 private IceUdpTransportInfo.Setup peerDtlsSetup;
182 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
183 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
184 private ScheduledFuture<?> ringingTimeoutFuture;
185 private final long created = System.currentTimeMillis() / 1000L;
186
187 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
188 super(jingleConnectionManager, id, initiator);
189 final Conversation conversation =
190 jingleConnectionManager
191 .getXmppConnectionService()
192 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
193 this.message =
194 new Message(
195 conversation,
196 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
197 Message.TYPE_RTP_SESSION,
198 id.sessionId);
199 }
200
201 private static State reasonToState(Reason reason) {
202 switch (reason) {
203 case SUCCESS:
204 return State.TERMINATED_SUCCESS;
205 case DECLINE:
206 case BUSY:
207 return State.TERMINATED_DECLINED_OR_BUSY;
208 case CANCEL:
209 case TIMEOUT:
210 return State.TERMINATED_CANCEL_OR_TIMEOUT;
211 case SECURITY_ERROR:
212 return State.TERMINATED_SECURITY_ERROR;
213 case FAILED_APPLICATION:
214 case UNSUPPORTED_TRANSPORTS:
215 case UNSUPPORTED_APPLICATIONS:
216 return State.TERMINATED_APPLICATION_FAILURE;
217 default:
218 return State.TERMINATED_CONNECTIVITY_ERROR;
219 }
220 }
221
222 @Override
223 synchronized void deliverPacket(final JinglePacket jinglePacket) {
224 switch (jinglePacket.getAction()) {
225 case SESSION_INITIATE:
226 receiveSessionInitiate(jinglePacket);
227 break;
228 case TRANSPORT_INFO:
229 receiveTransportInfo(jinglePacket);
230 break;
231 case SESSION_ACCEPT:
232 receiveSessionAccept(jinglePacket);
233 break;
234 case SESSION_TERMINATE:
235 receiveSessionTerminate(jinglePacket);
236 break;
237 case CONTENT_ADD:
238 receiveContentAdd(jinglePacket);
239 break;
240 case CONTENT_ACCEPT:
241 receiveContentAccept(jinglePacket);
242 break;
243 case CONTENT_REJECT:
244 receiveContentReject(jinglePacket);
245 break;
246 case CONTENT_REMOVE:
247 receiveContentRemove(jinglePacket);
248 break;
249 case CONTENT_MODIFY:
250 receiveContentModify(jinglePacket);
251 break;
252 default:
253 respondOk(jinglePacket);
254 Log.d(
255 Config.LOGTAG,
256 String.format(
257 "%s: received unhandled jingle action %s",
258 id.account.getJid().asBareJid(), jinglePacket.getAction()));
259 break;
260 }
261 }
262
263 @Override
264 synchronized void notifyRebound() {
265 if (isTerminated()) {
266 return;
267 }
268 webRTCWrapper.close();
269 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
270 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
271 }
272 if (isInState(
273 State.SESSION_INITIALIZED,
274 State.SESSION_INITIALIZED_PRE_APPROVED,
275 State.SESSION_ACCEPTED)) {
276 // we might have already changed resources (full jid) at this point; so this might not
277 // even reach the other party
278 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
279 } else {
280 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
281 finish();
282 }
283 }
284
285 public boolean applyDtmfTone(String tone) {
286 return webRTCWrapper.applyDtmfTone(tone);
287 }
288
289 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
290 respondOk(jinglePacket);
291 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
292 final State previous = this.state;
293 Log.d(
294 Config.LOGTAG,
295 id.account.getJid().asBareJid()
296 + ": received session terminate reason="
297 + wrapper.reason
298 + "("
299 + Strings.nullToEmpty(wrapper.text)
300 + ") while in state "
301 + previous);
302 if (TERMINATED.contains(previous)) {
303 Log.d(
304 Config.LOGTAG,
305 id.account.getJid().asBareJid()
306 + ": ignoring session terminate because already in "
307 + previous);
308 return;
309 }
310 webRTCWrapper.close();
311 final State target = reasonToState(wrapper.reason);
312 transitionOrThrow(target);
313 writeLogMessage(target);
314 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
315 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
316 }
317 finish();
318 }
319
320 private void receiveTransportInfo(final JinglePacket jinglePacket) {
321 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
322 // INITIALIZED only after transport-info has been received
323 if (isInState(
324 State.NULL,
325 State.PROCEED,
326 State.SESSION_INITIALIZED,
327 State.SESSION_INITIALIZED_PRE_APPROVED,
328 State.SESSION_ACCEPTED)) {
329 final RtpContentMap contentMap;
330 try {
331 contentMap = RtpContentMap.of(jinglePacket);
332 } catch (final IllegalArgumentException | NullPointerException e) {
333 Log.d(
334 Config.LOGTAG,
335 id.account.getJid().asBareJid()
336 + ": improperly formatted contents; ignoring",
337 e);
338 respondOk(jinglePacket);
339 return;
340 }
341 receiveTransportInfo(jinglePacket, contentMap);
342 } else {
343 if (isTerminated()) {
344 respondOk(jinglePacket);
345 Log.d(
346 Config.LOGTAG,
347 id.account.getJid().asBareJid()
348 + ": ignoring out-of-order transport info; we where already terminated");
349 } else {
350 Log.d(
351 Config.LOGTAG,
352 id.account.getJid().asBareJid()
353 + ": received transport info while in state="
354 + this.state);
355 terminateWithOutOfOrder(jinglePacket);
356 }
357 }
358 }
359
360 private void receiveTransportInfo(
361 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
362 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
363 contentMap.contents.entrySet();
364 if (this.state == State.SESSION_ACCEPTED) {
365 // zero candidates + modified credentials are an ICE restart offer
366 if (checkForIceRestart(jinglePacket, contentMap)) {
367 return;
368 }
369 respondOk(jinglePacket);
370 try {
371 processCandidates(candidates);
372 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
373 Log.w(
374 Config.LOGTAG,
375 id.account.getJid().asBareJid()
376 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
377 }
378 } else {
379 respondOk(jinglePacket);
380 pendingIceCandidates.addAll(candidates);
381 }
382 }
383
384 private void receiveContentAdd(final JinglePacket jinglePacket) {
385 final RtpContentMap modification;
386 try {
387 modification = RtpContentMap.of(jinglePacket);
388 modification.requireContentDescriptions();
389 } catch (final RuntimeException e) {
390 Log.d(
391 Config.LOGTAG,
392 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
393 Throwables.getRootCause(e));
394 respondOk(jinglePacket);
395 webRTCWrapper.close();
396 sendSessionTerminate(Reason.of(e), e.getMessage());
397 return;
398 }
399 if (isInState(State.SESSION_ACCEPTED)) {
400 receiveContentAdd(jinglePacket, modification);
401 } else {
402 terminateWithOutOfOrder(jinglePacket);
403 }
404 }
405
406 private void receiveContentAdd(
407 final JinglePacket jinglePacket, final RtpContentMap modification) {
408 final RtpContentMap remote = getRemoteContentMap();
409 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
410 respondOk(jinglePacket);
411 this.webRTCWrapper.close();
412 sendSessionTerminate(
413 Reason.FAILED_APPLICATION,
414 String.format(
415 "contents with names %s already exists",
416 Joiner.on(", ").join(modification.getNames())));
417 return;
418 }
419 final ContentAddition contentAddition =
420 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
421
422 final RtpContentMap outgoing = this.outgoingContentAdd;
423 final Set<ContentAddition.Summary> outgoingContentAddSummary =
424 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
425
426 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
427 if (isInitiator()) {
428 Log.d(
429 Config.LOGTAG,
430 id.getAccount().getJid().asBareJid()
431 + ": respond with tie break to matching content-add offer");
432 respondWithTieBreak(jinglePacket);
433 } else {
434 Log.d(
435 Config.LOGTAG,
436 id.getAccount().getJid().asBareJid()
437 + ": automatically accept matching content-add offer");
438 acceptContentAdd(contentAddition.summary, modification);
439 }
440 return;
441 }
442
443 // once we can display multiple video tracks we can be more loose with this condition
444 // theoretically it should also be fine to automatically accept audio only contents
445 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
446 Log.d(
447 Config.LOGTAG,
448 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
449 this.incomingContentAdd = modification;
450 respondOk(jinglePacket);
451 updateEndUserState();
452 } else {
453 respondOk(jinglePacket);
454 // TODO do we want to add a reason?
455 rejectContentAdd(modification);
456 }
457 }
458
459 private void receiveContentAccept(final JinglePacket jinglePacket) {
460 final RtpContentMap receivedContentAccept;
461 try {
462 receivedContentAccept = RtpContentMap.of(jinglePacket);
463 receivedContentAccept.requireContentDescriptions();
464 } catch (final RuntimeException e) {
465 Log.d(
466 Config.LOGTAG,
467 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
468 Throwables.getRootCause(e));
469 respondOk(jinglePacket);
470 webRTCWrapper.close();
471 sendSessionTerminate(Reason.of(e), e.getMessage());
472 return;
473 }
474
475 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
476 if (outgoingContentAdd == null) {
477 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
478 terminateWithOutOfOrder(jinglePacket);
479 return;
480 }
481 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
482 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
483 this.outgoingContentAdd = null;
484 respondOk(jinglePacket);
485 receiveContentAccept(receivedContentAccept);
486 } else {
487 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
488 terminateWithOutOfOrder(jinglePacket);
489 }
490 }
491
492 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
493 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
494 final RtpContentMap modifiedContentMap =
495 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
496
497 setRemoteContentMap(modifiedContentMap);
498
499 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
500
501 final org.webrtc.SessionDescription sdp =
502 new org.webrtc.SessionDescription(
503 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
504
505 try {
506 this.webRTCWrapper.setRemoteDescription(sdp).get();
507 } catch (final Exception e) {
508 final Throwable cause = Throwables.getRootCause(e);
509 Log.d(
510 Config.LOGTAG,
511 id.getAccount().getJid().asBareJid()
512 + ": unable to set remote description after receiving content-accept",
513 cause);
514 webRTCWrapper.close();
515 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
516 return;
517 }
518 processCandidates(receivedContentAccept.contents.entrySet());
519 updateEndUserState();
520 Log.d(
521 Config.LOGTAG,
522 id.getAccount().getJid().asBareJid()
523 + ": remote has accepted content-add "
524 + ContentAddition.summary(receivedContentAccept));
525 }
526
527 private void receiveContentModify(final JinglePacket jinglePacket) {
528 // TODO check session accepted
529 final Map<String, Content.Senders> modification =
530 Maps.transformEntries(
531 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
532 final boolean isInitiator = isInitiator();
533 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
534 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
535 final Set<String> currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet();
536 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
537 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
538 respondOk(jinglePacket);
539 final RtpContentMap modifiedContentMap;
540 try {
541 modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification);
542 } catch (final IllegalArgumentException e) {
543 webRTCWrapper.close();
544 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
545 return;
546 }
547 this.outgoingContentAdd = modifiedContentMap;
548 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add");
549 } else if (remoteContentMap != null && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
550 respondOk(jinglePacket);
551 final RtpContentMap modifiedRemoteContentMap;
552 try {
553 modifiedRemoteContentMap = remoteContentMap.modifiedSendersChecked(isInitiator, modification);
554 } catch (final IllegalArgumentException e) {
555 webRTCWrapper.close();
556 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
557 return;
558 }
559 final SessionDescription offer;
560 try {
561 offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator());
562 } catch (final IllegalArgumentException | NullPointerException e) {
563 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-modify to SDP", e);
564 webRTCWrapper.close();
565 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
566 return;
567 }
568 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": auto accepting content-modification");
569 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
570 } else {
571 Log.d(Config.LOGTAG,"received unsupported content modification "+modification);
572 respondWithItemNotFound(jinglePacket);
573 }
574 }
575
576 private void autoAcceptContentModify(final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
577 this.setRemoteContentMap(modifiedRemoteContentMap);
578 final org.webrtc.SessionDescription sdp =
579 new org.webrtc.SessionDescription(
580 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
581 try {
582 this.webRTCWrapper.setRemoteDescription(sdp).get();
583 // auto accept is only done when we already have tracks
584 final SessionDescription answer = setLocalSessionDescription();
585 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
586 modifyLocalContentMap(rtpContentMap);
587 // we do not need to send an answer but do we have to resend the candidates currently in SDP?
588 //resendCandidatesFromSdp(answer);
589 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
590 } catch (final Exception e) {
591 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
592 webRTCWrapper.close();
593 sendSessionTerminate(Reason.FAILED_APPLICATION);
594 }
595 }
596
597 private void resendCandidatesFromSdp(final SessionDescription answer) {
598 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
599 for(final SessionDescription.Media media : answer.media) {
600 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
601 if (Strings.isNullOrEmpty(mid)) {
602 continue;
603 }
604 for(final String sdpCandidate : media.attributes.get("candidate")) {
605 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
606 if (candidate != null) {
607 candidateBuilder.put(mid,candidate);
608 }
609 }
610 }
611 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build();
612 sendTransportInfo(candidates);
613 }
614
615 private void receiveContentReject(final JinglePacket jinglePacket) {
616 final RtpContentMap receivedContentReject;
617 try {
618 receivedContentReject = RtpContentMap.of(jinglePacket);
619 } catch (final RuntimeException e) {
620 Log.d(
621 Config.LOGTAG,
622 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
623 Throwables.getRootCause(e));
624 respondOk(jinglePacket);
625 this.webRTCWrapper.close();
626 sendSessionTerminate(Reason.of(e), e.getMessage());
627 return;
628 }
629
630 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
631 if (outgoingContentAdd == null) {
632 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
633 terminateWithOutOfOrder(jinglePacket);
634 return;
635 }
636 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
637 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
638 this.outgoingContentAdd = null;
639 respondOk(jinglePacket);
640 Log.d(Config.LOGTAG,jinglePacket.toString());
641 receiveContentReject(ourSummary);
642 } else {
643 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
644 terminateWithOutOfOrder(jinglePacket);
645 }
646 }
647
648 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
649 try {
650 this.webRTCWrapper.removeTrack(Media.VIDEO);
651 final RtpContentMap localContentMap = customRollback();
652 modifyLocalContentMap(localContentMap);
653 } catch (final Exception e) {
654 final Throwable cause = Throwables.getRootCause(e);
655 Log.d(
656 Config.LOGTAG,
657 id.getAccount().getJid().asBareJid()
658 + ": unable to rollback local description after receiving content-reject",
659 cause);
660 webRTCWrapper.close();
661 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
662 return;
663 }
664 Log.d(
665 Config.LOGTAG,
666 id.getAccount().getJid().asBareJid()
667 + ": remote has rejected our content-add "
668 + summary);
669 }
670
671 private void receiveContentRemove(final JinglePacket jinglePacket) {
672 final RtpContentMap receivedContentRemove;
673 try {
674 receivedContentRemove = RtpContentMap.of(jinglePacket);
675 receivedContentRemove.requireContentDescriptions();
676 } catch (final RuntimeException e) {
677 Log.d(
678 Config.LOGTAG,
679 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
680 Throwables.getRootCause(e));
681 respondOk(jinglePacket);
682 this.webRTCWrapper.close();
683 sendSessionTerminate(Reason.of(e), e.getMessage());
684 return;
685 }
686 respondOk(jinglePacket);
687 receiveContentRemove(receivedContentRemove);
688 }
689
690 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
691 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
692 final Set<ContentAddition.Summary> contentAddSummary =
693 incomingContentAdd == null
694 ? Collections.emptySet()
695 : ContentAddition.summary(incomingContentAdd);
696 final Set<ContentAddition.Summary> removeSummary =
697 ContentAddition.summary(receivedContentRemove);
698 if (contentAddSummary.equals(removeSummary)) {
699 this.incomingContentAdd = null;
700 updateEndUserState();
701 } else {
702 webRTCWrapper.close();
703 sendSessionTerminate(
704 Reason.FAILED_APPLICATION,
705 String.format(
706 "%s only supports %s as a means to retract a not yet accepted %s",
707 BuildConfig.APP_NAME,
708 JinglePacket.Action.CONTENT_REMOVE,
709 JinglePacket.Action.CONTENT_ADD));
710 }
711 }
712
713 public synchronized void retractContentAdd() {
714 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
715 if (outgoingContentAdd == null) {
716 throw new IllegalStateException("Not outgoing content add");
717 }
718 try {
719 webRTCWrapper.removeTrack(Media.VIDEO);
720 final RtpContentMap localContentMap = customRollback();
721 modifyLocalContentMap(localContentMap);
722 } catch (final Exception e) {
723 final Throwable cause = Throwables.getRootCause(e);
724 Log.d(
725 Config.LOGTAG,
726 id.getAccount().getJid().asBareJid()
727 + ": unable to rollback local description after trying to retract content-add",
728 cause);
729 webRTCWrapper.close();
730 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
731 return;
732 }
733 this.outgoingContentAdd = null;
734 final JinglePacket retract =
735 outgoingContentAdd
736 .toStub()
737 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
738 this.send(retract);
739 Log.d(
740 Config.LOGTAG,
741 id.getAccount().getJid()
742 + ": retract content-add "
743 + ContentAddition.summary(outgoingContentAdd));
744 }
745
746 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
747 final SessionDescription sdp = setLocalSessionDescription();
748 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
749 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
750 this.webRTCWrapper
751 .setRemoteDescription(
752 new org.webrtc.SessionDescription(
753 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
754 .get();
755 return localRtpContentMap;
756 }
757
758 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
759 final RtpContentMap currentRemote = getRemoteContentMap();
760 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
761 if (diff.isEmpty()) {
762 throw new IllegalStateException(
763 "Unexpected rollback condition. No difference between local and remote");
764 }
765 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
766 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
767 final RtpContentMap nextRemote =
768 currentRemote.addContent(
769 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
770 return SessionDescription.of(nextRemote, !isInitiator());
771 }
772 throw new IllegalStateException(
773 "Unexpected rollback condition. Senders were not uniformly none");
774 }
775
776 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
777 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
778 if (incomingContentAdd == null) {
779 throw new IllegalStateException("No incoming content add");
780 }
781
782 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
783 this.incomingContentAdd = null;
784 acceptContentAdd(contentAddition, incomingContentAdd);
785 } else {
786 throw new IllegalStateException("Accepted content add does not match pending content-add");
787 }
788 }
789
790 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
791 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
792 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
793 this.setRemoteContentMap(modifiedContentMap);
794
795 final SessionDescription offer;
796 try {
797 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
798 } catch (final IllegalArgumentException | NullPointerException e) {
799 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
800 webRTCWrapper.close();
801 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
802 return;
803 }
804 this.incomingContentAdd = null;
805 acceptContentAdd(contentAddition, offer);
806 }
807
808 private void acceptContentAdd(
809 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
810 final org.webrtc.SessionDescription sdp =
811 new org.webrtc.SessionDescription(
812 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
813 try {
814 this.webRTCWrapper.setRemoteDescription(sdp).get();
815
816 // TODO add tracks for 'media' where contentAddition.senders matches
817
818 // TODO if senders.sending(isInitiator())
819
820 this.webRTCWrapper.addTrack(Media.VIDEO);
821
822 // TODO add additional transceivers for recv only cases
823
824 final SessionDescription answer = setLocalSessionDescription();
825 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
826
827 final RtpContentMap contentAcceptMap =
828 rtpContentMap.toContentModification(
829 Collections2.transform(contentAddition, ca -> ca.name));
830 Log.d(
831 Config.LOGTAG,
832 id.getAccount().getJid().asBareJid()
833 + ": sending content-accept "
834 + ContentAddition.summary(contentAcceptMap));
835 modifyLocalContentMap(rtpContentMap);
836 sendContentAccept(contentAcceptMap);
837 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
838 } catch (final Exception e) {
839 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
840 webRTCWrapper.close();
841 sendSessionTerminate(Reason.FAILED_APPLICATION);
842 }
843 }
844
845 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
846 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
847 send(jinglePacket);
848 }
849
850 public synchronized void rejectContentAdd() {
851 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
852 if (incomingContentAdd == null) {
853 throw new IllegalStateException("No incoming content add");
854 }
855 this.incomingContentAdd = null;
856 updateEndUserState();
857 rejectContentAdd(incomingContentAdd);
858 }
859
860 private void rejectContentAdd(final RtpContentMap contentMap) {
861 final JinglePacket jinglePacket =
862 contentMap
863 .toStub()
864 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
865 Log.d(
866 Config.LOGTAG,
867 id.getAccount().getJid().asBareJid()
868 + ": rejecting content "
869 + ContentAddition.summary(contentMap));
870 send(jinglePacket);
871 }
872
873 private boolean checkForIceRestart(
874 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
875 final RtpContentMap existing = getRemoteContentMap();
876 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
877 final IceUdpTransportInfo.Credentials newCredentials;
878 try {
879 existingCredentials = existing.getCredentials();
880 newCredentials = rtpContentMap.getDistinctCredentials();
881 } catch (final IllegalStateException e) {
882 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
883 return false;
884 }
885 if (existingCredentials.contains(newCredentials)) {
886 return false;
887 }
888 // TODO an alternative approach is to check if we already got an iq result to our
889 // ICE-restart
890 // and if that's the case we are seeing an answer.
891 // This might be more spec compliant but also more error prone potentially
892 final boolean isSignalStateStable = this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
893 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
894 final boolean isOffer = rtpContentMap.emptyCandidates();
895 final RtpContentMap restartContentMap;
896 try {
897 if (isOffer) {
898 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
899 restartContentMap =
900 existing.modifiedCredentials(
901 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
902 } else {
903 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
904 Log.d(
905 Config.LOGTAG,
906 "received confirmation of ICE restart"
907 + newCredentials
908 + " peer_setup="
909 + setup);
910 // DTLS setup attribute needs to be rewritten to reflect current peer state
911 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
912 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
913 }
914 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
915 return isOffer;
916 } else {
917 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
918 respondWithTieBreak(jinglePacket);
919 return true;
920 }
921 } catch (final Exception exception) {
922 respondOk(jinglePacket);
923 final Throwable rootCause = Throwables.getRootCause(exception);
924 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
925 // If this happens a termination is already in progress
926 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
927 return true;
928 }
929 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
930 webRTCWrapper.close();
931 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
932 return true;
933 }
934 }
935
936 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
937 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
938 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
939 throw new IllegalStateException("Invalid peer setup");
940 }
941 return peerSetup;
942 }
943
944 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
945 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
946 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
947 }
948 this.peerDtlsSetup = setup;
949 }
950
951 private boolean applyIceRestart(
952 final JinglePacket jinglePacket,
953 final RtpContentMap restartContentMap,
954 final boolean isOffer)
955 throws ExecutionException, InterruptedException {
956 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
957 final org.webrtc.SessionDescription.Type type =
958 isOffer
959 ? org.webrtc.SessionDescription.Type.OFFER
960 : org.webrtc.SessionDescription.Type.ANSWER;
961 org.webrtc.SessionDescription sdp =
962 new org.webrtc.SessionDescription(type, sessionDescription.toString());
963 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
964 if (isInitiator()) {
965 // We ignore the offer and respond with tie-break. This will clause the responder
966 // not to apply the content map
967 return false;
968 }
969 }
970 webRTCWrapper.setRemoteDescription(sdp).get();
971 setRemoteContentMap(restartContentMap);
972 if (isOffer) {
973 final SessionDescription localSessionDescription = setLocalSessionDescription();
974 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
975 // We need to respond OK before sending any candidates
976 respondOk(jinglePacket);
977 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
978 } else {
979 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
980 }
981 return true;
982 }
983
984 private void processCandidates(
985 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
986 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
987 processCandidate(content);
988 }
989 }
990
991 private void processCandidate(
992 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
993 final RtpContentMap rtpContentMap = getRemoteContentMap();
994 final List<String> indices = toIdentificationTags(rtpContentMap);
995 final String sdpMid = content.getKey(); // aka content name
996 final IceUdpTransportInfo transport = content.getValue().transport;
997 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
998
999 // TODO check that credentials remained the same
1000
1001 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1002 final String sdp;
1003 try {
1004 sdp = candidate.toSdpAttribute(credentials.ufrag);
1005 } catch (final IllegalArgumentException e) {
1006 Log.d(
1007 Config.LOGTAG,
1008 id.account.getJid().asBareJid()
1009 + ": ignoring invalid ICE candidate "
1010 + e.getMessage());
1011 continue;
1012 }
1013 final int mLineIndex = indices.indexOf(sdpMid);
1014 if (mLineIndex < 0) {
1015 Log.w(
1016 Config.LOGTAG,
1017 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1018 }
1019 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1020 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1021 this.webRTCWrapper.addIceCandidate(iceCandidate);
1022 }
1023 }
1024
1025 private RtpContentMap getRemoteContentMap() {
1026 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1027 }
1028
1029 private RtpContentMap getLocalContentMap() {
1030 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1031 }
1032
1033 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1034 final Group originalGroup = rtpContentMap.group;
1035 final List<String> identificationTags =
1036 originalGroup == null
1037 ? rtpContentMap.getNames()
1038 : originalGroup.getIdentificationTags();
1039 if (identificationTags.size() == 0) {
1040 Log.w(
1041 Config.LOGTAG,
1042 id.account.getJid().asBareJid()
1043 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1044 }
1045 return identificationTags;
1046 }
1047
1048 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1049 final JinglePacket jinglePacket, final boolean expectVerification) {
1050 final RtpContentMap receivedContentMap;
1051 try {
1052 receivedContentMap = RtpContentMap.of(jinglePacket);
1053 } catch (final Exception e) {
1054 return Futures.immediateFailedFuture(e);
1055 }
1056 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1057 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1058 id.account
1059 .getAxolotlService()
1060 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1061 return Futures.transform(
1062 future,
1063 omemoVerifiedPayload -> {
1064 // TODO test if an exception here triggers a correct abort
1065 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1066 Log.d(
1067 Config.LOGTAG,
1068 id.account.getJid().asBareJid()
1069 + ": received verifiable DTLS fingerprint via "
1070 + omemoVerification);
1071 return omemoVerifiedPayload.getPayload();
1072 },
1073 MoreExecutors.directExecutor());
1074 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1075 return Futures.immediateFailedFuture(
1076 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1077 } else {
1078 return Futures.immediateFuture(receivedContentMap);
1079 }
1080 }
1081
1082 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1083 if (isInitiator()) {
1084 Log.d(
1085 Config.LOGTAG,
1086 String.format(
1087 "%s: received session-initiate even though we were initiating",
1088 id.account.getJid().asBareJid()));
1089 if (isTerminated()) {
1090 Log.d(
1091 Config.LOGTAG,
1092 String.format(
1093 "%s: got a reason to terminate with out-of-order. but already in state %s",
1094 id.account.getJid().asBareJid(), getState()));
1095 respondWithOutOfOrder(jinglePacket);
1096 } else {
1097 terminateWithOutOfOrder(jinglePacket);
1098 }
1099 return;
1100 }
1101 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1102 Futures.addCallback(
1103 future,
1104 new FutureCallback<RtpContentMap>() {
1105 @Override
1106 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1107 receiveSessionInitiate(jinglePacket, rtpContentMap);
1108 }
1109
1110 @Override
1111 public void onFailure(@NonNull final Throwable throwable) {
1112 respondOk(jinglePacket);
1113 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1114 }
1115 },
1116 MoreExecutors.directExecutor());
1117 }
1118
1119 private void receiveSessionInitiate(
1120 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1121 try {
1122 contentMap.requireContentDescriptions();
1123 contentMap.requireDTLSFingerprint(true);
1124 } catch (final RuntimeException e) {
1125 Log.d(
1126 Config.LOGTAG,
1127 id.account.getJid().asBareJid() + ": improperly formatted contents",
1128 Throwables.getRootCause(e));
1129 respondOk(jinglePacket);
1130 sendSessionTerminate(Reason.of(e), e.getMessage());
1131 return;
1132 }
1133 Log.d(
1134 Config.LOGTAG,
1135 "processing session-init with " + contentMap.contents.size() + " contents");
1136 final State target;
1137 if (this.state == State.PROCEED) {
1138 Preconditions.checkState(
1139 proposedMedia != null && proposedMedia.size() > 0,
1140 "proposed media must be set when processing pre-approved session-initiate");
1141 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1142 sendSessionTerminate(
1143 Reason.SECURITY_ERROR,
1144 String.format(
1145 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1146 this.proposedMedia, contentMap.getMedia()));
1147 return;
1148 }
1149 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1150 } else {
1151 target = State.SESSION_INITIALIZED;
1152 }
1153 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1154 respondOk(jinglePacket);
1155 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1156 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1157 Log.d(
1158 Config.LOGTAG,
1159 id.account.getJid().asBareJid()
1160 + ": automatically accepting session-initiate");
1161 sendSessionAccept();
1162 } else {
1163 Log.d(
1164 Config.LOGTAG,
1165 id.account.getJid().asBareJid()
1166 + ": received not pre-approved session-initiate. start ringing");
1167 startRinging();
1168 }
1169 } else {
1170 Log.d(
1171 Config.LOGTAG,
1172 String.format(
1173 "%s: received session-initiate while in state %s",
1174 id.account.getJid().asBareJid(), state));
1175 terminateWithOutOfOrder(jinglePacket);
1176 }
1177 }
1178
1179 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1180 if (!isInitiator()) {
1181 Log.d(
1182 Config.LOGTAG,
1183 String.format(
1184 "%s: received session-accept even though we were responding",
1185 id.account.getJid().asBareJid()));
1186 terminateWithOutOfOrder(jinglePacket);
1187 return;
1188 }
1189 final ListenableFuture<RtpContentMap> future =
1190 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1191 Futures.addCallback(
1192 future,
1193 new FutureCallback<RtpContentMap>() {
1194 @Override
1195 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1196 receiveSessionAccept(jinglePacket, rtpContentMap);
1197 }
1198
1199 @Override
1200 public void onFailure(@NonNull final Throwable throwable) {
1201 respondOk(jinglePacket);
1202 Log.d(
1203 Config.LOGTAG,
1204 id.account.getJid().asBareJid()
1205 + ": improperly formatted contents in session-accept",
1206 throwable);
1207 webRTCWrapper.close();
1208 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1209 }
1210 },
1211 MoreExecutors.directExecutor());
1212 }
1213
1214 private void receiveSessionAccept(
1215 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1216 try {
1217 contentMap.requireContentDescriptions();
1218 contentMap.requireDTLSFingerprint();
1219 } catch (final RuntimeException e) {
1220 respondOk(jinglePacket);
1221 Log.d(
1222 Config.LOGTAG,
1223 id.account.getJid().asBareJid()
1224 + ": improperly formatted contents in session-accept",
1225 e);
1226 webRTCWrapper.close();
1227 sendSessionTerminate(Reason.of(e), e.getMessage());
1228 return;
1229 }
1230 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1231 if (!initiatorMedia.equals(contentMap.getMedia())) {
1232 sendSessionTerminate(
1233 Reason.SECURITY_ERROR,
1234 String.format(
1235 "Your session-included included media %s but our session-initiate was %s",
1236 this.proposedMedia, contentMap.getMedia()));
1237 return;
1238 }
1239 Log.d(
1240 Config.LOGTAG,
1241 "processing session-accept with " + contentMap.contents.size() + " contents");
1242 if (transition(State.SESSION_ACCEPTED)) {
1243 respondOk(jinglePacket);
1244 receiveSessionAccept(contentMap);
1245 } else {
1246 Log.d(
1247 Config.LOGTAG,
1248 String.format(
1249 "%s: received session-accept while in state %s",
1250 id.account.getJid().asBareJid(), state));
1251 respondOk(jinglePacket);
1252 }
1253 }
1254
1255 private void receiveSessionAccept(final RtpContentMap contentMap) {
1256 this.responderRtpContentMap = contentMap;
1257 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1258 final SessionDescription sessionDescription;
1259 try {
1260 sessionDescription = SessionDescription.of(contentMap, false);
1261 } catch (final IllegalArgumentException | NullPointerException e) {
1262 Log.d(
1263 Config.LOGTAG,
1264 id.account.getJid().asBareJid()
1265 + ": unable convert offer from session-accept to SDP",
1266 e);
1267 webRTCWrapper.close();
1268 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1269 return;
1270 }
1271 final org.webrtc.SessionDescription answer =
1272 new org.webrtc.SessionDescription(
1273 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1274 try {
1275 this.webRTCWrapper.setRemoteDescription(answer).get();
1276 } catch (final Exception e) {
1277 Log.d(
1278 Config.LOGTAG,
1279 id.account.getJid().asBareJid()
1280 + ": unable to set remote description after receiving session-accept",
1281 Throwables.getRootCause(e));
1282 webRTCWrapper.close();
1283 sendSessionTerminate(
1284 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1285 return;
1286 }
1287 processCandidates(contentMap.contents.entrySet());
1288 }
1289
1290 private void sendSessionAccept() {
1291 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1292 if (rtpContentMap == null) {
1293 throw new IllegalStateException("initiator RTP Content Map has not been set");
1294 }
1295 final SessionDescription offer;
1296 try {
1297 offer = SessionDescription.of(rtpContentMap, true);
1298 } catch (final IllegalArgumentException | NullPointerException e) {
1299 Log.d(
1300 Config.LOGTAG,
1301 id.account.getJid().asBareJid()
1302 + ": unable convert offer from session-initiate to SDP",
1303 e);
1304 webRTCWrapper.close();
1305 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1306 return;
1307 }
1308 sendSessionAccept(rtpContentMap.getMedia(), offer);
1309 }
1310
1311 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1312 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1313 }
1314
1315 private synchronized void sendSessionAccept(
1316 final Set<Media> media,
1317 final SessionDescription offer,
1318 final List<PeerConnection.IceServer> iceServers) {
1319 if (isTerminated()) {
1320 Log.w(
1321 Config.LOGTAG,
1322 id.account.getJid().asBareJid()
1323 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1324 return;
1325 }
1326 try {
1327 setupWebRTC(media, iceServers);
1328 } catch (final WebRTCWrapper.InitializationException e) {
1329 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1330 webRTCWrapper.close();
1331 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1332 return;
1333 }
1334 final org.webrtc.SessionDescription sdp =
1335 new org.webrtc.SessionDescription(
1336 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1337 try {
1338 this.webRTCWrapper.setRemoteDescription(sdp).get();
1339 addIceCandidatesFromBlackLog();
1340 org.webrtc.SessionDescription webRTCSessionDescription =
1341 this.webRTCWrapper.setLocalDescription().get();
1342 prepareSessionAccept(webRTCSessionDescription);
1343 } catch (final Exception e) {
1344 failureToAcceptSession(e);
1345 }
1346 }
1347
1348 private void failureToAcceptSession(final Throwable throwable) {
1349 if (isTerminated()) {
1350 return;
1351 }
1352 final Throwable rootCause = Throwables.getRootCause(throwable);
1353 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1354 webRTCWrapper.close();
1355 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1356 }
1357
1358 private void addIceCandidatesFromBlackLog() {
1359 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1360 while ((foo = this.pendingIceCandidates.poll()) != null) {
1361 processCandidate(foo);
1362 Log.d(
1363 Config.LOGTAG,
1364 id.account.getJid().asBareJid() + ": added candidate from back log");
1365 }
1366 }
1367
1368 private void prepareSessionAccept(
1369 final org.webrtc.SessionDescription webRTCSessionDescription) {
1370 final SessionDescription sessionDescription =
1371 SessionDescription.parse(webRTCSessionDescription.description);
1372 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1373 this.responderRtpContentMap = respondingRtpContentMap;
1374 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1375 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1376 prepareOutgoingContentMap(respondingRtpContentMap);
1377 Futures.addCallback(
1378 outgoingContentMapFuture,
1379 new FutureCallback<RtpContentMap>() {
1380 @Override
1381 public void onSuccess(final RtpContentMap outgoingContentMap) {
1382 sendSessionAccept(outgoingContentMap);
1383 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1384 }
1385
1386 @Override
1387 public void onFailure(@NonNull Throwable throwable) {
1388 failureToAcceptSession(throwable);
1389 }
1390 },
1391 MoreExecutors.directExecutor());
1392 }
1393
1394 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1395 if (isTerminated()) {
1396 Log.w(
1397 Config.LOGTAG,
1398 id.account.getJid().asBareJid()
1399 + ": preparing session accept was too slow. already terminated. nothing to do.");
1400 return;
1401 }
1402 transitionOrThrow(State.SESSION_ACCEPTED);
1403 final JinglePacket sessionAccept =
1404 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1405 send(sessionAccept);
1406 }
1407
1408 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1409 final RtpContentMap rtpContentMap) {
1410 if (this.omemoVerification.hasDeviceId()) {
1411 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1412 verifiedPayloadFuture =
1413 id.account
1414 .getAxolotlService()
1415 .encrypt(
1416 rtpContentMap,
1417 id.with,
1418 omemoVerification.getDeviceId());
1419 return Futures.transform(
1420 verifiedPayloadFuture,
1421 verifiedPayload -> {
1422 omemoVerification.setOrEnsureEqual(verifiedPayload);
1423 return verifiedPayload.getPayload();
1424 },
1425 MoreExecutors.directExecutor());
1426 } else {
1427 return Futures.immediateFuture(rtpContentMap);
1428 }
1429 }
1430
1431 synchronized void deliveryMessage(
1432 final Jid from,
1433 final Element message,
1434 final String serverMessageId,
1435 final long timestamp) {
1436 Log.d(
1437 Config.LOGTAG,
1438 id.account.getJid().asBareJid()
1439 + ": delivered message to JingleRtpConnection "
1440 + message);
1441 switch (message.getName()) {
1442 case "propose":
1443 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1444 break;
1445 case "proceed":
1446 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1447 break;
1448 case "retract":
1449 receiveRetract(from, serverMessageId, timestamp);
1450 break;
1451 case "reject":
1452 receiveReject(from, serverMessageId, timestamp);
1453 break;
1454 case "accept":
1455 receiveAccept(from, serverMessageId, timestamp);
1456 break;
1457 default:
1458 break;
1459 }
1460 }
1461
1462 void deliverFailedProceed(final String message) {
1463 Log.d(
1464 Config.LOGTAG,
1465 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1466 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1467 webRTCWrapper.close();
1468 Log.d(
1469 Config.LOGTAG,
1470 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1471 this.finish();
1472 }
1473 }
1474
1475 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1476 final boolean originatedFromMyself =
1477 from.asBareJid().equals(id.account.getJid().asBareJid());
1478 if (originatedFromMyself) {
1479 if (transition(State.ACCEPTED)) {
1480 acceptedOnOtherDevice(serverMsgId, timestamp);
1481 } else {
1482 Log.d(
1483 Config.LOGTAG,
1484 id.account.getJid().asBareJid()
1485 + ": unable to transition to accept because already in state="
1486 + this.state);
1487 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1488 }
1489 } else {
1490 Log.d(
1491 Config.LOGTAG,
1492 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1493 }
1494 }
1495
1496 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1497 if (serverMsgId != null) {
1498 this.message.setServerMsgId(serverMsgId);
1499 }
1500 this.message.setTime(timestamp);
1501 this.message.setCarbon(true); // indicate that call was accepted on other device
1502 this.writeLogMessageSuccess(0);
1503 this.xmppConnectionService
1504 .getNotificationService()
1505 .cancelIncomingCallNotification();
1506 this.finish();
1507 }
1508
1509 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1510 final boolean originatedFromMyself =
1511 from.asBareJid().equals(id.account.getJid().asBareJid());
1512 // reject from another one of my clients
1513 if (originatedFromMyself) {
1514 receiveRejectFromMyself(serverMsgId, timestamp);
1515 } else if (isInitiator()) {
1516 if (from.equals(id.with)) {
1517 receiveRejectFromResponder();
1518 } else {
1519 Log.d(
1520 Config.LOGTAG,
1521 id.account.getJid()
1522 + ": ignoring reject from "
1523 + from
1524 + " for session with "
1525 + id.with);
1526 }
1527 } else {
1528 Log.d(
1529 Config.LOGTAG,
1530 id.account.getJid()
1531 + ": ignoring reject from "
1532 + from
1533 + " for session with "
1534 + id.with);
1535 }
1536 }
1537
1538 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1539 if (transition(State.REJECTED)) {
1540 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1541 this.finish();
1542 if (serverMsgId != null) {
1543 this.message.setServerMsgId(serverMsgId);
1544 }
1545 this.message.setTime(timestamp);
1546 this.message.setCarbon(true); // indicate that call was rejected on other device
1547 writeLogMessageMissed();
1548 } else {
1549 Log.d(
1550 Config.LOGTAG,
1551 "not able to transition into REJECTED because already in " + this.state);
1552 }
1553 }
1554
1555 private void receiveRejectFromResponder() {
1556 if (isInState(State.PROCEED)) {
1557 Log.d(
1558 Config.LOGTAG,
1559 id.account.getJid()
1560 + ": received reject while still in proceed. callee reconsidered");
1561 closeTransitionLogFinish(State.REJECTED_RACED);
1562 return;
1563 }
1564 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1565 Log.d(
1566 Config.LOGTAG,
1567 id.account.getJid()
1568 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1569 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1570 return;
1571 }
1572 Log.d(
1573 Config.LOGTAG,
1574 id.account.getJid()
1575 + ": ignoring reject from responder because already in state "
1576 + this.state);
1577 }
1578
1579 private void receivePropose(
1580 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1581 final boolean originatedFromMyself =
1582 from.asBareJid().equals(id.account.getJid().asBareJid());
1583 if (originatedFromMyself) {
1584 Log.d(
1585 Config.LOGTAG,
1586 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1587 } else if (transition(
1588 State.PROPOSED,
1589 () -> {
1590 final Collection<RtpDescription> descriptions =
1591 Collections2.transform(
1592 Collections2.filter(
1593 propose.getDescriptions(),
1594 d -> d instanceof RtpDescription),
1595 input -> (RtpDescription) input);
1596 final Collection<Media> media =
1597 Collections2.transform(descriptions, RtpDescription::getMedia);
1598 Preconditions.checkState(
1599 !media.contains(Media.UNKNOWN),
1600 "RTP descriptions contain unknown media");
1601 Log.d(
1602 Config.LOGTAG,
1603 id.account.getJid().asBareJid()
1604 + ": received session proposal from "
1605 + from
1606 + " for "
1607 + media);
1608 this.proposedMedia = Sets.newHashSet(media);
1609 })) {
1610 if (serverMsgId != null) {
1611 this.message.setServerMsgId(serverMsgId);
1612 }
1613 this.message.setTime(timestamp);
1614 startRinging();
1615 } else {
1616 Log.d(
1617 Config.LOGTAG,
1618 id.account.getJid()
1619 + ": ignoring session proposal because already in "
1620 + state);
1621 }
1622 }
1623
1624 private void startRinging() {
1625 Log.d(
1626 Config.LOGTAG,
1627 id.account.getJid().asBareJid()
1628 + ": received call from "
1629 + id.with
1630 + ". start ringing");
1631 ringingTimeoutFuture =
1632 jingleConnectionManager.schedule(
1633 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1634 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1635 }
1636
1637 private synchronized void ringingTimeout() {
1638 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1639 switch (this.state) {
1640 case PROPOSED:
1641 message.markUnread();
1642 rejectCallFromProposed();
1643 break;
1644 case SESSION_INITIALIZED:
1645 message.markUnread();
1646 rejectCallFromSessionInitiate();
1647 break;
1648 }
1649 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1650 }
1651
1652 private void cancelRingingTimeout() {
1653 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1654 if (future != null && !future.isCancelled()) {
1655 future.cancel(false);
1656 }
1657 }
1658
1659 private void receiveProceed(
1660 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1661 final Set<Media> media =
1662 Preconditions.checkNotNull(
1663 this.proposedMedia, "Proposed media has to be set before handling proceed");
1664 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1665 if (from.equals(id.with)) {
1666 if (isInitiator()) {
1667 if (transition(State.PROCEED)) {
1668 if (serverMsgId != null) {
1669 this.message.setServerMsgId(serverMsgId);
1670 }
1671 this.message.setTime(timestamp);
1672 final Integer remoteDeviceId = proceed.getDeviceId();
1673 if (isOmemoEnabled()) {
1674 this.omemoVerification.setDeviceId(remoteDeviceId);
1675 } else {
1676 if (remoteDeviceId != null) {
1677 Log.d(
1678 Config.LOGTAG,
1679 id.account.getJid().asBareJid()
1680 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1681 }
1682 this.omemoVerification.setDeviceId(null);
1683 }
1684 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1685 } else {
1686 Log.d(
1687 Config.LOGTAG,
1688 String.format(
1689 "%s: ignoring proceed because already in %s",
1690 id.account.getJid().asBareJid(), this.state));
1691 }
1692 } else {
1693 Log.d(
1694 Config.LOGTAG,
1695 String.format(
1696 "%s: ignoring proceed because we were not initializing",
1697 id.account.getJid().asBareJid()));
1698 }
1699 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1700 if (transition(State.ACCEPTED)) {
1701 Log.d(
1702 Config.LOGTAG,
1703 id.account.getJid().asBareJid()
1704 + ": moved session with "
1705 + id.with
1706 + " into state accepted after received carbon copied proceed");
1707 acceptedOnOtherDevice(serverMsgId, timestamp);
1708 }
1709 } else {
1710 Log.d(
1711 Config.LOGTAG,
1712 String.format(
1713 "%s: ignoring proceed from %s. was expected from %s",
1714 id.account.getJid().asBareJid(), from, id.with));
1715 }
1716 }
1717
1718 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1719 if (from.equals(id.with)) {
1720 final State target =
1721 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1722 if (transition(target)) {
1723 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1724 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1725 Log.d(
1726 Config.LOGTAG,
1727 id.account.getJid().asBareJid()
1728 + ": session with "
1729 + id.with
1730 + " has been retracted (serverMsgId="
1731 + serverMsgId
1732 + ")");
1733 if (serverMsgId != null) {
1734 this.message.setServerMsgId(serverMsgId);
1735 }
1736 this.message.setTime(timestamp);
1737 if (target == State.RETRACTED) {
1738 this.message.markUnread();
1739 }
1740 writeLogMessageMissed();
1741 finish();
1742 } else {
1743 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1744 }
1745 } else {
1746 // TODO parse retract from self
1747 Log.d(
1748 Config.LOGTAG,
1749 id.account.getJid().asBareJid()
1750 + ": received retract from "
1751 + from
1752 + ". expected retract from"
1753 + id.with
1754 + ". ignoring");
1755 }
1756 }
1757
1758 public void sendSessionInitiate() {
1759 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1760 }
1761
1762 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1763 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1764 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1765 }
1766
1767 private synchronized void sendSessionInitiate(
1768 final Set<Media> media,
1769 final State targetState,
1770 final List<PeerConnection.IceServer> iceServers) {
1771 if (isTerminated()) {
1772 Log.w(
1773 Config.LOGTAG,
1774 id.account.getJid().asBareJid()
1775 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1776 return;
1777 }
1778 try {
1779 setupWebRTC(media, iceServers);
1780 } catch (final WebRTCWrapper.InitializationException e) {
1781 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1782 webRTCWrapper.close();
1783 sendRetract(Reason.ofThrowable(e));
1784 return;
1785 }
1786 try {
1787 org.webrtc.SessionDescription webRTCSessionDescription =
1788 this.webRTCWrapper.setLocalDescription().get();
1789 prepareSessionInitiate(webRTCSessionDescription, targetState);
1790 } catch (final Exception e) {
1791 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1792 // exceptions
1793 failureToInitiateSession(e, targetState);
1794 }
1795 }
1796
1797 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1798 if (isTerminated()) {
1799 return;
1800 }
1801 Log.d(
1802 Config.LOGTAG,
1803 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1804 Throwables.getRootCause(throwable));
1805 webRTCWrapper.close();
1806 final Reason reason = Reason.ofThrowable(throwable);
1807 if (isInState(targetState)) {
1808 sendSessionTerminate(reason, throwable.getMessage());
1809 } else {
1810 sendRetract(reason);
1811 }
1812 }
1813
1814 private void sendRetract(final Reason reason) {
1815 // TODO embed reason into retract
1816 sendJingleMessage("retract", id.with.asBareJid());
1817 transitionOrThrow(reasonToState(reason));
1818 this.finish();
1819 }
1820
1821 private void prepareSessionInitiate(
1822 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1823 final SessionDescription sessionDescription =
1824 SessionDescription.parse(webRTCSessionDescription.description);
1825 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1826 this.initiatorRtpContentMap = rtpContentMap;
1827 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1828 encryptSessionInitiate(rtpContentMap);
1829 Futures.addCallback(
1830 outgoingContentMapFuture,
1831 new FutureCallback<RtpContentMap>() {
1832 @Override
1833 public void onSuccess(final RtpContentMap outgoingContentMap) {
1834 sendSessionInitiate(outgoingContentMap, targetState);
1835 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1836 }
1837
1838 @Override
1839 public void onFailure(@NonNull final Throwable throwable) {
1840 failureToInitiateSession(throwable, targetState);
1841 }
1842 },
1843 MoreExecutors.directExecutor());
1844 }
1845
1846 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1847 if (isTerminated()) {
1848 Log.w(
1849 Config.LOGTAG,
1850 id.account.getJid().asBareJid()
1851 + ": preparing session was too slow. already terminated. nothing to do.");
1852 return;
1853 }
1854 this.transitionOrThrow(targetState);
1855 final JinglePacket sessionInitiate =
1856 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1857 send(sessionInitiate);
1858 }
1859
1860 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1861 final RtpContentMap rtpContentMap) {
1862 if (this.omemoVerification.hasDeviceId()) {
1863 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1864 verifiedPayloadFuture =
1865 id.account
1866 .getAxolotlService()
1867 .encrypt(
1868 rtpContentMap,
1869 id.with,
1870 omemoVerification.getDeviceId());
1871 final ListenableFuture<RtpContentMap> future =
1872 Futures.transform(
1873 verifiedPayloadFuture,
1874 verifiedPayload -> {
1875 omemoVerification.setSessionFingerprint(
1876 verifiedPayload.getFingerprint());
1877 return verifiedPayload.getPayload();
1878 },
1879 MoreExecutors.directExecutor());
1880 if (Config.REQUIRE_RTP_VERIFICATION) {
1881 return future;
1882 }
1883 return Futures.catching(
1884 future,
1885 CryptoFailedException.class,
1886 e -> {
1887 Log.w(
1888 Config.LOGTAG,
1889 id.account.getJid().asBareJid()
1890 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1891 e);
1892 return rtpContentMap;
1893 },
1894 MoreExecutors.directExecutor());
1895 } else {
1896 return Futures.immediateFuture(rtpContentMap);
1897 }
1898 }
1899
1900 private void sendSessionTerminate(final Reason reason) {
1901 sendSessionTerminate(reason, null);
1902 }
1903
1904 private void sendSessionTerminate(final Reason reason, final String text) {
1905 final State previous = this.state;
1906 final State target = reasonToState(reason);
1907 transitionOrThrow(target);
1908 if (previous != State.NULL) {
1909 writeLogMessage(target);
1910 }
1911 final JinglePacket jinglePacket =
1912 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1913 jinglePacket.setReason(reason, text);
1914 Log.d(Config.LOGTAG, jinglePacket.toString());
1915 send(jinglePacket);
1916 finish();
1917 }
1918
1919 private void sendTransportInfo(
1920 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1921 final RtpContentMap transportInfo;
1922 try {
1923 final RtpContentMap rtpContentMap =
1924 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1925 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1926 } catch (final Exception e) {
1927 Log.d(
1928 Config.LOGTAG,
1929 id.account.getJid().asBareJid()
1930 + ": unable to prepare transport-info from candidate for content="
1931 + contentName);
1932 return;
1933 }
1934 final JinglePacket jinglePacket =
1935 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1936 send(jinglePacket);
1937 }
1938
1939 private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
1940 // TODO send all candidates in one transport-info
1941 }
1942
1943
1944 private void send(final JinglePacket jinglePacket) {
1945 jinglePacket.setTo(id.with);
1946 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1947 }
1948
1949 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1950 if (response.getType() == IqPacket.TYPE.ERROR) {
1951 handleIqErrorResponse(response);
1952 return;
1953 }
1954 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1955 handleIqTimeoutResponse(response);
1956 }
1957 }
1958
1959 private void handleIqErrorResponse(final IqPacket response) {
1960 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1961 final String errorCondition = response.getErrorCondition();
1962 Log.d(
1963 Config.LOGTAG,
1964 id.account.getJid().asBareJid()
1965 + ": received IQ-error from "
1966 + response.getFrom()
1967 + " in RTP session. "
1968 + errorCondition);
1969 if (isTerminated()) {
1970 Log.i(
1971 Config.LOGTAG,
1972 id.account.getJid().asBareJid()
1973 + ": ignoring error because session was already terminated");
1974 return;
1975 }
1976 this.webRTCWrapper.close();
1977 final State target;
1978 if (Arrays.asList(
1979 "service-unavailable",
1980 "recipient-unavailable",
1981 "remote-server-not-found",
1982 "remote-server-timeout")
1983 .contains(errorCondition)) {
1984 target = State.TERMINATED_CONNECTIVITY_ERROR;
1985 } else {
1986 target = State.TERMINATED_APPLICATION_FAILURE;
1987 }
1988 transitionOrThrow(target);
1989 this.finish();
1990 }
1991
1992 private void handleIqTimeoutResponse(final IqPacket response) {
1993 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1994 Log.d(
1995 Config.LOGTAG,
1996 id.account.getJid().asBareJid()
1997 + ": received IQ timeout in RTP session with "
1998 + id.with
1999 + ". terminating with connectivity error");
2000 if (isTerminated()) {
2001 Log.i(
2002 Config.LOGTAG,
2003 id.account.getJid().asBareJid()
2004 + ": ignoring error because session was already terminated");
2005 return;
2006 }
2007 this.webRTCWrapper.close();
2008 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
2009 this.finish();
2010 }
2011
2012 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
2013 Log.d(
2014 Config.LOGTAG,
2015 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
2016 this.webRTCWrapper.close();
2017 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
2018 respondWithOutOfOrder(jinglePacket);
2019 this.finish();
2020 }
2021
2022 private void respondWithTieBreak(final JinglePacket jinglePacket) {
2023 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
2024 }
2025
2026 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
2027 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
2028 }
2029
2030 private void respondWithItemNotFound(final JinglePacket jinglePacket) {
2031 respondWithJingleError(jinglePacket, null, "item-not-found", "cancel");
2032 }
2033
2034 void respondWithJingleError(
2035 final IqPacket original,
2036 String jingleCondition,
2037 String condition,
2038 String conditionType) {
2039 jingleConnectionManager.respondWithJingleError(
2040 id.account, original, jingleCondition, condition, conditionType);
2041 }
2042
2043 private void respondOk(final JinglePacket jinglePacket) {
2044 xmppConnectionService.sendIqPacket(
2045 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
2046 }
2047
2048 public RtpEndUserState getEndUserState() {
2049 switch (this.state) {
2050 case NULL:
2051 case PROPOSED:
2052 case SESSION_INITIALIZED:
2053 if (isInitiator()) {
2054 return RtpEndUserState.RINGING;
2055 } else {
2056 return RtpEndUserState.INCOMING_CALL;
2057 }
2058 case PROCEED:
2059 if (isInitiator()) {
2060 return RtpEndUserState.RINGING;
2061 } else {
2062 return RtpEndUserState.ACCEPTING_CALL;
2063 }
2064 case SESSION_INITIALIZED_PRE_APPROVED:
2065 if (isInitiator()) {
2066 return RtpEndUserState.RINGING;
2067 } else {
2068 return RtpEndUserState.CONNECTING;
2069 }
2070 case SESSION_ACCEPTED:
2071 final ContentAddition ca = getPendingContentAddition();
2072 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2073 return RtpEndUserState.INCOMING_CONTENT_ADD;
2074 }
2075 return getPeerConnectionStateAsEndUserState();
2076 case REJECTED:
2077 case REJECTED_RACED:
2078 case TERMINATED_DECLINED_OR_BUSY:
2079 if (isInitiator()) {
2080 return RtpEndUserState.DECLINED_OR_BUSY;
2081 } else {
2082 return RtpEndUserState.ENDED;
2083 }
2084 case TERMINATED_SUCCESS:
2085 case ACCEPTED:
2086 case RETRACTED:
2087 case TERMINATED_CANCEL_OR_TIMEOUT:
2088 return RtpEndUserState.ENDED;
2089 case RETRACTED_RACED:
2090 if (isInitiator()) {
2091 return RtpEndUserState.ENDED;
2092 } else {
2093 return RtpEndUserState.RETRACTED;
2094 }
2095 case TERMINATED_CONNECTIVITY_ERROR:
2096 return zeroDuration()
2097 ? RtpEndUserState.CONNECTIVITY_ERROR
2098 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2099 case TERMINATED_APPLICATION_FAILURE:
2100 return RtpEndUserState.APPLICATION_ERROR;
2101 case TERMINATED_SECURITY_ERROR:
2102 return RtpEndUserState.SECURITY_ERROR;
2103 }
2104 throw new IllegalStateException(
2105 String.format("%s has no equivalent EndUserState", this.state));
2106 }
2107
2108 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2109 final PeerConnection.PeerConnectionState state;
2110 try {
2111 state = webRTCWrapper.getState();
2112 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2113 // We usually close the WebRTCWrapper *before* transitioning so we might still
2114 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2115 return RtpEndUserState.ENDING_CALL;
2116 }
2117 switch (state) {
2118 case CONNECTED:
2119 return RtpEndUserState.CONNECTED;
2120 case NEW:
2121 case CONNECTING:
2122 return RtpEndUserState.CONNECTING;
2123 case CLOSED:
2124 return RtpEndUserState.ENDING_CALL;
2125 default:
2126 return zeroDuration()
2127 ? RtpEndUserState.CONNECTIVITY_ERROR
2128 : RtpEndUserState.RECONNECTING;
2129 }
2130 }
2131
2132 public ContentAddition getPendingContentAddition() {
2133 final RtpContentMap in = this.incomingContentAdd;
2134 final RtpContentMap out = this.outgoingContentAdd;
2135 if (out != null) {
2136 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2137 } else if (in != null) {
2138 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2139 } else {
2140 return null;
2141 }
2142 }
2143
2144 public Set<Media> getMedia() {
2145 final State current = getState();
2146 if (current == State.NULL) {
2147 if (isInitiator()) {
2148 return Preconditions.checkNotNull(
2149 this.proposedMedia, "RTP connection has not been initialized properly");
2150 }
2151 throw new IllegalStateException("RTP connection has not been initialized yet");
2152 }
2153 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2154 return Preconditions.checkNotNull(
2155 this.proposedMedia, "RTP connection has not been initialized properly");
2156 }
2157 final RtpContentMap localContentMap = getLocalContentMap();
2158 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2159 if (localContentMap != null) {
2160 return localContentMap.getMedia();
2161 } else if (initiatorContentMap != null) {
2162 return initiatorContentMap.getMedia();
2163 } else if (isTerminated()) {
2164 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2165 } else {
2166 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2167 }
2168 }
2169
2170 public boolean isVerified() {
2171 final String fingerprint = this.omemoVerification.getFingerprint();
2172 if (fingerprint == null) {
2173 return false;
2174 }
2175 final FingerprintStatus status =
2176 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2177 return status != null && status.isVerified();
2178 }
2179
2180 public boolean addMedia(final Media media) {
2181 final Set<Media> currentMedia = getMedia();
2182 if (currentMedia.contains(media)) {
2183 throw new IllegalStateException(String.format("%s has already been proposed", media));
2184 }
2185 // TODO add state protection - can only add while ACCEPTED or so
2186 Log.d(Config.LOGTAG,"adding media: "+media);
2187 return webRTCWrapper.addTrack(media);
2188 }
2189
2190 public synchronized void acceptCall() {
2191 switch (this.state) {
2192 case PROPOSED:
2193 cancelRingingTimeout();
2194 acceptCallFromProposed();
2195 break;
2196 case SESSION_INITIALIZED:
2197 cancelRingingTimeout();
2198 acceptCallFromSessionInitialized();
2199 break;
2200 case ACCEPTED:
2201 Log.w(
2202 Config.LOGTAG,
2203 id.account.getJid().asBareJid()
2204 + ": the call has already been accepted with another client. UI was just lagging behind");
2205 break;
2206 case PROCEED:
2207 case SESSION_ACCEPTED:
2208 Log.w(
2209 Config.LOGTAG,
2210 id.account.getJid().asBareJid()
2211 + ": the call has already been accepted. user probably double tapped the UI");
2212 break;
2213 default:
2214 throw new IllegalStateException("Can not accept call from " + this.state);
2215 }
2216 }
2217
2218 public void notifyPhoneCall() {
2219 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2220 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2221 rejectCall();
2222 } else {
2223 endCall();
2224 }
2225 }
2226
2227 public synchronized void rejectCall() {
2228 if (isTerminated()) {
2229 Log.w(
2230 Config.LOGTAG,
2231 id.account.getJid().asBareJid()
2232 + ": received rejectCall() when session has already been terminated. nothing to do");
2233 return;
2234 }
2235 switch (this.state) {
2236 case PROPOSED:
2237 rejectCallFromProposed();
2238 break;
2239 case SESSION_INITIALIZED:
2240 rejectCallFromSessionInitiate();
2241 break;
2242 default:
2243 throw new IllegalStateException("Can not reject call from " + this.state);
2244 }
2245 }
2246
2247 public synchronized void endCall() {
2248 if (isTerminated()) {
2249 Log.w(
2250 Config.LOGTAG,
2251 id.account.getJid().asBareJid()
2252 + ": received endCall() when session has already been terminated. nothing to do");
2253 return;
2254 }
2255 if (isInState(State.PROPOSED) && !isInitiator()) {
2256 rejectCallFromProposed();
2257 return;
2258 }
2259 if (isInState(State.PROCEED)) {
2260 if (isInitiator()) {
2261 retractFromProceed();
2262 } else {
2263 rejectCallFromProceed();
2264 }
2265 return;
2266 }
2267 if (isInitiator()
2268 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2269 this.webRTCWrapper.close();
2270 sendSessionTerminate(Reason.CANCEL);
2271 return;
2272 }
2273 if (isInState(State.SESSION_INITIALIZED)) {
2274 rejectCallFromSessionInitiate();
2275 return;
2276 }
2277 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2278 this.webRTCWrapper.close();
2279 sendSessionTerminate(Reason.SUCCESS);
2280 return;
2281 }
2282 if (isInState(
2283 State.TERMINATED_APPLICATION_FAILURE,
2284 State.TERMINATED_CONNECTIVITY_ERROR,
2285 State.TERMINATED_DECLINED_OR_BUSY)) {
2286 Log.d(
2287 Config.LOGTAG,
2288 "ignoring request to end call because already in state " + this.state);
2289 return;
2290 }
2291 throw new IllegalStateException(
2292 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2293 }
2294
2295 private void retractFromProceed() {
2296 Log.d(Config.LOGTAG, "retract from proceed");
2297 this.sendJingleMessage("retract");
2298 closeTransitionLogFinish(State.RETRACTED_RACED);
2299 }
2300
2301 private void closeTransitionLogFinish(final State state) {
2302 this.webRTCWrapper.close();
2303 transitionOrThrow(state);
2304 writeLogMessage(state);
2305 finish();
2306 }
2307
2308 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2309 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2310 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2311 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2312 }
2313
2314 private void acceptCallFromProposed() {
2315 transitionOrThrow(State.PROCEED);
2316 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2317 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2318 this.sendJingleMessage("proceed");
2319 }
2320
2321 private void rejectCallFromProposed() {
2322 transitionOrThrow(State.REJECTED);
2323 writeLogMessageMissed();
2324 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2325 this.sendJingleMessage("reject");
2326 finish();
2327 }
2328
2329 private void rejectCallFromProceed() {
2330 this.sendJingleMessage("reject");
2331 closeTransitionLogFinish(State.REJECTED_RACED);
2332 }
2333
2334 private void rejectCallFromSessionInitiate() {
2335 webRTCWrapper.close();
2336 sendSessionTerminate(Reason.DECLINE);
2337 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2338 }
2339
2340 private void sendJingleMessage(final String action) {
2341 sendJingleMessage(action, id.with);
2342 }
2343
2344 private void sendJingleMessage(final String action, final Jid to) {
2345 final MessagePacket messagePacket = new MessagePacket();
2346 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2347 messagePacket.setTo(to);
2348 final Element intent =
2349 messagePacket
2350 .addChild(action, Namespace.JINGLE_MESSAGE)
2351 .setAttribute("id", id.sessionId);
2352 if ("proceed".equals(action)) {
2353 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2354 if (isOmemoEnabled()) {
2355 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2356 final Element device =
2357 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2358 device.setAttribute("id", deviceId);
2359 }
2360 }
2361 messagePacket.addChild("store", "urn:xmpp:hints");
2362 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2363 }
2364
2365 private boolean isOmemoEnabled() {
2366 final Conversational conversational = message.getConversation();
2367 if (conversational instanceof Conversation) {
2368 return ((Conversation) conversational).getNextEncryption()
2369 == Message.ENCRYPTION_AXOLOTL;
2370 }
2371 return false;
2372 }
2373
2374 private void acceptCallFromSessionInitialized() {
2375 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2376 sendSessionAccept();
2377 }
2378
2379 private synchronized boolean isInState(State... state) {
2380 return Arrays.asList(state).contains(this.state);
2381 }
2382
2383 private boolean transition(final State target) {
2384 return transition(target, null);
2385 }
2386
2387 private synchronized boolean transition(final State target, final Runnable runnable) {
2388 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2389 if (validTransitions != null && validTransitions.contains(target)) {
2390 this.state = target;
2391 if (runnable != null) {
2392 runnable.run();
2393 }
2394 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2395 updateEndUserState();
2396 updateOngoingCallNotification();
2397 return true;
2398 } else {
2399 return false;
2400 }
2401 }
2402
2403 void transitionOrThrow(final State target) {
2404 if (!transition(target)) {
2405 throw new IllegalStateException(
2406 String.format("Unable to transition from %s to %s", this.state, target));
2407 }
2408 }
2409
2410 @Override
2411 public void onIceCandidate(final IceCandidate iceCandidate) {
2412 final RtpContentMap rtpContentMap =
2413 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2414 final IceUdpTransportInfo.Credentials credentials;
2415 try {
2416 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2417 } catch (final IllegalArgumentException e) {
2418 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2419 return;
2420 }
2421 final String uFrag = credentials.ufrag;
2422 final IceUdpTransportInfo.Candidate candidate =
2423 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2424 if (candidate == null) {
2425 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2426 return;
2427 }
2428 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2429 sendTransportInfo(iceCandidate.sdpMid, candidate);
2430 }
2431
2432 @Override
2433 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2434 Log.d(
2435 Config.LOGTAG,
2436 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2437 this.stateHistory.add(newState);
2438 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2439 this.sessionDuration.start();
2440 updateOngoingCallNotification();
2441 } else if (this.sessionDuration.isRunning()) {
2442 this.sessionDuration.stop();
2443 updateOngoingCallNotification();
2444 }
2445
2446 final boolean neverConnected =
2447 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2448
2449 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2450 if (neverConnected) {
2451 if (isTerminated()) {
2452 Log.d(
2453 Config.LOGTAG,
2454 id.account.getJid().asBareJid()
2455 + ": not sending session-terminate after connectivity error because session is already in state "
2456 + this.state);
2457 return;
2458 }
2459 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2460 return;
2461 } else {
2462 this.restartIce();
2463 }
2464 }
2465 updateEndUserState();
2466 }
2467
2468 private void restartIce() {
2469 this.stateHistory.clear();
2470 this.webRTCWrapper.restartIceAsync();
2471 }
2472
2473 @Override
2474 public void onRenegotiationNeeded() {
2475 this.webRTCWrapper.execute(this::renegotiate);
2476 }
2477
2478 private void renegotiate() {
2479 final SessionDescription sessionDescription;
2480 try {
2481 sessionDescription = setLocalSessionDescription();
2482 } catch (final Exception e) {
2483 final Throwable cause = Throwables.getRootCause(e);
2484 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2485 webRTCWrapper.close();
2486 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2487 return;
2488 }
2489 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2490 final RtpContentMap currentContentMap = getLocalContentMap();
2491 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2492 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2493
2494 Log.d(
2495 Config.LOGTAG,
2496 id.getAccount().getJid().asBareJid()
2497 + ": renegotiate. iceRestart="
2498 + iceRestart
2499 + " content id diff="
2500 + diff);
2501
2502 if (diff.hasModifications() && iceRestart) {
2503 webRTCWrapper.close();
2504 sendSessionTerminate(
2505 Reason.FAILED_APPLICATION,
2506 "WebRTC unexpectedly tried to modify content and transport at once");
2507 return;
2508 }
2509
2510 if (iceRestart) {
2511 initiateIceRestart(rtpContentMap);
2512 return;
2513 } else if (diff.isEmpty()) {
2514 Log.d(
2515 Config.LOGTAG,
2516 "renegotiation. nothing to do. SignalingState="
2517 + this.webRTCWrapper.getSignalingState());
2518 }
2519
2520 if (diff.added.size() > 0) {
2521 modifyLocalContentMap(rtpContentMap);
2522 sendContentAdd(rtpContentMap, diff.added);
2523 }
2524 }
2525
2526 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2527 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2528 final JinglePacket jinglePacket =
2529 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2530 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2531 jinglePacket.setTo(id.with);
2532 xmppConnectionService.sendIqPacket(
2533 id.account,
2534 jinglePacket,
2535 (account, response) -> {
2536 if (response.getType() == IqPacket.TYPE.RESULT) {
2537 Log.d(Config.LOGTAG, "received success to our ice restart");
2538 setLocalContentMap(rtpContentMap);
2539 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2540 return;
2541 }
2542 if (response.getType() == IqPacket.TYPE.ERROR) {
2543 if (isTieBreak(response)) {
2544 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2545 return;
2546 }
2547 handleIqErrorResponse(response);
2548 }
2549 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2550 handleIqTimeoutResponse(response);
2551 }
2552 });
2553 }
2554
2555 private boolean isTieBreak(final IqPacket response) {
2556 final Element error = response.findChild("error");
2557 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2558 }
2559
2560 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2561 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2562 this.outgoingContentAdd = contentAdd;
2563 final JinglePacket jinglePacket =
2564 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2565 jinglePacket.setTo(id.with);
2566 xmppConnectionService.sendIqPacket(
2567 id.account,
2568 jinglePacket,
2569 (connection, response) -> {
2570 if (response.getType() == IqPacket.TYPE.RESULT) {
2571 Log.d(
2572 Config.LOGTAG,
2573 id.getAccount().getJid().asBareJid()
2574 + ": received ACK to our content-add");
2575 return;
2576 }
2577 if (response.getType() == IqPacket.TYPE.ERROR) {
2578 if (isTieBreak(response)) {
2579 this.outgoingContentAdd = null;
2580 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2581 return;
2582 }
2583 handleIqErrorResponse(response);
2584 }
2585 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2586 handleIqTimeoutResponse(response);
2587 }
2588 });
2589 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2590 }
2591
2592 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2593 if (isInitiator()) {
2594 this.initiatorRtpContentMap = rtpContentMap;
2595 } else {
2596 this.responderRtpContentMap = rtpContentMap;
2597 }
2598 }
2599
2600 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2601 if (isInitiator()) {
2602 this.responderRtpContentMap = rtpContentMap;
2603 } else {
2604 this.initiatorRtpContentMap = rtpContentMap;
2605 }
2606 }
2607
2608 // this method is to be used for content map modifications that modify media
2609 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2610 final RtpContentMap activeContents = rtpContentMap.activeContents();
2611 setLocalContentMap(activeContents);
2612 this.webRTCWrapper.switchSpeakerPhonePreference(
2613 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2614 updateEndUserState();
2615 }
2616
2617 private SessionDescription setLocalSessionDescription()
2618 throws ExecutionException, InterruptedException {
2619 final org.webrtc.SessionDescription sessionDescription =
2620 this.webRTCWrapper.setLocalDescription().get();
2621 return SessionDescription.parse(sessionDescription.description);
2622 }
2623
2624 private void closeWebRTCSessionAfterFailedConnection() {
2625 this.webRTCWrapper.close();
2626 synchronized (this) {
2627 if (isTerminated()) {
2628 Log.d(
2629 Config.LOGTAG,
2630 id.account.getJid().asBareJid()
2631 + ": no need to send session-terminate after failed connection. Other party already did");
2632 return;
2633 }
2634 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2635 }
2636 }
2637
2638 public boolean zeroDuration() {
2639 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2640 }
2641
2642 public long getCallDuration() {
2643 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2644 }
2645
2646 public AppRTCAudioManager getAudioManager() {
2647 return webRTCWrapper.getAudioManager();
2648 }
2649
2650 public boolean isMicrophoneEnabled() {
2651 return webRTCWrapper.isMicrophoneEnabled();
2652 }
2653
2654 public boolean setMicrophoneEnabled(final boolean enabled) {
2655 return webRTCWrapper.setMicrophoneEnabled(enabled);
2656 }
2657
2658 public boolean isVideoEnabled() {
2659 return webRTCWrapper.isVideoEnabled();
2660 }
2661
2662 public void setVideoEnabled(final boolean enabled) {
2663 webRTCWrapper.setVideoEnabled(enabled);
2664 }
2665
2666 public boolean isCameraSwitchable() {
2667 return webRTCWrapper.isCameraSwitchable();
2668 }
2669
2670 public boolean isFrontCamera() {
2671 return webRTCWrapper.isFrontCamera();
2672 }
2673
2674 public ListenableFuture<Boolean> switchCamera() {
2675 return webRTCWrapper.switchCamera();
2676 }
2677
2678 @Override
2679 public void onAudioDeviceChanged(
2680 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2681 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2682 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2683 selectedAudioDevice, availableAudioDevices);
2684 }
2685
2686 private void updateEndUserState() {
2687 final RtpEndUserState endUserState = getEndUserState();
2688 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2689 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2690 id.account, id.with, id.sessionId, endUserState);
2691 }
2692
2693 private void updateOngoingCallNotification() {
2694 final State state = this.state;
2695 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2696 final boolean reconnecting;
2697 if (state == State.SESSION_ACCEPTED) {
2698 reconnecting =
2699 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2700 } else {
2701 reconnecting = false;
2702 }
2703 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2704 } else {
2705 xmppConnectionService.removeOngoingCall();
2706 }
2707 }
2708
2709 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2710 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2711 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2712 request.setTo(id.account.getDomain());
2713 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2714 xmppConnectionService.sendIqPacket(
2715 id.account,
2716 request,
2717 (account, response) -> {
2718 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2719 new ImmutableList.Builder<>();
2720 if (response.getType() == IqPacket.TYPE.RESULT) {
2721 final Element services =
2722 response.findChild(
2723 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2724 final List<Element> children =
2725 services == null
2726 ? Collections.emptyList()
2727 : services.getChildren();
2728 for (final Element child : children) {
2729 if ("service".equals(child.getName())) {
2730 final String type = child.getAttribute("type");
2731 final String host = child.getAttribute("host");
2732 final String sport = child.getAttribute("port");
2733 final Integer port =
2734 sport == null ? null : Ints.tryParse(sport);
2735 final String transport = child.getAttribute("transport");
2736 final String username = child.getAttribute("username");
2737 final String password = child.getAttribute("password");
2738 if (Strings.isNullOrEmpty(host) || port == null) {
2739 continue;
2740 }
2741 if (port < 0 || port > 65535) {
2742 continue;
2743 }
2744
2745
2746
2747
2748 if (Arrays.asList("stun", "stuns", "turn", "turns")
2749 .contains(type)
2750 && Arrays.asList("udp", "tcp").contains(transport)) {
2751 if (Arrays.asList("stuns", "turns").contains(type)
2752 && "udp".equals(transport)) {
2753 Log.d(
2754 Config.LOGTAG,
2755 id.account.getJid().asBareJid()
2756 + ": skipping invalid combination of udp/tls in external services");
2757 continue;
2758 }
2759
2760 // STUN URLs do not support a query section since M110
2761 final String uri;
2762 if (Arrays.asList("stun","stuns").contains(type)) {
2763 uri = String.format("%s:%s%s", type, IP.wrapIPv6(host),port);
2764 } else {
2765 uri = String.format(
2766 "%s:%s:%s?transport=%s",
2767 type,
2768 IP.wrapIPv6(host),
2769 port,
2770 transport);
2771 }
2772
2773 final PeerConnection.IceServer.Builder iceServerBuilder =
2774 PeerConnection.IceServer.builder(uri);
2775 iceServerBuilder.setTlsCertPolicy(
2776 PeerConnection.TlsCertPolicy
2777 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2778 if (username != null && password != null) {
2779 iceServerBuilder.setUsername(username);
2780 iceServerBuilder.setPassword(password);
2781 } else if (Arrays.asList("turn", "turns").contains(type)) {
2782 // The WebRTC spec requires throwing an
2783 // InvalidAccessError when username (from libwebrtc
2784 // source coder)
2785 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2786 Log.d(
2787 Config.LOGTAG,
2788 id.account.getJid().asBareJid()
2789 + ": skipping "
2790 + type
2791 + "/"
2792 + transport
2793 + " without username and password");
2794 continue;
2795 }
2796 final PeerConnection.IceServer iceServer =
2797 iceServerBuilder.createIceServer();
2798 Log.d(
2799 Config.LOGTAG,
2800 id.account.getJid().asBareJid()
2801 + ": discovered ICE Server: "
2802 + iceServer);
2803 listBuilder.add(iceServer);
2804 }
2805 }
2806 }
2807 }
2808 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2809 if (iceServers.size() == 0) {
2810 Log.w(
2811 Config.LOGTAG,
2812 id.account.getJid().asBareJid()
2813 + ": no ICE server found "
2814 + response);
2815 }
2816 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2817 });
2818 } else {
2819 Log.w(
2820 Config.LOGTAG,
2821 id.account.getJid().asBareJid() + ": has no external service discovery");
2822 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2823 }
2824 }
2825
2826 private void finish() {
2827 if (isTerminated()) {
2828 this.cancelRingingTimeout();
2829 this.webRTCWrapper.verifyClosed();
2830 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2831 this.jingleConnectionManager.finishConnectionOrThrow(this);
2832 try {
2833 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2834 log.getParentFile().mkdirs();
2835 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2836 } catch (final IOException e) { }
2837 } else {
2838 throw new IllegalStateException(
2839 String.format("Unable to call finish from %s", this.state));
2840 }
2841 }
2842
2843 private void writeLogMessage(final State state) {
2844 final long duration = getCallDuration();
2845 if (state == State.TERMINATED_SUCCESS
2846 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2847 writeLogMessageSuccess(duration);
2848 } else {
2849 writeLogMessageMissed();
2850 }
2851 }
2852
2853 private void writeLogMessageSuccess(final long duration) {
2854 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2855 this.writeMessage();
2856 }
2857
2858 private void writeLogMessageMissed() {
2859 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2860 this.writeMessage();
2861 }
2862
2863 private void writeMessage() {
2864 final Conversational conversational = message.getConversation();
2865 if (conversational instanceof Conversation) {
2866 ((Conversation) conversational).add(this.message);
2867 xmppConnectionService.createMessageAsync(message);
2868 xmppConnectionService.updateConversationUi();
2869 } else {
2870 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2871 }
2872 }
2873
2874 public State getState() {
2875 return this.state;
2876 }
2877
2878 boolean isTerminated() {
2879 return TERMINATED.contains(this.state);
2880 }
2881
2882 public Optional<VideoTrack> getLocalVideoTrack() {
2883 return webRTCWrapper.getLocalVideoTrack();
2884 }
2885
2886 public Optional<VideoTrack> getRemoteVideoTrack() {
2887 return webRTCWrapper.getRemoteVideoTrack();
2888 }
2889
2890 public EglBase.Context getEglBaseContext() {
2891 return webRTCWrapper.getEglBaseContext();
2892 }
2893
2894 void setProposedMedia(final Set<Media> media) {
2895 this.proposedMedia = media;
2896 }
2897
2898 public void fireStateUpdate() {
2899 final RtpEndUserState endUserState = getEndUserState();
2900 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2901 id.account, id.with, id.sessionId, endUserState);
2902 }
2903
2904 public boolean isSwitchToVideoAvailable() {
2905 final boolean prerequisite =
2906 Media.audioOnly(getMedia())
2907 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2908 .contains(getEndUserState());
2909 return prerequisite && remoteHasVideoFeature();
2910 }
2911
2912 private boolean remoteHasVideoFeature() {
2913 final Contact contact = id.getContact();
2914 final Presence presence =
2915 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2916 final ServiceDiscoveryResult serviceDiscoveryResult =
2917 presence == null ? null : presence.getServiceDiscoveryResult();
2918 final List<String> features =
2919 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
2920 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
2921 }
2922
2923 private interface OnIceServersDiscovered {
2924 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2925 }
2926}