1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4import android.os.Environment;
5
6import androidx.annotation.NonNull;
7import androidx.annotation.Nullable;
8
9import com.google.common.base.Joiner;
10import com.google.common.base.Optional;
11import com.google.common.base.Preconditions;
12import com.google.common.base.Stopwatch;
13import com.google.common.base.Strings;
14import com.google.common.base.Throwables;
15import com.google.common.collect.Collections2;
16import com.google.common.collect.ImmutableList;
17import com.google.common.collect.ImmutableMap;
18import com.google.common.collect.ImmutableMultimap;
19import com.google.common.collect.ImmutableSet;
20import com.google.common.collect.Iterables;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Multimap;
23import com.google.common.collect.Sets;
24import com.google.common.primitives.Ints;
25import com.google.common.util.concurrent.FutureCallback;
26import com.google.common.util.concurrent.Futures;
27import com.google.common.util.concurrent.ListenableFuture;
28import com.google.common.util.concurrent.MoreExecutors;
29
30import org.webrtc.DtmfSender;
31import org.webrtc.EglBase;
32import org.webrtc.IceCandidate;
33import org.webrtc.PeerConnection;
34import org.webrtc.VideoTrack;
35
36import java.io.File;
37import java.io.IOException;
38import java.util.Arrays;
39import java.util.Collection;
40import java.util.Collections;
41import java.util.LinkedList;
42import java.util.List;
43import java.util.Map;
44import java.util.Queue;
45import java.util.Set;
46import java.util.concurrent.ExecutionException;
47import java.util.concurrent.ScheduledFuture;
48import java.util.concurrent.TimeUnit;
49
50import eu.siacs.conversations.BuildConfig;
51import eu.siacs.conversations.Config;
52import eu.siacs.conversations.crypto.axolotl.AxolotlService;
53import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
54import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
55import eu.siacs.conversations.entities.Account;
56import eu.siacs.conversations.entities.Contact;
57import eu.siacs.conversations.entities.Conversation;
58import eu.siacs.conversations.entities.Conversational;
59import eu.siacs.conversations.entities.Message;
60import eu.siacs.conversations.entities.Presence;
61import eu.siacs.conversations.entities.RtpSessionStatus;
62import eu.siacs.conversations.entities.ServiceDiscoveryResult;
63import eu.siacs.conversations.services.AppRTCAudioManager;
64import eu.siacs.conversations.utils.IP;
65import eu.siacs.conversations.xml.Element;
66import eu.siacs.conversations.xml.Namespace;
67import eu.siacs.conversations.xmpp.Jid;
68import eu.siacs.conversations.xmpp.OnIqPacketReceived;
69import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
70import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
71import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
72import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
73import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
74import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
75import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
76import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
77import eu.siacs.conversations.xmpp.stanzas.IqPacket;
78import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
79
80public class JingleRtpConnection extends AbstractJingleConnection
81 implements WebRTCWrapper.EventCallback {
82
83 public static final List<State> STATES_SHOWING_ONGOING_CALL =
84 Arrays.asList(
85 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
86 private static final long BUSY_TIME_OUT = 30;
87 private static final List<State> TERMINATED =
88 Arrays.asList(
89 State.ACCEPTED,
90 State.REJECTED,
91 State.REJECTED_RACED,
92 State.RETRACTED,
93 State.RETRACTED_RACED,
94 State.TERMINATED_SUCCESS,
95 State.TERMINATED_DECLINED_OR_BUSY,
96 State.TERMINATED_CONNECTIVITY_ERROR,
97 State.TERMINATED_CANCEL_OR_TIMEOUT,
98 State.TERMINATED_APPLICATION_FAILURE,
99 State.TERMINATED_SECURITY_ERROR);
100
101 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
102
103 static {
104 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
105 new ImmutableMap.Builder<>();
106 transitionBuilder.put(
107 State.NULL,
108 ImmutableList.of(
109 State.PROPOSED,
110 State.SESSION_INITIALIZED,
111 State.TERMINATED_APPLICATION_FAILURE,
112 State.TERMINATED_SECURITY_ERROR));
113 transitionBuilder.put(
114 State.PROPOSED,
115 ImmutableList.of(
116 State.ACCEPTED,
117 State.PROCEED,
118 State.REJECTED,
119 State.RETRACTED,
120 State.TERMINATED_APPLICATION_FAILURE,
121 State.TERMINATED_SECURITY_ERROR,
122 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
123 // rebinds
124 ));
125 transitionBuilder.put(
126 State.PROCEED,
127 ImmutableList.of(
128 State.REJECTED_RACED,
129 State.RETRACTED_RACED,
130 State.SESSION_INITIALIZED_PRE_APPROVED,
131 State.TERMINATED_SUCCESS,
132 State.TERMINATED_APPLICATION_FAILURE,
133 State.TERMINATED_SECURITY_ERROR,
134 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
135 // bounces of the proceed message
136 ));
137 transitionBuilder.put(
138 State.SESSION_INITIALIZED,
139 ImmutableList.of(
140 State.SESSION_ACCEPTED,
141 State.TERMINATED_SUCCESS,
142 State.TERMINATED_DECLINED_OR_BUSY,
143 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
144 // and IQ timeouts
145 State.TERMINATED_CANCEL_OR_TIMEOUT,
146 State.TERMINATED_APPLICATION_FAILURE,
147 State.TERMINATED_SECURITY_ERROR));
148 transitionBuilder.put(
149 State.SESSION_INITIALIZED_PRE_APPROVED,
150 ImmutableList.of(
151 State.SESSION_ACCEPTED,
152 State.TERMINATED_SUCCESS,
153 State.TERMINATED_DECLINED_OR_BUSY,
154 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
155 // and IQ timeouts
156 State.TERMINATED_CANCEL_OR_TIMEOUT,
157 State.TERMINATED_APPLICATION_FAILURE,
158 State.TERMINATED_SECURITY_ERROR));
159 transitionBuilder.put(
160 State.SESSION_ACCEPTED,
161 ImmutableList.of(
162 State.TERMINATED_SUCCESS,
163 State.TERMINATED_DECLINED_OR_BUSY,
164 State.TERMINATED_CONNECTIVITY_ERROR,
165 State.TERMINATED_CANCEL_OR_TIMEOUT,
166 State.TERMINATED_APPLICATION_FAILURE,
167 State.TERMINATED_SECURITY_ERROR));
168 VALID_TRANSITIONS = transitionBuilder.build();
169 }
170
171 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
172 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
173 pendingIceCandidates = new LinkedList<>();
174 private final OmemoVerification omemoVerification = new OmemoVerification();
175 private final Message message;
176 private State state = State.NULL;
177 private Set<Media> proposedMedia;
178 private RtpContentMap initiatorRtpContentMap;
179 private RtpContentMap responderRtpContentMap;
180 private RtpContentMap incomingContentAdd;
181 private RtpContentMap outgoingContentAdd;
182 private IceUdpTransportInfo.Setup peerDtlsSetup;
183 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
184 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
185 private ScheduledFuture<?> ringingTimeoutFuture;
186 private final long created = System.currentTimeMillis() / 1000L;
187
188 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
189 super(jingleConnectionManager, id, initiator);
190 final Conversation conversation =
191 jingleConnectionManager
192 .getXmppConnectionService()
193 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
194 this.message =
195 new Message(
196 conversation,
197 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
198 Message.TYPE_RTP_SESSION,
199 id.sessionId);
200 }
201
202 private static State reasonToState(Reason reason) {
203 switch (reason) {
204 case SUCCESS:
205 return State.TERMINATED_SUCCESS;
206 case DECLINE:
207 case BUSY:
208 return State.TERMINATED_DECLINED_OR_BUSY;
209 case CANCEL:
210 case TIMEOUT:
211 return State.TERMINATED_CANCEL_OR_TIMEOUT;
212 case SECURITY_ERROR:
213 return State.TERMINATED_SECURITY_ERROR;
214 case FAILED_APPLICATION:
215 case UNSUPPORTED_TRANSPORTS:
216 case UNSUPPORTED_APPLICATIONS:
217 return State.TERMINATED_APPLICATION_FAILURE;
218 default:
219 return State.TERMINATED_CONNECTIVITY_ERROR;
220 }
221 }
222
223 @Override
224 synchronized void deliverPacket(final JinglePacket jinglePacket) {
225 switch (jinglePacket.getAction()) {
226 case SESSION_INITIATE:
227 receiveSessionInitiate(jinglePacket);
228 break;
229 case TRANSPORT_INFO:
230 receiveTransportInfo(jinglePacket);
231 break;
232 case SESSION_ACCEPT:
233 receiveSessionAccept(jinglePacket);
234 break;
235 case SESSION_TERMINATE:
236 receiveSessionTerminate(jinglePacket);
237 break;
238 case CONTENT_ADD:
239 receiveContentAdd(jinglePacket);
240 break;
241 case CONTENT_ACCEPT:
242 receiveContentAccept(jinglePacket);
243 break;
244 case CONTENT_REJECT:
245 receiveContentReject(jinglePacket);
246 break;
247 case CONTENT_REMOVE:
248 receiveContentRemove(jinglePacket);
249 break;
250 case CONTENT_MODIFY:
251 receiveContentModify(jinglePacket);
252 break;
253 default:
254 respondOk(jinglePacket);
255 Log.d(
256 Config.LOGTAG,
257 String.format(
258 "%s: received unhandled jingle action %s",
259 id.account.getJid().asBareJid(), jinglePacket.getAction()));
260 break;
261 }
262 }
263
264 @Override
265 synchronized void notifyRebound() {
266 if (isTerminated()) {
267 return;
268 }
269 webRTCWrapper.close();
270 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
271 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
272 }
273 if (isInState(
274 State.SESSION_INITIALIZED,
275 State.SESSION_INITIALIZED_PRE_APPROVED,
276 State.SESSION_ACCEPTED)) {
277 // we might have already changed resources (full jid) at this point; so this might not
278 // even reach the other party
279 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
280 } else {
281 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
282 finish();
283 }
284 }
285
286 public boolean applyDtmfTone(String tone) {
287 return webRTCWrapper.applyDtmfTone(tone);
288 }
289
290 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
291 respondOk(jinglePacket);
292 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
293 final State previous = this.state;
294 Log.d(
295 Config.LOGTAG,
296 id.account.getJid().asBareJid()
297 + ": received session terminate reason="
298 + wrapper.reason
299 + "("
300 + Strings.nullToEmpty(wrapper.text)
301 + ") while in state "
302 + previous);
303 if (TERMINATED.contains(previous)) {
304 Log.d(
305 Config.LOGTAG,
306 id.account.getJid().asBareJid()
307 + ": ignoring session terminate because already in "
308 + previous);
309 return;
310 }
311 webRTCWrapper.close();
312 final State target = reasonToState(wrapper.reason);
313 transitionOrThrow(target);
314 writeLogMessage(target);
315 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
316 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
317 }
318 finish();
319 }
320
321 private void receiveTransportInfo(final JinglePacket jinglePacket) {
322 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
323 // INITIALIZED only after transport-info has been received
324 if (isInState(
325 State.NULL,
326 State.PROCEED,
327 State.SESSION_INITIALIZED,
328 State.SESSION_INITIALIZED_PRE_APPROVED,
329 State.SESSION_ACCEPTED)) {
330 final RtpContentMap contentMap;
331 try {
332 contentMap = RtpContentMap.of(jinglePacket);
333 } catch (final IllegalArgumentException | NullPointerException e) {
334 Log.d(
335 Config.LOGTAG,
336 id.account.getJid().asBareJid()
337 + ": improperly formatted contents; ignoring",
338 e);
339 respondOk(jinglePacket);
340 return;
341 }
342 receiveTransportInfo(jinglePacket, contentMap);
343 } else {
344 if (isTerminated()) {
345 respondOk(jinglePacket);
346 Log.d(
347 Config.LOGTAG,
348 id.account.getJid().asBareJid()
349 + ": ignoring out-of-order transport info; we where already terminated");
350 } else {
351 Log.d(
352 Config.LOGTAG,
353 id.account.getJid().asBareJid()
354 + ": received transport info while in state="
355 + this.state);
356 terminateWithOutOfOrder(jinglePacket);
357 }
358 }
359 }
360
361 private void receiveTransportInfo(
362 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
363 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
364 contentMap.contents.entrySet();
365 final RtpContentMap remote = getRemoteContentMap();
366 final Set<String> remoteContentIds = remote == null ? Collections.emptySet() : remote.contents.keySet();
367 if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) {
368 Log.d(Config.LOGTAG,"received transport-info for unknown contents "+contentMap.contents.keySet()+" (known: "+remoteContentIds+")");
369 respondOk(jinglePacket);
370 pendingIceCandidates.addAll(candidates);
371 return;
372 }
373 if (this.state != State.SESSION_ACCEPTED) {
374 Log.d(Config.LOGTAG,"received transport-info prematurely. adding to backlog");
375 respondOk(jinglePacket);
376 pendingIceCandidates.addAll(candidates);
377 return;
378 }
379 // zero candidates + modified credentials are an ICE restart offer
380 if (checkForIceRestart(jinglePacket, contentMap)) {
381 return;
382 }
383 respondOk(jinglePacket);
384 try {
385 processCandidates(candidates);
386 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
387 Log.w(
388 Config.LOGTAG,
389 id.account.getJid().asBareJid()
390 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
391 }
392 }
393
394 private void receiveContentAdd(final JinglePacket jinglePacket) {
395 final RtpContentMap modification;
396 try {
397 modification = RtpContentMap.of(jinglePacket);
398 modification.requireContentDescriptions();
399 } catch (final RuntimeException e) {
400 Log.d(
401 Config.LOGTAG,
402 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
403 Throwables.getRootCause(e));
404 respondOk(jinglePacket);
405 webRTCWrapper.close();
406 sendSessionTerminate(Reason.of(e), e.getMessage());
407 return;
408 }
409 if (isInState(State.SESSION_ACCEPTED)) {
410 receiveContentAdd(jinglePacket, modification);
411 } else {
412 terminateWithOutOfOrder(jinglePacket);
413 }
414 }
415
416 private void receiveContentAdd(
417 final JinglePacket jinglePacket, final RtpContentMap modification) {
418 final RtpContentMap remote = getRemoteContentMap();
419 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
420 respondOk(jinglePacket);
421 this.webRTCWrapper.close();
422 sendSessionTerminate(
423 Reason.FAILED_APPLICATION,
424 String.format(
425 "contents with names %s already exists",
426 Joiner.on(", ").join(modification.getNames())));
427 return;
428 }
429 final ContentAddition contentAddition =
430 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
431
432 final RtpContentMap outgoing = this.outgoingContentAdd;
433 final Set<ContentAddition.Summary> outgoingContentAddSummary =
434 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
435
436 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
437 if (isInitiator()) {
438 Log.d(
439 Config.LOGTAG,
440 id.getAccount().getJid().asBareJid()
441 + ": respond with tie break to matching content-add offer");
442 respondWithTieBreak(jinglePacket);
443 } else {
444 Log.d(
445 Config.LOGTAG,
446 id.getAccount().getJid().asBareJid()
447 + ": automatically accept matching content-add offer");
448 acceptContentAdd(contentAddition.summary, modification);
449 }
450 return;
451 }
452
453 // once we can display multiple video tracks we can be more loose with this condition
454 // theoretically it should also be fine to automatically accept audio only contents
455 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
456 Log.d(
457 Config.LOGTAG,
458 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
459 this.incomingContentAdd = modification;
460 respondOk(jinglePacket);
461 updateEndUserState();
462 } else {
463 respondOk(jinglePacket);
464 // TODO do we want to add a reason?
465 rejectContentAdd(modification);
466 }
467 }
468
469 private void receiveContentAccept(final JinglePacket jinglePacket) {
470 final RtpContentMap receivedContentAccept;
471 try {
472 receivedContentAccept = RtpContentMap.of(jinglePacket);
473 receivedContentAccept.requireContentDescriptions();
474 } catch (final RuntimeException e) {
475 Log.d(
476 Config.LOGTAG,
477 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
478 Throwables.getRootCause(e));
479 respondOk(jinglePacket);
480 webRTCWrapper.close();
481 sendSessionTerminate(Reason.of(e), e.getMessage());
482 return;
483 }
484
485 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
486 if (outgoingContentAdd == null) {
487 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
488 terminateWithOutOfOrder(jinglePacket);
489 return;
490 }
491 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
492 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
493 this.outgoingContentAdd = null;
494 respondOk(jinglePacket);
495 receiveContentAccept(receivedContentAccept);
496 } else {
497 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
498 terminateWithOutOfOrder(jinglePacket);
499 }
500 }
501
502 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
503 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
504 final RtpContentMap modifiedContentMap =
505 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
506
507 setRemoteContentMap(modifiedContentMap);
508
509 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
510
511 final org.webrtc.SessionDescription sdp =
512 new org.webrtc.SessionDescription(
513 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
514
515 try {
516 this.webRTCWrapper.setRemoteDescription(sdp).get();
517 } catch (final Exception e) {
518 final Throwable cause = Throwables.getRootCause(e);
519 Log.d(
520 Config.LOGTAG,
521 id.getAccount().getJid().asBareJid()
522 + ": unable to set remote description after receiving content-accept",
523 cause);
524 webRTCWrapper.close();
525 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
526 return;
527 }
528 processCandidates(receivedContentAccept.contents.entrySet());
529 updateEndUserState();
530 Log.d(
531 Config.LOGTAG,
532 id.getAccount().getJid().asBareJid()
533 + ": remote has accepted content-add "
534 + ContentAddition.summary(receivedContentAccept));
535 }
536
537 private void receiveContentModify(final JinglePacket jinglePacket) {
538 // TODO check session accepted
539 final Map<String, Content.Senders> modification =
540 Maps.transformEntries(
541 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
542 final boolean isInitiator = isInitiator();
543 final RtpContentMap currentOutgoing = this.outgoingContentAdd;
544 final RtpContentMap remoteContentMap = this.getRemoteContentMap();
545 final Set<String> currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet();
546 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
547 if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
548 respondOk(jinglePacket);
549 final RtpContentMap modifiedContentMap;
550 try {
551 modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification);
552 } catch (final IllegalArgumentException e) {
553 webRTCWrapper.close();
554 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
555 return;
556 }
557 this.outgoingContentAdd = modifiedContentMap;
558 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add");
559 } else if (remoteContentMap != null && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
560 respondOk(jinglePacket);
561 final RtpContentMap modifiedRemoteContentMap;
562 try {
563 modifiedRemoteContentMap = remoteContentMap.modifiedSendersChecked(isInitiator, modification);
564 } catch (final IllegalArgumentException e) {
565 webRTCWrapper.close();
566 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
567 return;
568 }
569 final SessionDescription offer;
570 try {
571 offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator());
572 } catch (final IllegalArgumentException | NullPointerException e) {
573 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-modify to SDP", e);
574 webRTCWrapper.close();
575 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
576 return;
577 }
578 Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": auto accepting content-modification");
579 this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
580 } else {
581 Log.d(Config.LOGTAG,"received unsupported content modification "+modification);
582 respondWithItemNotFound(jinglePacket);
583 }
584 }
585
586 private void autoAcceptContentModify(final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
587 this.setRemoteContentMap(modifiedRemoteContentMap);
588 final org.webrtc.SessionDescription sdp =
589 new org.webrtc.SessionDescription(
590 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
591 try {
592 this.webRTCWrapper.setRemoteDescription(sdp).get();
593 // auto accept is only done when we already have tracks
594 final SessionDescription answer = setLocalSessionDescription();
595 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
596 modifyLocalContentMap(rtpContentMap);
597 // we do not need to send an answer but do we have to resend the candidates currently in SDP?
598 //resendCandidatesFromSdp(answer);
599 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
600 } catch (final Exception e) {
601 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
602 webRTCWrapper.close();
603 sendSessionTerminate(Reason.FAILED_APPLICATION);
604 }
605 }
606
607 private void resendCandidatesFromSdp(final SessionDescription answer) {
608 final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
609 for(final SessionDescription.Media media : answer.media) {
610 final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
611 if (Strings.isNullOrEmpty(mid)) {
612 continue;
613 }
614 for(final String sdpCandidate : media.attributes.get("candidate")) {
615 final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
616 if (candidate != null) {
617 candidateBuilder.put(mid,candidate);
618 }
619 }
620 }
621 final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build();
622 sendTransportInfo(candidates);
623 }
624
625 private void receiveContentReject(final JinglePacket jinglePacket) {
626 final RtpContentMap receivedContentReject;
627 try {
628 receivedContentReject = RtpContentMap.of(jinglePacket);
629 } catch (final RuntimeException e) {
630 Log.d(
631 Config.LOGTAG,
632 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
633 Throwables.getRootCause(e));
634 respondOk(jinglePacket);
635 this.webRTCWrapper.close();
636 sendSessionTerminate(Reason.of(e), e.getMessage());
637 return;
638 }
639
640 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
641 if (outgoingContentAdd == null) {
642 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
643 terminateWithOutOfOrder(jinglePacket);
644 return;
645 }
646 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
647 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
648 this.outgoingContentAdd = null;
649 respondOk(jinglePacket);
650 Log.d(Config.LOGTAG,jinglePacket.toString());
651 receiveContentReject(ourSummary);
652 } else {
653 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
654 terminateWithOutOfOrder(jinglePacket);
655 }
656 }
657
658 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
659 try {
660 this.webRTCWrapper.removeTrack(Media.VIDEO);
661 final RtpContentMap localContentMap = customRollback();
662 modifyLocalContentMap(localContentMap);
663 } catch (final Exception e) {
664 final Throwable cause = Throwables.getRootCause(e);
665 Log.d(
666 Config.LOGTAG,
667 id.getAccount().getJid().asBareJid()
668 + ": unable to rollback local description after receiving content-reject",
669 cause);
670 webRTCWrapper.close();
671 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
672 return;
673 }
674 Log.d(
675 Config.LOGTAG,
676 id.getAccount().getJid().asBareJid()
677 + ": remote has rejected our content-add "
678 + summary);
679 }
680
681 private void receiveContentRemove(final JinglePacket jinglePacket) {
682 final RtpContentMap receivedContentRemove;
683 try {
684 receivedContentRemove = RtpContentMap.of(jinglePacket);
685 receivedContentRemove.requireContentDescriptions();
686 } catch (final RuntimeException e) {
687 Log.d(
688 Config.LOGTAG,
689 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
690 Throwables.getRootCause(e));
691 respondOk(jinglePacket);
692 this.webRTCWrapper.close();
693 sendSessionTerminate(Reason.of(e), e.getMessage());
694 return;
695 }
696 respondOk(jinglePacket);
697 receiveContentRemove(receivedContentRemove);
698 }
699
700 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
701 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
702 final Set<ContentAddition.Summary> contentAddSummary =
703 incomingContentAdd == null
704 ? Collections.emptySet()
705 : ContentAddition.summary(incomingContentAdd);
706 final Set<ContentAddition.Summary> removeSummary =
707 ContentAddition.summary(receivedContentRemove);
708 if (contentAddSummary.equals(removeSummary)) {
709 this.incomingContentAdd = null;
710 updateEndUserState();
711 } else {
712 webRTCWrapper.close();
713 sendSessionTerminate(
714 Reason.FAILED_APPLICATION,
715 String.format(
716 "%s only supports %s as a means to retract a not yet accepted %s",
717 BuildConfig.APP_NAME,
718 JinglePacket.Action.CONTENT_REMOVE,
719 JinglePacket.Action.CONTENT_ADD));
720 }
721 }
722
723 public synchronized void retractContentAdd() {
724 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
725 if (outgoingContentAdd == null) {
726 throw new IllegalStateException("Not outgoing content add");
727 }
728 try {
729 webRTCWrapper.removeTrack(Media.VIDEO);
730 final RtpContentMap localContentMap = customRollback();
731 modifyLocalContentMap(localContentMap);
732 } catch (final Exception e) {
733 final Throwable cause = Throwables.getRootCause(e);
734 Log.d(
735 Config.LOGTAG,
736 id.getAccount().getJid().asBareJid()
737 + ": unable to rollback local description after trying to retract content-add",
738 cause);
739 webRTCWrapper.close();
740 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
741 return;
742 }
743 this.outgoingContentAdd = null;
744 final JinglePacket retract =
745 outgoingContentAdd
746 .toStub()
747 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
748 this.send(retract);
749 Log.d(
750 Config.LOGTAG,
751 id.getAccount().getJid()
752 + ": retract content-add "
753 + ContentAddition.summary(outgoingContentAdd));
754 }
755
756 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
757 final SessionDescription sdp = setLocalSessionDescription();
758 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
759 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
760 this.webRTCWrapper
761 .setRemoteDescription(
762 new org.webrtc.SessionDescription(
763 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
764 .get();
765 return localRtpContentMap;
766 }
767
768 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
769 final RtpContentMap currentRemote = getRemoteContentMap();
770 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
771 if (diff.isEmpty()) {
772 throw new IllegalStateException(
773 "Unexpected rollback condition. No difference between local and remote");
774 }
775 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
776 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
777 final RtpContentMap nextRemote =
778 currentRemote.addContent(
779 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
780 return SessionDescription.of(nextRemote, !isInitiator());
781 }
782 throw new IllegalStateException(
783 "Unexpected rollback condition. Senders were not uniformly none");
784 }
785
786 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
787 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
788 if (incomingContentAdd == null) {
789 throw new IllegalStateException("No incoming content add");
790 }
791
792 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
793 this.incomingContentAdd = null;
794 final Set<Content.Senders> senders = incomingContentAdd.getSenders();
795 Log.d(Config.LOGTAG,"senders of incoming content-add: "+senders);
796 if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) {
797 Log.d(Config.LOGTAG,"content addition is receive only. we want to upgrade to 'both'");
798 final RtpContentMap modifiedSenders = incomingContentAdd.modifiedSenders(Content.Senders.BOTH);
799 final JinglePacket proposedContentModification = modifiedSenders.toStub().toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId);
800 proposedContentModification.setTo(id.with);
801 xmppConnectionService.sendIqPacket(id.account, proposedContentModification, (account, response) -> {
802 if (response.getType() == IqPacket.TYPE.RESULT) {
803 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has accepted our upgrade to senders=both");
804 acceptContentAdd(ContentAddition.summary(modifiedSenders), modifiedSenders);
805 } else {
806 Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has rejected our upgrade to senders=both");
807 acceptContentAdd(contentAddition, incomingContentAdd);
808 }
809 });
810 }
811 } else {
812 throw new IllegalStateException("Accepted content add does not match pending content-add");
813 }
814 }
815
816 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
817 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
818 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
819 this.setRemoteContentMap(modifiedContentMap);
820
821 final SessionDescription offer;
822 try {
823 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
824 } catch (final IllegalArgumentException | NullPointerException e) {
825 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
826 webRTCWrapper.close();
827 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
828 return;
829 }
830 this.incomingContentAdd = null;
831 acceptContentAdd(contentAddition, offer);
832 }
833
834 private void acceptContentAdd(
835 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
836 final org.webrtc.SessionDescription sdp =
837 new org.webrtc.SessionDescription(
838 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
839 try {
840 this.webRTCWrapper.setRemoteDescription(sdp).get();
841
842 // TODO add tracks for 'media' where contentAddition.senders matches
843
844 // TODO if senders.sending(isInitiator())
845
846 this.webRTCWrapper.addTrack(Media.VIDEO);
847
848 // TODO add additional transceivers for recv only cases
849
850 final SessionDescription answer = setLocalSessionDescription();
851 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
852
853 final RtpContentMap contentAcceptMap =
854 rtpContentMap.toContentModification(
855 Collections2.transform(contentAddition, ca -> ca.name));
856 Log.d(
857 Config.LOGTAG,
858 id.getAccount().getJid().asBareJid()
859 + ": sending content-accept "
860 + ContentAddition.summary(contentAcceptMap));
861
862 addIceCandidatesFromBlackLog();
863
864 modifyLocalContentMap(rtpContentMap);
865 sendContentAccept(contentAcceptMap);
866 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
867 } catch (final Exception e) {
868 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
869 webRTCWrapper.close();
870 sendSessionTerminate(Reason.FAILED_APPLICATION);
871 }
872 }
873
874 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
875 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
876 send(jinglePacket);
877 }
878
879 public synchronized void rejectContentAdd() {
880 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
881 if (incomingContentAdd == null) {
882 throw new IllegalStateException("No incoming content add");
883 }
884 this.incomingContentAdd = null;
885 updateEndUserState();
886 rejectContentAdd(incomingContentAdd);
887 }
888
889 private void rejectContentAdd(final RtpContentMap contentMap) {
890 final JinglePacket jinglePacket =
891 contentMap
892 .toStub()
893 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
894 Log.d(
895 Config.LOGTAG,
896 id.getAccount().getJid().asBareJid()
897 + ": rejecting content "
898 + ContentAddition.summary(contentMap));
899 send(jinglePacket);
900 }
901
902 private boolean checkForIceRestart(
903 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
904 final RtpContentMap existing = getRemoteContentMap();
905 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
906 final IceUdpTransportInfo.Credentials newCredentials;
907 try {
908 existingCredentials = existing.getCredentials();
909 newCredentials = rtpContentMap.getDistinctCredentials();
910 } catch (final IllegalStateException e) {
911 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
912 return false;
913 }
914 if (existingCredentials.contains(newCredentials)) {
915 return false;
916 }
917 // TODO an alternative approach is to check if we already got an iq result to our
918 // ICE-restart
919 // and if that's the case we are seeing an answer.
920 // This might be more spec compliant but also more error prone potentially
921 final boolean isSignalStateStable = this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE;
922 // TODO a stable signal state can be another indicator that we have an offer to restart ICE
923 final boolean isOffer = rtpContentMap.emptyCandidates();
924 final RtpContentMap restartContentMap;
925 try {
926 if (isOffer) {
927 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
928 restartContentMap =
929 existing.modifiedCredentials(
930 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
931 } else {
932 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
933 Log.d(
934 Config.LOGTAG,
935 "received confirmation of ICE restart"
936 + newCredentials
937 + " peer_setup="
938 + setup);
939 // DTLS setup attribute needs to be rewritten to reflect current peer state
940 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
941 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
942 }
943 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
944 return isOffer;
945 } else {
946 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
947 respondWithTieBreak(jinglePacket);
948 return true;
949 }
950 } catch (final Exception exception) {
951 respondOk(jinglePacket);
952 final Throwable rootCause = Throwables.getRootCause(exception);
953 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
954 // If this happens a termination is already in progress
955 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
956 return true;
957 }
958 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
959 webRTCWrapper.close();
960 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
961 return true;
962 }
963 }
964
965 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
966 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
967 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
968 throw new IllegalStateException("Invalid peer setup");
969 }
970 return peerSetup;
971 }
972
973 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
974 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
975 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
976 }
977 this.peerDtlsSetup = setup;
978 }
979
980 private boolean applyIceRestart(
981 final JinglePacket jinglePacket,
982 final RtpContentMap restartContentMap,
983 final boolean isOffer)
984 throws ExecutionException, InterruptedException {
985 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
986 final org.webrtc.SessionDescription.Type type =
987 isOffer
988 ? org.webrtc.SessionDescription.Type.OFFER
989 : org.webrtc.SessionDescription.Type.ANSWER;
990 org.webrtc.SessionDescription sdp =
991 new org.webrtc.SessionDescription(type, sessionDescription.toString());
992 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
993 if (isInitiator()) {
994 // We ignore the offer and respond with tie-break. This will clause the responder
995 // not to apply the content map
996 return false;
997 }
998 }
999 webRTCWrapper.setRemoteDescription(sdp).get();
1000 setRemoteContentMap(restartContentMap);
1001 if (isOffer) {
1002 final SessionDescription localSessionDescription = setLocalSessionDescription();
1003 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
1004 // We need to respond OK before sending any candidates
1005 respondOk(jinglePacket);
1006 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1007 } else {
1008 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
1009 }
1010 return true;
1011 }
1012
1013 private void processCandidates(
1014 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
1015 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
1016 processCandidate(content);
1017 }
1018 }
1019
1020 private void processCandidate(
1021 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
1022 final RtpContentMap rtpContentMap = getRemoteContentMap();
1023 final List<String> indices = toIdentificationTags(rtpContentMap);
1024 final String sdpMid = content.getKey(); // aka content name
1025 final IceUdpTransportInfo transport = content.getValue().transport;
1026 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
1027
1028 // TODO check that credentials remained the same
1029
1030 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
1031 final String sdp;
1032 try {
1033 sdp = candidate.toSdpAttribute(credentials.ufrag);
1034 } catch (final IllegalArgumentException e) {
1035 Log.d(
1036 Config.LOGTAG,
1037 id.account.getJid().asBareJid()
1038 + ": ignoring invalid ICE candidate "
1039 + e.getMessage());
1040 continue;
1041 }
1042 final int mLineIndex = indices.indexOf(sdpMid);
1043 if (mLineIndex < 0) {
1044 Log.w(
1045 Config.LOGTAG,
1046 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
1047 }
1048 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
1049 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
1050 this.webRTCWrapper.addIceCandidate(iceCandidate);
1051 }
1052 }
1053
1054 private RtpContentMap getRemoteContentMap() {
1055 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
1056 }
1057
1058 private RtpContentMap getLocalContentMap() {
1059 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1060 }
1061
1062 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
1063 final Group originalGroup = rtpContentMap.group;
1064 final List<String> identificationTags =
1065 originalGroup == null
1066 ? rtpContentMap.getNames()
1067 : originalGroup.getIdentificationTags();
1068 if (identificationTags.size() == 0) {
1069 Log.w(
1070 Config.LOGTAG,
1071 id.account.getJid().asBareJid()
1072 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
1073 }
1074 return identificationTags;
1075 }
1076
1077 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
1078 final JinglePacket jinglePacket, final boolean expectVerification) {
1079 final RtpContentMap receivedContentMap;
1080 try {
1081 receivedContentMap = RtpContentMap.of(jinglePacket);
1082 } catch (final Exception e) {
1083 return Futures.immediateFailedFuture(e);
1084 }
1085 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
1086 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
1087 id.account
1088 .getAxolotlService()
1089 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
1090 return Futures.transform(
1091 future,
1092 omemoVerifiedPayload -> {
1093 // TODO test if an exception here triggers a correct abort
1094 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
1095 Log.d(
1096 Config.LOGTAG,
1097 id.account.getJid().asBareJid()
1098 + ": received verifiable DTLS fingerprint via "
1099 + omemoVerification);
1100 return omemoVerifiedPayload.getPayload();
1101 },
1102 MoreExecutors.directExecutor());
1103 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
1104 return Futures.immediateFailedFuture(
1105 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
1106 } else {
1107 return Futures.immediateFuture(receivedContentMap);
1108 }
1109 }
1110
1111 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
1112 if (isInitiator()) {
1113 Log.d(
1114 Config.LOGTAG,
1115 String.format(
1116 "%s: received session-initiate even though we were initiating",
1117 id.account.getJid().asBareJid()));
1118 if (isTerminated()) {
1119 Log.d(
1120 Config.LOGTAG,
1121 String.format(
1122 "%s: got a reason to terminate with out-of-order. but already in state %s",
1123 id.account.getJid().asBareJid(), getState()));
1124 respondWithOutOfOrder(jinglePacket);
1125 } else {
1126 terminateWithOutOfOrder(jinglePacket);
1127 }
1128 return;
1129 }
1130 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1131 Futures.addCallback(
1132 future,
1133 new FutureCallback<RtpContentMap>() {
1134 @Override
1135 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1136 receiveSessionInitiate(jinglePacket, rtpContentMap);
1137 }
1138
1139 @Override
1140 public void onFailure(@NonNull final Throwable throwable) {
1141 respondOk(jinglePacket);
1142 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1143 }
1144 },
1145 MoreExecutors.directExecutor());
1146 }
1147
1148 private void receiveSessionInitiate(
1149 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1150 try {
1151 contentMap.requireContentDescriptions();
1152 contentMap.requireDTLSFingerprint(true);
1153 } catch (final RuntimeException e) {
1154 Log.d(
1155 Config.LOGTAG,
1156 id.account.getJid().asBareJid() + ": improperly formatted contents",
1157 Throwables.getRootCause(e));
1158 respondOk(jinglePacket);
1159 sendSessionTerminate(Reason.of(e), e.getMessage());
1160 return;
1161 }
1162 Log.d(
1163 Config.LOGTAG,
1164 "processing session-init with " + contentMap.contents.size() + " contents");
1165 final State target;
1166 if (this.state == State.PROCEED) {
1167 Preconditions.checkState(
1168 proposedMedia != null && proposedMedia.size() > 0,
1169 "proposed media must be set when processing pre-approved session-initiate");
1170 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1171 sendSessionTerminate(
1172 Reason.SECURITY_ERROR,
1173 String.format(
1174 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1175 this.proposedMedia, contentMap.getMedia()));
1176 return;
1177 }
1178 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1179 } else {
1180 target = State.SESSION_INITIALIZED;
1181 }
1182 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1183 respondOk(jinglePacket);
1184 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1185 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1186 Log.d(
1187 Config.LOGTAG,
1188 id.account.getJid().asBareJid()
1189 + ": automatically accepting session-initiate");
1190 sendSessionAccept();
1191 } else {
1192 Log.d(
1193 Config.LOGTAG,
1194 id.account.getJid().asBareJid()
1195 + ": received not pre-approved session-initiate. start ringing");
1196 startRinging();
1197 }
1198 } else {
1199 Log.d(
1200 Config.LOGTAG,
1201 String.format(
1202 "%s: received session-initiate while in state %s",
1203 id.account.getJid().asBareJid(), state));
1204 terminateWithOutOfOrder(jinglePacket);
1205 }
1206 }
1207
1208 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1209 if (!isInitiator()) {
1210 Log.d(
1211 Config.LOGTAG,
1212 String.format(
1213 "%s: received session-accept even though we were responding",
1214 id.account.getJid().asBareJid()));
1215 terminateWithOutOfOrder(jinglePacket);
1216 return;
1217 }
1218 final ListenableFuture<RtpContentMap> future =
1219 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1220 Futures.addCallback(
1221 future,
1222 new FutureCallback<RtpContentMap>() {
1223 @Override
1224 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1225 receiveSessionAccept(jinglePacket, rtpContentMap);
1226 }
1227
1228 @Override
1229 public void onFailure(@NonNull final Throwable throwable) {
1230 respondOk(jinglePacket);
1231 Log.d(
1232 Config.LOGTAG,
1233 id.account.getJid().asBareJid()
1234 + ": improperly formatted contents in session-accept",
1235 throwable);
1236 webRTCWrapper.close();
1237 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1238 }
1239 },
1240 MoreExecutors.directExecutor());
1241 }
1242
1243 private void receiveSessionAccept(
1244 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1245 try {
1246 contentMap.requireContentDescriptions();
1247 contentMap.requireDTLSFingerprint();
1248 } catch (final RuntimeException e) {
1249 respondOk(jinglePacket);
1250 Log.d(
1251 Config.LOGTAG,
1252 id.account.getJid().asBareJid()
1253 + ": improperly formatted contents in session-accept",
1254 e);
1255 webRTCWrapper.close();
1256 sendSessionTerminate(Reason.of(e), e.getMessage());
1257 return;
1258 }
1259 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1260 if (!initiatorMedia.equals(contentMap.getMedia())) {
1261 sendSessionTerminate(
1262 Reason.SECURITY_ERROR,
1263 String.format(
1264 "Your session-included included media %s but our session-initiate was %s",
1265 this.proposedMedia, contentMap.getMedia()));
1266 return;
1267 }
1268 Log.d(
1269 Config.LOGTAG,
1270 "processing session-accept with " + contentMap.contents.size() + " contents");
1271 if (transition(State.SESSION_ACCEPTED)) {
1272 respondOk(jinglePacket);
1273 receiveSessionAccept(contentMap);
1274 } else {
1275 Log.d(
1276 Config.LOGTAG,
1277 String.format(
1278 "%s: received session-accept while in state %s",
1279 id.account.getJid().asBareJid(), state));
1280 respondOk(jinglePacket);
1281 }
1282 }
1283
1284 private void receiveSessionAccept(final RtpContentMap contentMap) {
1285 this.responderRtpContentMap = contentMap;
1286 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1287 final SessionDescription sessionDescription;
1288 try {
1289 sessionDescription = SessionDescription.of(contentMap, false);
1290 } catch (final IllegalArgumentException | NullPointerException e) {
1291 Log.d(
1292 Config.LOGTAG,
1293 id.account.getJid().asBareJid()
1294 + ": unable convert offer from session-accept to SDP",
1295 e);
1296 webRTCWrapper.close();
1297 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1298 return;
1299 }
1300 final org.webrtc.SessionDescription answer =
1301 new org.webrtc.SessionDescription(
1302 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1303 try {
1304 this.webRTCWrapper.setRemoteDescription(answer).get();
1305 } catch (final Exception e) {
1306 Log.d(
1307 Config.LOGTAG,
1308 id.account.getJid().asBareJid()
1309 + ": unable to set remote description after receiving session-accept",
1310 Throwables.getRootCause(e));
1311 webRTCWrapper.close();
1312 sendSessionTerminate(
1313 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1314 return;
1315 }
1316 processCandidates(contentMap.contents.entrySet());
1317 }
1318
1319 private void sendSessionAccept() {
1320 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1321 if (rtpContentMap == null) {
1322 throw new IllegalStateException("initiator RTP Content Map has not been set");
1323 }
1324 final SessionDescription offer;
1325 try {
1326 offer = SessionDescription.of(rtpContentMap, true);
1327 } catch (final IllegalArgumentException | NullPointerException e) {
1328 Log.d(
1329 Config.LOGTAG,
1330 id.account.getJid().asBareJid()
1331 + ": unable convert offer from session-initiate to SDP",
1332 e);
1333 webRTCWrapper.close();
1334 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1335 return;
1336 }
1337 sendSessionAccept(rtpContentMap.getMedia(), offer);
1338 }
1339
1340 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1341 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1342 }
1343
1344 private synchronized void sendSessionAccept(
1345 final Set<Media> media,
1346 final SessionDescription offer,
1347 final List<PeerConnection.IceServer> iceServers) {
1348 if (isTerminated()) {
1349 Log.w(
1350 Config.LOGTAG,
1351 id.account.getJid().asBareJid()
1352 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1353 return;
1354 }
1355 try {
1356 setupWebRTC(media, iceServers);
1357 } catch (final WebRTCWrapper.InitializationException e) {
1358 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1359 webRTCWrapper.close();
1360 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1361 return;
1362 }
1363 final org.webrtc.SessionDescription sdp =
1364 new org.webrtc.SessionDescription(
1365 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1366 try {
1367 this.webRTCWrapper.setRemoteDescription(sdp).get();
1368 addIceCandidatesFromBlackLog();
1369 org.webrtc.SessionDescription webRTCSessionDescription =
1370 this.webRTCWrapper.setLocalDescription().get();
1371 prepareSessionAccept(webRTCSessionDescription);
1372 } catch (final Exception e) {
1373 failureToAcceptSession(e);
1374 }
1375 }
1376
1377 private void failureToAcceptSession(final Throwable throwable) {
1378 if (isTerminated()) {
1379 return;
1380 }
1381 final Throwable rootCause = Throwables.getRootCause(throwable);
1382 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1383 webRTCWrapper.close();
1384 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1385 }
1386
1387 private void addIceCandidatesFromBlackLog() {
1388 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1389 while ((foo = this.pendingIceCandidates.poll()) != null) {
1390 processCandidate(foo);
1391 Log.d(
1392 Config.LOGTAG,
1393 id.account.getJid().asBareJid() + ": added candidate from back log");
1394 }
1395 }
1396
1397 private void prepareSessionAccept(
1398 final org.webrtc.SessionDescription webRTCSessionDescription) {
1399 final SessionDescription sessionDescription =
1400 SessionDescription.parse(webRTCSessionDescription.description);
1401 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1402 this.responderRtpContentMap = respondingRtpContentMap;
1403 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1404 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1405 prepareOutgoingContentMap(respondingRtpContentMap);
1406 Futures.addCallback(
1407 outgoingContentMapFuture,
1408 new FutureCallback<RtpContentMap>() {
1409 @Override
1410 public void onSuccess(final RtpContentMap outgoingContentMap) {
1411 sendSessionAccept(outgoingContentMap);
1412 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1413 }
1414
1415 @Override
1416 public void onFailure(@NonNull Throwable throwable) {
1417 failureToAcceptSession(throwable);
1418 }
1419 },
1420 MoreExecutors.directExecutor());
1421 }
1422
1423 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1424 if (isTerminated()) {
1425 Log.w(
1426 Config.LOGTAG,
1427 id.account.getJid().asBareJid()
1428 + ": preparing session accept was too slow. already terminated. nothing to do.");
1429 return;
1430 }
1431 transitionOrThrow(State.SESSION_ACCEPTED);
1432 final JinglePacket sessionAccept =
1433 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1434 send(sessionAccept);
1435 }
1436
1437 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1438 final RtpContentMap rtpContentMap) {
1439 if (this.omemoVerification.hasDeviceId()) {
1440 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1441 verifiedPayloadFuture =
1442 id.account
1443 .getAxolotlService()
1444 .encrypt(
1445 rtpContentMap,
1446 id.with,
1447 omemoVerification.getDeviceId());
1448 return Futures.transform(
1449 verifiedPayloadFuture,
1450 verifiedPayload -> {
1451 omemoVerification.setOrEnsureEqual(verifiedPayload);
1452 return verifiedPayload.getPayload();
1453 },
1454 MoreExecutors.directExecutor());
1455 } else {
1456 return Futures.immediateFuture(rtpContentMap);
1457 }
1458 }
1459
1460 synchronized void deliveryMessage(
1461 final Jid from,
1462 final Element message,
1463 final String serverMessageId,
1464 final long timestamp) {
1465 Log.d(
1466 Config.LOGTAG,
1467 id.account.getJid().asBareJid()
1468 + ": delivered message to JingleRtpConnection "
1469 + message);
1470 switch (message.getName()) {
1471 case "propose":
1472 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1473 break;
1474 case "proceed":
1475 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1476 break;
1477 case "retract":
1478 receiveRetract(from, serverMessageId, timestamp);
1479 break;
1480 case "reject":
1481 receiveReject(from, serverMessageId, timestamp);
1482 break;
1483 case "accept":
1484 receiveAccept(from, serverMessageId, timestamp);
1485 break;
1486 default:
1487 break;
1488 }
1489 }
1490
1491 void deliverFailedProceed(final String message) {
1492 Log.d(
1493 Config.LOGTAG,
1494 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1495 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1496 webRTCWrapper.close();
1497 Log.d(
1498 Config.LOGTAG,
1499 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1500 this.finish();
1501 }
1502 }
1503
1504 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1505 final boolean originatedFromMyself =
1506 from.asBareJid().equals(id.account.getJid().asBareJid());
1507 if (originatedFromMyself) {
1508 if (transition(State.ACCEPTED)) {
1509 acceptedOnOtherDevice(serverMsgId, timestamp);
1510 } else {
1511 Log.d(
1512 Config.LOGTAG,
1513 id.account.getJid().asBareJid()
1514 + ": unable to transition to accept because already in state="
1515 + this.state);
1516 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1517 }
1518 } else {
1519 Log.d(
1520 Config.LOGTAG,
1521 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1522 }
1523 }
1524
1525 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1526 if (serverMsgId != null) {
1527 this.message.setServerMsgId(serverMsgId);
1528 }
1529 this.message.setTime(timestamp);
1530 this.message.setCarbon(true); // indicate that call was accepted on other device
1531 this.writeLogMessageSuccess(0);
1532 this.xmppConnectionService
1533 .getNotificationService()
1534 .cancelIncomingCallNotification();
1535 this.finish();
1536 }
1537
1538 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1539 final boolean originatedFromMyself =
1540 from.asBareJid().equals(id.account.getJid().asBareJid());
1541 // reject from another one of my clients
1542 if (originatedFromMyself) {
1543 receiveRejectFromMyself(serverMsgId, timestamp);
1544 } else if (isInitiator()) {
1545 if (from.equals(id.with)) {
1546 receiveRejectFromResponder();
1547 } else {
1548 Log.d(
1549 Config.LOGTAG,
1550 id.account.getJid()
1551 + ": ignoring reject from "
1552 + from
1553 + " for session with "
1554 + id.with);
1555 }
1556 } else {
1557 Log.d(
1558 Config.LOGTAG,
1559 id.account.getJid()
1560 + ": ignoring reject from "
1561 + from
1562 + " for session with "
1563 + id.with);
1564 }
1565 }
1566
1567 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1568 if (transition(State.REJECTED)) {
1569 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1570 this.finish();
1571 if (serverMsgId != null) {
1572 this.message.setServerMsgId(serverMsgId);
1573 }
1574 this.message.setTime(timestamp);
1575 this.message.setCarbon(true); // indicate that call was rejected on other device
1576 writeLogMessageMissed();
1577 } else {
1578 Log.d(
1579 Config.LOGTAG,
1580 "not able to transition into REJECTED because already in " + this.state);
1581 }
1582 }
1583
1584 private void receiveRejectFromResponder() {
1585 if (isInState(State.PROCEED)) {
1586 Log.d(
1587 Config.LOGTAG,
1588 id.account.getJid()
1589 + ": received reject while still in proceed. callee reconsidered");
1590 closeTransitionLogFinish(State.REJECTED_RACED);
1591 return;
1592 }
1593 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1594 Log.d(
1595 Config.LOGTAG,
1596 id.account.getJid()
1597 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1598 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1599 return;
1600 }
1601 Log.d(
1602 Config.LOGTAG,
1603 id.account.getJid()
1604 + ": ignoring reject from responder because already in state "
1605 + this.state);
1606 }
1607
1608 private void receivePropose(
1609 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1610 final boolean originatedFromMyself =
1611 from.asBareJid().equals(id.account.getJid().asBareJid());
1612 if (originatedFromMyself) {
1613 Log.d(
1614 Config.LOGTAG,
1615 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1616 } else if (transition(
1617 State.PROPOSED,
1618 () -> {
1619 final Collection<RtpDescription> descriptions =
1620 Collections2.transform(
1621 Collections2.filter(
1622 propose.getDescriptions(),
1623 d -> d instanceof RtpDescription),
1624 input -> (RtpDescription) input);
1625 final Collection<Media> media =
1626 Collections2.transform(descriptions, RtpDescription::getMedia);
1627 Preconditions.checkState(
1628 !media.contains(Media.UNKNOWN),
1629 "RTP descriptions contain unknown media");
1630 Log.d(
1631 Config.LOGTAG,
1632 id.account.getJid().asBareJid()
1633 + ": received session proposal from "
1634 + from
1635 + " for "
1636 + media);
1637 this.proposedMedia = Sets.newHashSet(media);
1638 })) {
1639 if (serverMsgId != null) {
1640 this.message.setServerMsgId(serverMsgId);
1641 }
1642 this.message.setTime(timestamp);
1643 startRinging();
1644 } else {
1645 Log.d(
1646 Config.LOGTAG,
1647 id.account.getJid()
1648 + ": ignoring session proposal because already in "
1649 + state);
1650 }
1651 }
1652
1653 private void startRinging() {
1654 Log.d(
1655 Config.LOGTAG,
1656 id.account.getJid().asBareJid()
1657 + ": received call from "
1658 + id.with
1659 + ". start ringing");
1660 ringingTimeoutFuture =
1661 jingleConnectionManager.schedule(
1662 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1663 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1664 }
1665
1666 private synchronized void ringingTimeout() {
1667 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1668 switch (this.state) {
1669 case PROPOSED:
1670 message.markUnread();
1671 rejectCallFromProposed();
1672 break;
1673 case SESSION_INITIALIZED:
1674 message.markUnread();
1675 rejectCallFromSessionInitiate();
1676 break;
1677 }
1678 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1679 }
1680
1681 private void cancelRingingTimeout() {
1682 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1683 if (future != null && !future.isCancelled()) {
1684 future.cancel(false);
1685 }
1686 }
1687
1688 private void receiveProceed(
1689 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1690 final Set<Media> media =
1691 Preconditions.checkNotNull(
1692 this.proposedMedia, "Proposed media has to be set before handling proceed");
1693 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1694 if (from.equals(id.with)) {
1695 if (isInitiator()) {
1696 if (transition(State.PROCEED)) {
1697 if (serverMsgId != null) {
1698 this.message.setServerMsgId(serverMsgId);
1699 }
1700 this.message.setTime(timestamp);
1701 final Integer remoteDeviceId = proceed.getDeviceId();
1702 if (isOmemoEnabled()) {
1703 this.omemoVerification.setDeviceId(remoteDeviceId);
1704 } else {
1705 if (remoteDeviceId != null) {
1706 Log.d(
1707 Config.LOGTAG,
1708 id.account.getJid().asBareJid()
1709 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1710 }
1711 this.omemoVerification.setDeviceId(null);
1712 }
1713 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1714 } else {
1715 Log.d(
1716 Config.LOGTAG,
1717 String.format(
1718 "%s: ignoring proceed because already in %s",
1719 id.account.getJid().asBareJid(), this.state));
1720 }
1721 } else {
1722 Log.d(
1723 Config.LOGTAG,
1724 String.format(
1725 "%s: ignoring proceed because we were not initializing",
1726 id.account.getJid().asBareJid()));
1727 }
1728 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1729 if (transition(State.ACCEPTED)) {
1730 Log.d(
1731 Config.LOGTAG,
1732 id.account.getJid().asBareJid()
1733 + ": moved session with "
1734 + id.with
1735 + " into state accepted after received carbon copied proceed");
1736 acceptedOnOtherDevice(serverMsgId, timestamp);
1737 }
1738 } else {
1739 Log.d(
1740 Config.LOGTAG,
1741 String.format(
1742 "%s: ignoring proceed from %s. was expected from %s",
1743 id.account.getJid().asBareJid(), from, id.with));
1744 }
1745 }
1746
1747 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1748 if (from.equals(id.with)) {
1749 final State target =
1750 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1751 if (transition(target)) {
1752 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1753 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1754 Log.d(
1755 Config.LOGTAG,
1756 id.account.getJid().asBareJid()
1757 + ": session with "
1758 + id.with
1759 + " has been retracted (serverMsgId="
1760 + serverMsgId
1761 + ")");
1762 if (serverMsgId != null) {
1763 this.message.setServerMsgId(serverMsgId);
1764 }
1765 this.message.setTime(timestamp);
1766 if (target == State.RETRACTED) {
1767 this.message.markUnread();
1768 }
1769 writeLogMessageMissed();
1770 finish();
1771 } else {
1772 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1773 }
1774 } else {
1775 // TODO parse retract from self
1776 Log.d(
1777 Config.LOGTAG,
1778 id.account.getJid().asBareJid()
1779 + ": received retract from "
1780 + from
1781 + ". expected retract from"
1782 + id.with
1783 + ". ignoring");
1784 }
1785 }
1786
1787 public void sendSessionInitiate() {
1788 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1789 }
1790
1791 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1792 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1793 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1794 }
1795
1796 private synchronized void sendSessionInitiate(
1797 final Set<Media> media,
1798 final State targetState,
1799 final List<PeerConnection.IceServer> iceServers) {
1800 if (isTerminated()) {
1801 Log.w(
1802 Config.LOGTAG,
1803 id.account.getJid().asBareJid()
1804 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1805 return;
1806 }
1807 try {
1808 setupWebRTC(media, iceServers);
1809 } catch (final WebRTCWrapper.InitializationException e) {
1810 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1811 webRTCWrapper.close();
1812 sendRetract(Reason.ofThrowable(e));
1813 return;
1814 }
1815 try {
1816 org.webrtc.SessionDescription webRTCSessionDescription =
1817 this.webRTCWrapper.setLocalDescription().get();
1818 prepareSessionInitiate(webRTCSessionDescription, targetState);
1819 } catch (final Exception e) {
1820 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1821 // exceptions
1822 failureToInitiateSession(e, targetState);
1823 }
1824 }
1825
1826 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1827 if (isTerminated()) {
1828 return;
1829 }
1830 Log.d(
1831 Config.LOGTAG,
1832 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1833 Throwables.getRootCause(throwable));
1834 webRTCWrapper.close();
1835 final Reason reason = Reason.ofThrowable(throwable);
1836 if (isInState(targetState)) {
1837 sendSessionTerminate(reason, throwable.getMessage());
1838 } else {
1839 sendRetract(reason);
1840 }
1841 }
1842
1843 private void sendRetract(final Reason reason) {
1844 // TODO embed reason into retract
1845 sendJingleMessage("retract", id.with.asBareJid());
1846 transitionOrThrow(reasonToState(reason));
1847 this.finish();
1848 }
1849
1850 private void prepareSessionInitiate(
1851 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1852 final SessionDescription sessionDescription =
1853 SessionDescription.parse(webRTCSessionDescription.description);
1854 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1855 this.initiatorRtpContentMap = rtpContentMap;
1856 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1857 encryptSessionInitiate(rtpContentMap);
1858 Futures.addCallback(
1859 outgoingContentMapFuture,
1860 new FutureCallback<RtpContentMap>() {
1861 @Override
1862 public void onSuccess(final RtpContentMap outgoingContentMap) {
1863 sendSessionInitiate(outgoingContentMap, targetState);
1864 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1865 }
1866
1867 @Override
1868 public void onFailure(@NonNull final Throwable throwable) {
1869 failureToInitiateSession(throwable, targetState);
1870 }
1871 },
1872 MoreExecutors.directExecutor());
1873 }
1874
1875 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1876 if (isTerminated()) {
1877 Log.w(
1878 Config.LOGTAG,
1879 id.account.getJid().asBareJid()
1880 + ": preparing session was too slow. already terminated. nothing to do.");
1881 return;
1882 }
1883 this.transitionOrThrow(targetState);
1884 final JinglePacket sessionInitiate =
1885 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1886 send(sessionInitiate);
1887 }
1888
1889 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1890 final RtpContentMap rtpContentMap) {
1891 if (this.omemoVerification.hasDeviceId()) {
1892 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1893 verifiedPayloadFuture =
1894 id.account
1895 .getAxolotlService()
1896 .encrypt(
1897 rtpContentMap,
1898 id.with,
1899 omemoVerification.getDeviceId());
1900 final ListenableFuture<RtpContentMap> future =
1901 Futures.transform(
1902 verifiedPayloadFuture,
1903 verifiedPayload -> {
1904 omemoVerification.setSessionFingerprint(
1905 verifiedPayload.getFingerprint());
1906 return verifiedPayload.getPayload();
1907 },
1908 MoreExecutors.directExecutor());
1909 if (Config.REQUIRE_RTP_VERIFICATION) {
1910 return future;
1911 }
1912 return Futures.catching(
1913 future,
1914 CryptoFailedException.class,
1915 e -> {
1916 Log.w(
1917 Config.LOGTAG,
1918 id.account.getJid().asBareJid()
1919 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1920 e);
1921 return rtpContentMap;
1922 },
1923 MoreExecutors.directExecutor());
1924 } else {
1925 return Futures.immediateFuture(rtpContentMap);
1926 }
1927 }
1928
1929 private void sendSessionTerminate(final Reason reason) {
1930 sendSessionTerminate(reason, null);
1931 }
1932
1933 private void sendSessionTerminate(final Reason reason, final String text) {
1934 final State previous = this.state;
1935 final State target = reasonToState(reason);
1936 transitionOrThrow(target);
1937 if (previous != State.NULL) {
1938 writeLogMessage(target);
1939 }
1940 final JinglePacket jinglePacket =
1941 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1942 jinglePacket.setReason(reason, text);
1943 Log.d(Config.LOGTAG, jinglePacket.toString());
1944 send(jinglePacket);
1945 finish();
1946 }
1947
1948 private void sendTransportInfo(
1949 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1950 final RtpContentMap transportInfo;
1951 try {
1952 final RtpContentMap rtpContentMap =
1953 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1954 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1955 } catch (final Exception e) {
1956 Log.d(
1957 Config.LOGTAG,
1958 id.account.getJid().asBareJid()
1959 + ": unable to prepare transport-info from candidate for content="
1960 + contentName);
1961 return;
1962 }
1963 final JinglePacket jinglePacket =
1964 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1965 send(jinglePacket);
1966 }
1967
1968 private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
1969 // TODO send all candidates in one transport-info
1970 }
1971
1972
1973 private void send(final JinglePacket jinglePacket) {
1974 jinglePacket.setTo(id.with);
1975 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1976 }
1977
1978 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1979 if (response.getType() == IqPacket.TYPE.ERROR) {
1980 handleIqErrorResponse(response);
1981 return;
1982 }
1983 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1984 handleIqTimeoutResponse(response);
1985 }
1986 }
1987
1988 private void handleIqErrorResponse(final IqPacket response) {
1989 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1990 final String errorCondition = response.getErrorCondition();
1991 Log.d(
1992 Config.LOGTAG,
1993 id.account.getJid().asBareJid()
1994 + ": received IQ-error from "
1995 + response.getFrom()
1996 + " in RTP session. "
1997 + errorCondition);
1998 if (isTerminated()) {
1999 Log.i(
2000 Config.LOGTAG,
2001 id.account.getJid().asBareJid()
2002 + ": ignoring error because session was already terminated");
2003 return;
2004 }
2005 this.webRTCWrapper.close();
2006 final State target;
2007 if (Arrays.asList(
2008 "service-unavailable",
2009 "recipient-unavailable",
2010 "remote-server-not-found",
2011 "remote-server-timeout")
2012 .contains(errorCondition)) {
2013 target = State.TERMINATED_CONNECTIVITY_ERROR;
2014 } else {
2015 target = State.TERMINATED_APPLICATION_FAILURE;
2016 }
2017 transitionOrThrow(target);
2018 this.finish();
2019 }
2020
2021 private void handleIqTimeoutResponse(final IqPacket response) {
2022 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
2023 Log.d(
2024 Config.LOGTAG,
2025 id.account.getJid().asBareJid()
2026 + ": received IQ timeout in RTP session with "
2027 + id.with
2028 + ". terminating with connectivity error");
2029 if (isTerminated()) {
2030 Log.i(
2031 Config.LOGTAG,
2032 id.account.getJid().asBareJid()
2033 + ": ignoring error because session was already terminated");
2034 return;
2035 }
2036 this.webRTCWrapper.close();
2037 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
2038 this.finish();
2039 }
2040
2041 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
2042 Log.d(
2043 Config.LOGTAG,
2044 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
2045 this.webRTCWrapper.close();
2046 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
2047 respondWithOutOfOrder(jinglePacket);
2048 this.finish();
2049 }
2050
2051 private void respondWithTieBreak(final JinglePacket jinglePacket) {
2052 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
2053 }
2054
2055 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
2056 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
2057 }
2058
2059 private void respondWithItemNotFound(final JinglePacket jinglePacket) {
2060 respondWithJingleError(jinglePacket, null, "item-not-found", "cancel");
2061 }
2062
2063 void respondWithJingleError(
2064 final IqPacket original,
2065 String jingleCondition,
2066 String condition,
2067 String conditionType) {
2068 jingleConnectionManager.respondWithJingleError(
2069 id.account, original, jingleCondition, condition, conditionType);
2070 }
2071
2072 private void respondOk(final JinglePacket jinglePacket) {
2073 xmppConnectionService.sendIqPacket(
2074 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
2075 }
2076
2077 public RtpEndUserState getEndUserState() {
2078 switch (this.state) {
2079 case NULL:
2080 case PROPOSED:
2081 case SESSION_INITIALIZED:
2082 if (isInitiator()) {
2083 return RtpEndUserState.RINGING;
2084 } else {
2085 return RtpEndUserState.INCOMING_CALL;
2086 }
2087 case PROCEED:
2088 if (isInitiator()) {
2089 return RtpEndUserState.RINGING;
2090 } else {
2091 return RtpEndUserState.ACCEPTING_CALL;
2092 }
2093 case SESSION_INITIALIZED_PRE_APPROVED:
2094 if (isInitiator()) {
2095 return RtpEndUserState.RINGING;
2096 } else {
2097 return RtpEndUserState.CONNECTING;
2098 }
2099 case SESSION_ACCEPTED:
2100 final ContentAddition ca = getPendingContentAddition();
2101 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
2102 return RtpEndUserState.INCOMING_CONTENT_ADD;
2103 }
2104 return getPeerConnectionStateAsEndUserState();
2105 case REJECTED:
2106 case REJECTED_RACED:
2107 case TERMINATED_DECLINED_OR_BUSY:
2108 if (isInitiator()) {
2109 return RtpEndUserState.DECLINED_OR_BUSY;
2110 } else {
2111 return RtpEndUserState.ENDED;
2112 }
2113 case TERMINATED_SUCCESS:
2114 case ACCEPTED:
2115 case RETRACTED:
2116 case TERMINATED_CANCEL_OR_TIMEOUT:
2117 return RtpEndUserState.ENDED;
2118 case RETRACTED_RACED:
2119 if (isInitiator()) {
2120 return RtpEndUserState.ENDED;
2121 } else {
2122 return RtpEndUserState.RETRACTED;
2123 }
2124 case TERMINATED_CONNECTIVITY_ERROR:
2125 return zeroDuration()
2126 ? RtpEndUserState.CONNECTIVITY_ERROR
2127 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2128 case TERMINATED_APPLICATION_FAILURE:
2129 return RtpEndUserState.APPLICATION_ERROR;
2130 case TERMINATED_SECURITY_ERROR:
2131 return RtpEndUserState.SECURITY_ERROR;
2132 }
2133 throw new IllegalStateException(
2134 String.format("%s has no equivalent EndUserState", this.state));
2135 }
2136
2137 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2138 final PeerConnection.PeerConnectionState state;
2139 try {
2140 state = webRTCWrapper.getState();
2141 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2142 // We usually close the WebRTCWrapper *before* transitioning so we might still
2143 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2144 return RtpEndUserState.ENDING_CALL;
2145 }
2146 switch (state) {
2147 case CONNECTED:
2148 return RtpEndUserState.CONNECTED;
2149 case NEW:
2150 case CONNECTING:
2151 return RtpEndUserState.CONNECTING;
2152 case CLOSED:
2153 return RtpEndUserState.ENDING_CALL;
2154 default:
2155 return zeroDuration()
2156 ? RtpEndUserState.CONNECTIVITY_ERROR
2157 : RtpEndUserState.RECONNECTING;
2158 }
2159 }
2160
2161 public ContentAddition getPendingContentAddition() {
2162 final RtpContentMap in = this.incomingContentAdd;
2163 final RtpContentMap out = this.outgoingContentAdd;
2164 if (out != null) {
2165 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2166 } else if (in != null) {
2167 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2168 } else {
2169 return null;
2170 }
2171 }
2172
2173 public Set<Media> getMedia() {
2174 final State current = getState();
2175 if (current == State.NULL) {
2176 if (isInitiator()) {
2177 return Preconditions.checkNotNull(
2178 this.proposedMedia, "RTP connection has not been initialized properly");
2179 }
2180 throw new IllegalStateException("RTP connection has not been initialized yet");
2181 }
2182 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2183 return Preconditions.checkNotNull(
2184 this.proposedMedia, "RTP connection has not been initialized properly");
2185 }
2186 final RtpContentMap localContentMap = getLocalContentMap();
2187 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2188 if (localContentMap != null) {
2189 return localContentMap.getMedia();
2190 } else if (initiatorContentMap != null) {
2191 return initiatorContentMap.getMedia();
2192 } else if (isTerminated()) {
2193 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2194 } else {
2195 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2196 }
2197 }
2198
2199 public boolean isVerified() {
2200 final String fingerprint = this.omemoVerification.getFingerprint();
2201 if (fingerprint == null) {
2202 return false;
2203 }
2204 final FingerprintStatus status =
2205 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2206 return status != null && status.isVerified();
2207 }
2208
2209 public boolean addMedia(final Media media) {
2210 final Set<Media> currentMedia = getMedia();
2211 if (currentMedia.contains(media)) {
2212 throw new IllegalStateException(String.format("%s has already been proposed", media));
2213 }
2214 // TODO add state protection - can only add while ACCEPTED or so
2215 Log.d(Config.LOGTAG,"adding media: "+media);
2216 return webRTCWrapper.addTrack(media);
2217 }
2218
2219 public synchronized void acceptCall() {
2220 switch (this.state) {
2221 case PROPOSED:
2222 cancelRingingTimeout();
2223 acceptCallFromProposed();
2224 break;
2225 case SESSION_INITIALIZED:
2226 cancelRingingTimeout();
2227 acceptCallFromSessionInitialized();
2228 break;
2229 case ACCEPTED:
2230 Log.w(
2231 Config.LOGTAG,
2232 id.account.getJid().asBareJid()
2233 + ": the call has already been accepted with another client. UI was just lagging behind");
2234 break;
2235 case PROCEED:
2236 case SESSION_ACCEPTED:
2237 Log.w(
2238 Config.LOGTAG,
2239 id.account.getJid().asBareJid()
2240 + ": the call has already been accepted. user probably double tapped the UI");
2241 break;
2242 default:
2243 throw new IllegalStateException("Can not accept call from " + this.state);
2244 }
2245 }
2246
2247 public void notifyPhoneCall() {
2248 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2249 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2250 rejectCall();
2251 } else {
2252 endCall();
2253 }
2254 }
2255
2256 public synchronized void rejectCall() {
2257 if (isTerminated()) {
2258 Log.w(
2259 Config.LOGTAG,
2260 id.account.getJid().asBareJid()
2261 + ": received rejectCall() when session has already been terminated. nothing to do");
2262 return;
2263 }
2264 switch (this.state) {
2265 case PROPOSED:
2266 rejectCallFromProposed();
2267 break;
2268 case SESSION_INITIALIZED:
2269 rejectCallFromSessionInitiate();
2270 break;
2271 default:
2272 throw new IllegalStateException("Can not reject call from " + this.state);
2273 }
2274 }
2275
2276 public synchronized void endCall() {
2277 if (isTerminated()) {
2278 Log.w(
2279 Config.LOGTAG,
2280 id.account.getJid().asBareJid()
2281 + ": received endCall() when session has already been terminated. nothing to do");
2282 return;
2283 }
2284 if (isInState(State.PROPOSED) && !isInitiator()) {
2285 rejectCallFromProposed();
2286 return;
2287 }
2288 if (isInState(State.PROCEED)) {
2289 if (isInitiator()) {
2290 retractFromProceed();
2291 } else {
2292 rejectCallFromProceed();
2293 }
2294 return;
2295 }
2296 if (isInitiator()
2297 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2298 this.webRTCWrapper.close();
2299 sendSessionTerminate(Reason.CANCEL);
2300 return;
2301 }
2302 if (isInState(State.SESSION_INITIALIZED)) {
2303 rejectCallFromSessionInitiate();
2304 return;
2305 }
2306 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2307 this.webRTCWrapper.close();
2308 sendSessionTerminate(Reason.SUCCESS);
2309 return;
2310 }
2311 if (isInState(
2312 State.TERMINATED_APPLICATION_FAILURE,
2313 State.TERMINATED_CONNECTIVITY_ERROR,
2314 State.TERMINATED_DECLINED_OR_BUSY)) {
2315 Log.d(
2316 Config.LOGTAG,
2317 "ignoring request to end call because already in state " + this.state);
2318 return;
2319 }
2320 throw new IllegalStateException(
2321 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2322 }
2323
2324 private void retractFromProceed() {
2325 Log.d(Config.LOGTAG, "retract from proceed");
2326 this.sendJingleMessage("retract");
2327 closeTransitionLogFinish(State.RETRACTED_RACED);
2328 }
2329
2330 private void closeTransitionLogFinish(final State state) {
2331 this.webRTCWrapper.close();
2332 transitionOrThrow(state);
2333 writeLogMessage(state);
2334 finish();
2335 }
2336
2337 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2338 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2339 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2340 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2341 }
2342
2343 private void acceptCallFromProposed() {
2344 transitionOrThrow(State.PROCEED);
2345 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2346 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2347 this.sendJingleMessage("proceed");
2348 }
2349
2350 private void rejectCallFromProposed() {
2351 transitionOrThrow(State.REJECTED);
2352 writeLogMessageMissed();
2353 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2354 this.sendJingleMessage("reject");
2355 finish();
2356 }
2357
2358 private void rejectCallFromProceed() {
2359 this.sendJingleMessage("reject");
2360 closeTransitionLogFinish(State.REJECTED_RACED);
2361 }
2362
2363 private void rejectCallFromSessionInitiate() {
2364 webRTCWrapper.close();
2365 sendSessionTerminate(Reason.DECLINE);
2366 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2367 }
2368
2369 private void sendJingleMessage(final String action) {
2370 sendJingleMessage(action, id.with);
2371 }
2372
2373 private void sendJingleMessage(final String action, final Jid to) {
2374 final MessagePacket messagePacket = new MessagePacket();
2375 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2376 messagePacket.setTo(to);
2377 final Element intent =
2378 messagePacket
2379 .addChild(action, Namespace.JINGLE_MESSAGE)
2380 .setAttribute("id", id.sessionId);
2381 if ("proceed".equals(action)) {
2382 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2383 if (isOmemoEnabled()) {
2384 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2385 final Element device =
2386 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2387 device.setAttribute("id", deviceId);
2388 }
2389 }
2390 messagePacket.addChild("store", "urn:xmpp:hints");
2391 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2392 }
2393
2394 private boolean isOmemoEnabled() {
2395 final Conversational conversational = message.getConversation();
2396 if (conversational instanceof Conversation) {
2397 return ((Conversation) conversational).getNextEncryption()
2398 == Message.ENCRYPTION_AXOLOTL;
2399 }
2400 return false;
2401 }
2402
2403 private void acceptCallFromSessionInitialized() {
2404 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2405 sendSessionAccept();
2406 }
2407
2408 private synchronized boolean isInState(State... state) {
2409 return Arrays.asList(state).contains(this.state);
2410 }
2411
2412 private boolean transition(final State target) {
2413 return transition(target, null);
2414 }
2415
2416 private synchronized boolean transition(final State target, final Runnable runnable) {
2417 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2418 if (validTransitions != null && validTransitions.contains(target)) {
2419 this.state = target;
2420 if (runnable != null) {
2421 runnable.run();
2422 }
2423 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2424 updateEndUserState();
2425 updateOngoingCallNotification();
2426 return true;
2427 } else {
2428 return false;
2429 }
2430 }
2431
2432 void transitionOrThrow(final State target) {
2433 if (!transition(target)) {
2434 throw new IllegalStateException(
2435 String.format("Unable to transition from %s to %s", this.state, target));
2436 }
2437 }
2438
2439 @Override
2440 public void onIceCandidate(final IceCandidate iceCandidate) {
2441 final RtpContentMap rtpContentMap =
2442 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2443 final IceUdpTransportInfo.Credentials credentials;
2444 try {
2445 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2446 } catch (final IllegalArgumentException e) {
2447 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2448 return;
2449 }
2450 final String uFrag = credentials.ufrag;
2451 final IceUdpTransportInfo.Candidate candidate =
2452 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2453 if (candidate == null) {
2454 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2455 return;
2456 }
2457 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2458 sendTransportInfo(iceCandidate.sdpMid, candidate);
2459 }
2460
2461 @Override
2462 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2463 Log.d(
2464 Config.LOGTAG,
2465 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2466 this.stateHistory.add(newState);
2467 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2468 this.sessionDuration.start();
2469 updateOngoingCallNotification();
2470 } else if (this.sessionDuration.isRunning()) {
2471 this.sessionDuration.stop();
2472 updateOngoingCallNotification();
2473 }
2474
2475 final boolean neverConnected =
2476 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2477
2478 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2479 if (neverConnected) {
2480 if (isTerminated()) {
2481 Log.d(
2482 Config.LOGTAG,
2483 id.account.getJid().asBareJid()
2484 + ": not sending session-terminate after connectivity error because session is already in state "
2485 + this.state);
2486 return;
2487 }
2488 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2489 return;
2490 } else {
2491 this.restartIce();
2492 }
2493 }
2494 updateEndUserState();
2495 }
2496
2497 private void restartIce() {
2498 this.stateHistory.clear();
2499 this.webRTCWrapper.restartIceAsync();
2500 }
2501
2502 @Override
2503 public void onRenegotiationNeeded() {
2504 this.webRTCWrapper.execute(this::renegotiate);
2505 }
2506
2507 private void renegotiate() {
2508 final SessionDescription sessionDescription;
2509 try {
2510 sessionDescription = setLocalSessionDescription();
2511 } catch (final Exception e) {
2512 final Throwable cause = Throwables.getRootCause(e);
2513 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2514 webRTCWrapper.close();
2515 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2516 return;
2517 }
2518 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2519 final RtpContentMap currentContentMap = getLocalContentMap();
2520 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2521 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2522
2523 Log.d(
2524 Config.LOGTAG,
2525 id.getAccount().getJid().asBareJid()
2526 + ": renegotiate. iceRestart="
2527 + iceRestart
2528 + " content id diff="
2529 + diff);
2530
2531 if (diff.hasModifications() && iceRestart) {
2532 webRTCWrapper.close();
2533 sendSessionTerminate(
2534 Reason.FAILED_APPLICATION,
2535 "WebRTC unexpectedly tried to modify content and transport at once");
2536 return;
2537 }
2538
2539 if (iceRestart) {
2540 initiateIceRestart(rtpContentMap);
2541 return;
2542 } else if (diff.isEmpty()) {
2543 Log.d(
2544 Config.LOGTAG,
2545 "renegotiation. nothing to do. SignalingState="
2546 + this.webRTCWrapper.getSignalingState());
2547 }
2548
2549 if (diff.added.size() > 0) {
2550 modifyLocalContentMap(rtpContentMap);
2551 sendContentAdd(rtpContentMap, diff.added);
2552 }
2553 }
2554
2555 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2556 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2557 final JinglePacket jinglePacket =
2558 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2559 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2560 jinglePacket.setTo(id.with);
2561 xmppConnectionService.sendIqPacket(
2562 id.account,
2563 jinglePacket,
2564 (account, response) -> {
2565 if (response.getType() == IqPacket.TYPE.RESULT) {
2566 Log.d(Config.LOGTAG, "received success to our ice restart");
2567 setLocalContentMap(rtpContentMap);
2568 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2569 return;
2570 }
2571 if (response.getType() == IqPacket.TYPE.ERROR) {
2572 if (isTieBreak(response)) {
2573 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2574 return;
2575 }
2576 handleIqErrorResponse(response);
2577 }
2578 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2579 handleIqTimeoutResponse(response);
2580 }
2581 });
2582 }
2583
2584 private boolean isTieBreak(final IqPacket response) {
2585 final Element error = response.findChild("error");
2586 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2587 }
2588
2589 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2590 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2591 this.outgoingContentAdd = contentAdd;
2592 final JinglePacket jinglePacket =
2593 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2594 jinglePacket.setTo(id.with);
2595 xmppConnectionService.sendIqPacket(
2596 id.account,
2597 jinglePacket,
2598 (connection, response) -> {
2599 if (response.getType() == IqPacket.TYPE.RESULT) {
2600 Log.d(
2601 Config.LOGTAG,
2602 id.getAccount().getJid().asBareJid()
2603 + ": received ACK to our content-add");
2604 return;
2605 }
2606 if (response.getType() == IqPacket.TYPE.ERROR) {
2607 if (isTieBreak(response)) {
2608 this.outgoingContentAdd = null;
2609 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2610 return;
2611 }
2612 handleIqErrorResponse(response);
2613 }
2614 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2615 handleIqTimeoutResponse(response);
2616 }
2617 });
2618 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2619 }
2620
2621 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2622 if (isInitiator()) {
2623 this.initiatorRtpContentMap = rtpContentMap;
2624 } else {
2625 this.responderRtpContentMap = rtpContentMap;
2626 }
2627 }
2628
2629 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2630 if (isInitiator()) {
2631 this.responderRtpContentMap = rtpContentMap;
2632 } else {
2633 this.initiatorRtpContentMap = rtpContentMap;
2634 }
2635 }
2636
2637 // this method is to be used for content map modifications that modify media
2638 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2639 final RtpContentMap activeContents = rtpContentMap.activeContents();
2640 setLocalContentMap(activeContents);
2641 this.webRTCWrapper.switchSpeakerPhonePreference(
2642 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2643 updateEndUserState();
2644 }
2645
2646 private SessionDescription setLocalSessionDescription()
2647 throws ExecutionException, InterruptedException {
2648 final org.webrtc.SessionDescription sessionDescription =
2649 this.webRTCWrapper.setLocalDescription().get();
2650 return SessionDescription.parse(sessionDescription.description);
2651 }
2652
2653 private void closeWebRTCSessionAfterFailedConnection() {
2654 this.webRTCWrapper.close();
2655 synchronized (this) {
2656 if (isTerminated()) {
2657 Log.d(
2658 Config.LOGTAG,
2659 id.account.getJid().asBareJid()
2660 + ": no need to send session-terminate after failed connection. Other party already did");
2661 return;
2662 }
2663 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2664 }
2665 }
2666
2667 public boolean zeroDuration() {
2668 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2669 }
2670
2671 public long getCallDuration() {
2672 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2673 }
2674
2675 public AppRTCAudioManager getAudioManager() {
2676 return webRTCWrapper.getAudioManager();
2677 }
2678
2679 public boolean isMicrophoneEnabled() {
2680 return webRTCWrapper.isMicrophoneEnabled();
2681 }
2682
2683 public boolean setMicrophoneEnabled(final boolean enabled) {
2684 return webRTCWrapper.setMicrophoneEnabled(enabled);
2685 }
2686
2687 public boolean isVideoEnabled() {
2688 return webRTCWrapper.isVideoEnabled();
2689 }
2690
2691 public void setVideoEnabled(final boolean enabled) {
2692 webRTCWrapper.setVideoEnabled(enabled);
2693 }
2694
2695 public boolean isCameraSwitchable() {
2696 return webRTCWrapper.isCameraSwitchable();
2697 }
2698
2699 public boolean isFrontCamera() {
2700 return webRTCWrapper.isFrontCamera();
2701 }
2702
2703 public ListenableFuture<Boolean> switchCamera() {
2704 return webRTCWrapper.switchCamera();
2705 }
2706
2707 @Override
2708 public void onAudioDeviceChanged(
2709 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2710 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2711 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2712 selectedAudioDevice, availableAudioDevices);
2713 }
2714
2715 private void updateEndUserState() {
2716 final RtpEndUserState endUserState = getEndUserState();
2717 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2718 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2719 id.account, id.with, id.sessionId, endUserState);
2720 }
2721
2722 private void updateOngoingCallNotification() {
2723 final State state = this.state;
2724 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2725 final boolean reconnecting;
2726 if (state == State.SESSION_ACCEPTED) {
2727 reconnecting =
2728 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2729 } else {
2730 reconnecting = false;
2731 }
2732 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2733 } else {
2734 xmppConnectionService.removeOngoingCall();
2735 }
2736 }
2737
2738 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2739 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2740 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2741 request.setTo(id.account.getDomain());
2742 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2743 xmppConnectionService.sendIqPacket(
2744 id.account,
2745 request,
2746 (account, response) -> {
2747 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2748 new ImmutableList.Builder<>();
2749 if (response.getType() == IqPacket.TYPE.RESULT) {
2750 final Element services =
2751 response.findChild(
2752 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2753 final List<Element> children =
2754 services == null
2755 ? Collections.emptyList()
2756 : services.getChildren();
2757 for (final Element child : children) {
2758 if ("service".equals(child.getName())) {
2759 final String type = child.getAttribute("type");
2760 final String host = child.getAttribute("host");
2761 final String sport = child.getAttribute("port");
2762 final Integer port =
2763 sport == null ? null : Ints.tryParse(sport);
2764 final String transport = child.getAttribute("transport");
2765 final String username = child.getAttribute("username");
2766 final String password = child.getAttribute("password");
2767 if (Strings.isNullOrEmpty(host) || port == null) {
2768 continue;
2769 }
2770 if (port < 0 || port > 65535) {
2771 continue;
2772 }
2773
2774
2775
2776
2777 if (Arrays.asList("stun", "stuns", "turn", "turns")
2778 .contains(type)
2779 && Arrays.asList("udp", "tcp").contains(transport)) {
2780 if (Arrays.asList("stuns", "turns").contains(type)
2781 && "udp".equals(transport)) {
2782 Log.d(
2783 Config.LOGTAG,
2784 id.account.getJid().asBareJid()
2785 + ": skipping invalid combination of udp/tls in external services");
2786 continue;
2787 }
2788
2789 // STUN URLs do not support a query section since M110
2790 final String uri;
2791 if (Arrays.asList("stun","stuns").contains(type)) {
2792 uri = String.format("%s:%s%s", type, IP.wrapIPv6(host),port);
2793 } else {
2794 uri = String.format(
2795 "%s:%s:%s?transport=%s",
2796 type,
2797 IP.wrapIPv6(host),
2798 port,
2799 transport);
2800 }
2801
2802 final PeerConnection.IceServer.Builder iceServerBuilder =
2803 PeerConnection.IceServer.builder(uri);
2804 iceServerBuilder.setTlsCertPolicy(
2805 PeerConnection.TlsCertPolicy
2806 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2807 if (username != null && password != null) {
2808 iceServerBuilder.setUsername(username);
2809 iceServerBuilder.setPassword(password);
2810 } else if (Arrays.asList("turn", "turns").contains(type)) {
2811 // The WebRTC spec requires throwing an
2812 // InvalidAccessError when username (from libwebrtc
2813 // source coder)
2814 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2815 Log.d(
2816 Config.LOGTAG,
2817 id.account.getJid().asBareJid()
2818 + ": skipping "
2819 + type
2820 + "/"
2821 + transport
2822 + " without username and password");
2823 continue;
2824 }
2825 final PeerConnection.IceServer iceServer =
2826 iceServerBuilder.createIceServer();
2827 Log.d(
2828 Config.LOGTAG,
2829 id.account.getJid().asBareJid()
2830 + ": discovered ICE Server: "
2831 + iceServer);
2832 listBuilder.add(iceServer);
2833 }
2834 }
2835 }
2836 }
2837 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2838 if (iceServers.size() == 0) {
2839 Log.w(
2840 Config.LOGTAG,
2841 id.account.getJid().asBareJid()
2842 + ": no ICE server found "
2843 + response);
2844 }
2845 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2846 });
2847 } else {
2848 Log.w(
2849 Config.LOGTAG,
2850 id.account.getJid().asBareJid() + ": has no external service discovery");
2851 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2852 }
2853 }
2854
2855 private void finish() {
2856 if (isTerminated()) {
2857 this.cancelRingingTimeout();
2858 this.webRTCWrapper.verifyClosed();
2859 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2860 this.jingleConnectionManager.finishConnectionOrThrow(this);
2861 try {
2862 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2863 log.getParentFile().mkdirs();
2864 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2865 } catch (final IOException e) { }
2866 } else {
2867 throw new IllegalStateException(
2868 String.format("Unable to call finish from %s", this.state));
2869 }
2870 }
2871
2872 private void writeLogMessage(final State state) {
2873 final long duration = getCallDuration();
2874 if (state == State.TERMINATED_SUCCESS
2875 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2876 writeLogMessageSuccess(duration);
2877 } else {
2878 writeLogMessageMissed();
2879 }
2880 }
2881
2882 private void writeLogMessageSuccess(final long duration) {
2883 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2884 this.writeMessage();
2885 }
2886
2887 private void writeLogMessageMissed() {
2888 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2889 this.writeMessage();
2890 }
2891
2892 private void writeMessage() {
2893 final Conversational conversational = message.getConversation();
2894 if (conversational instanceof Conversation) {
2895 ((Conversation) conversational).add(this.message);
2896 xmppConnectionService.createMessageAsync(message);
2897 xmppConnectionService.updateConversationUi();
2898 } else {
2899 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2900 }
2901 }
2902
2903 public State getState() {
2904 return this.state;
2905 }
2906
2907 boolean isTerminated() {
2908 return TERMINATED.contains(this.state);
2909 }
2910
2911 public Optional<VideoTrack> getLocalVideoTrack() {
2912 return webRTCWrapper.getLocalVideoTrack();
2913 }
2914
2915 public Optional<VideoTrack> getRemoteVideoTrack() {
2916 return webRTCWrapper.getRemoteVideoTrack();
2917 }
2918
2919 public EglBase.Context getEglBaseContext() {
2920 return webRTCWrapper.getEglBaseContext();
2921 }
2922
2923 void setProposedMedia(final Set<Media> media) {
2924 this.proposedMedia = media;
2925 }
2926
2927 public void fireStateUpdate() {
2928 final RtpEndUserState endUserState = getEndUserState();
2929 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2930 id.account, id.with, id.sessionId, endUserState);
2931 }
2932
2933 public boolean isSwitchToVideoAvailable() {
2934 final boolean prerequisite =
2935 Media.audioOnly(getMedia())
2936 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2937 .contains(getEndUserState());
2938 return prerequisite && remoteHasVideoFeature();
2939 }
2940
2941 private boolean remoteHasVideoFeature() {
2942 final Contact contact = id.getContact();
2943 final Presence presence =
2944 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2945 final ServiceDiscoveryResult serviceDiscoveryResult =
2946 presence == null ? null : presence.getServiceDiscoveryResult();
2947 final List<String> features =
2948 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
2949 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
2950 }
2951
2952 private interface OnIceServersDiscovered {
2953 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2954 }
2955}