1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4import android.os.Environment;
5
6import androidx.annotation.NonNull;
7import androidx.annotation.Nullable;
8
9import com.google.common.base.Joiner;
10import com.google.common.base.Optional;
11import com.google.common.base.Preconditions;
12import com.google.common.base.Stopwatch;
13import com.google.common.base.Strings;
14import com.google.common.base.Throwables;
15import com.google.common.collect.Collections2;
16import com.google.common.collect.ImmutableList;
17import com.google.common.collect.ImmutableMap;
18import com.google.common.collect.ImmutableSet;
19import com.google.common.collect.Maps;
20import com.google.common.collect.Sets;
21import com.google.common.primitives.Ints;
22import com.google.common.util.concurrent.FutureCallback;
23import com.google.common.util.concurrent.Futures;
24import com.google.common.util.concurrent.ListenableFuture;
25import com.google.common.util.concurrent.MoreExecutors;
26
27import org.webrtc.DtmfSender;
28import org.webrtc.EglBase;
29import org.webrtc.IceCandidate;
30import org.webrtc.PeerConnection;
31import org.webrtc.VideoTrack;
32
33import java.io.File;
34import java.io.IOException;
35import java.util.Arrays;
36import java.util.Collection;
37import java.util.Collections;
38import java.util.LinkedList;
39import java.util.List;
40import java.util.Map;
41import java.util.Queue;
42import java.util.Set;
43import java.util.concurrent.ExecutionException;
44import java.util.concurrent.ScheduledFuture;
45import java.util.concurrent.TimeUnit;
46
47import eu.siacs.conversations.BuildConfig;
48import eu.siacs.conversations.Config;
49import eu.siacs.conversations.crypto.axolotl.AxolotlService;
50import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;
51import eu.siacs.conversations.crypto.axolotl.FingerprintStatus;
52import eu.siacs.conversations.entities.Account;
53import eu.siacs.conversations.entities.Contact;
54import eu.siacs.conversations.entities.Conversation;
55import eu.siacs.conversations.entities.Conversational;
56import eu.siacs.conversations.entities.Message;
57import eu.siacs.conversations.entities.Presence;
58import eu.siacs.conversations.entities.RtpSessionStatus;
59import eu.siacs.conversations.entities.ServiceDiscoveryResult;
60import eu.siacs.conversations.services.AppRTCAudioManager;
61import eu.siacs.conversations.utils.IP;
62import eu.siacs.conversations.xml.Element;
63import eu.siacs.conversations.xml.Namespace;
64import eu.siacs.conversations.xmpp.Jid;
65import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
66import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
67import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
68import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
69import eu.siacs.conversations.xmpp.jingle.stanzas.Proceed;
70import eu.siacs.conversations.xmpp.jingle.stanzas.Propose;
71import eu.siacs.conversations.xmpp.jingle.stanzas.Reason;
72import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
73import eu.siacs.conversations.xmpp.stanzas.IqPacket;
74import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
75
76public class JingleRtpConnection extends AbstractJingleConnection
77 implements WebRTCWrapper.EventCallback {
78
79 public static final List<State> STATES_SHOWING_ONGOING_CALL =
80 Arrays.asList(
81 State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED);
82 private static final long BUSY_TIME_OUT = 30;
83 private static final List<State> TERMINATED =
84 Arrays.asList(
85 State.ACCEPTED,
86 State.REJECTED,
87 State.REJECTED_RACED,
88 State.RETRACTED,
89 State.RETRACTED_RACED,
90 State.TERMINATED_SUCCESS,
91 State.TERMINATED_DECLINED_OR_BUSY,
92 State.TERMINATED_CONNECTIVITY_ERROR,
93 State.TERMINATED_CANCEL_OR_TIMEOUT,
94 State.TERMINATED_APPLICATION_FAILURE,
95 State.TERMINATED_SECURITY_ERROR);
96
97 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
98
99 static {
100 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder =
101 new ImmutableMap.Builder<>();
102 transitionBuilder.put(
103 State.NULL,
104 ImmutableList.of(
105 State.PROPOSED,
106 State.SESSION_INITIALIZED,
107 State.TERMINATED_APPLICATION_FAILURE,
108 State.TERMINATED_SECURITY_ERROR));
109 transitionBuilder.put(
110 State.PROPOSED,
111 ImmutableList.of(
112 State.ACCEPTED,
113 State.PROCEED,
114 State.REJECTED,
115 State.RETRACTED,
116 State.TERMINATED_APPLICATION_FAILURE,
117 State.TERMINATED_SECURITY_ERROR,
118 State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection
119 // rebinds
120 ));
121 transitionBuilder.put(
122 State.PROCEED,
123 ImmutableList.of(
124 State.REJECTED_RACED,
125 State.RETRACTED_RACED,
126 State.SESSION_INITIALIZED_PRE_APPROVED,
127 State.TERMINATED_SUCCESS,
128 State.TERMINATED_APPLICATION_FAILURE,
129 State.TERMINATED_SECURITY_ERROR,
130 State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error
131 // bounces of the proceed message
132 ));
133 transitionBuilder.put(
134 State.SESSION_INITIALIZED,
135 ImmutableList.of(
136 State.SESSION_ACCEPTED,
137 State.TERMINATED_SUCCESS,
138 State.TERMINATED_DECLINED_OR_BUSY,
139 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
140 // and IQ timeouts
141 State.TERMINATED_CANCEL_OR_TIMEOUT,
142 State.TERMINATED_APPLICATION_FAILURE,
143 State.TERMINATED_SECURITY_ERROR));
144 transitionBuilder.put(
145 State.SESSION_INITIALIZED_PRE_APPROVED,
146 ImmutableList.of(
147 State.SESSION_ACCEPTED,
148 State.TERMINATED_SUCCESS,
149 State.TERMINATED_DECLINED_OR_BUSY,
150 State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors
151 // and IQ timeouts
152 State.TERMINATED_CANCEL_OR_TIMEOUT,
153 State.TERMINATED_APPLICATION_FAILURE,
154 State.TERMINATED_SECURITY_ERROR));
155 transitionBuilder.put(
156 State.SESSION_ACCEPTED,
157 ImmutableList.of(
158 State.TERMINATED_SUCCESS,
159 State.TERMINATED_DECLINED_OR_BUSY,
160 State.TERMINATED_CONNECTIVITY_ERROR,
161 State.TERMINATED_CANCEL_OR_TIMEOUT,
162 State.TERMINATED_APPLICATION_FAILURE,
163 State.TERMINATED_SECURITY_ERROR));
164 VALID_TRANSITIONS = transitionBuilder.build();
165 }
166
167 private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
168 private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>>
169 pendingIceCandidates = new LinkedList<>();
170 private final OmemoVerification omemoVerification = new OmemoVerification();
171 private final Message message;
172 private State state = State.NULL;
173 private Set<Media> proposedMedia;
174 private RtpContentMap initiatorRtpContentMap;
175 private RtpContentMap responderRtpContentMap;
176 private RtpContentMap incomingContentAdd;
177 private RtpContentMap outgoingContentAdd;
178 private IceUdpTransportInfo.Setup peerDtlsSetup;
179 private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
180 private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
181 private ScheduledFuture<?> ringingTimeoutFuture;
182 private final long created = System.currentTimeMillis() / 1000L;
183
184 JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
185 super(jingleConnectionManager, id, initiator);
186 final Conversation conversation =
187 jingleConnectionManager
188 .getXmppConnectionService()
189 .findOrCreateConversation(id.account, id.with.asBareJid(), false, false);
190 this.message =
191 new Message(
192 conversation,
193 isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED,
194 Message.TYPE_RTP_SESSION,
195 id.sessionId);
196 }
197
198 private static State reasonToState(Reason reason) {
199 switch (reason) {
200 case SUCCESS:
201 return State.TERMINATED_SUCCESS;
202 case DECLINE:
203 case BUSY:
204 return State.TERMINATED_DECLINED_OR_BUSY;
205 case CANCEL:
206 case TIMEOUT:
207 return State.TERMINATED_CANCEL_OR_TIMEOUT;
208 case SECURITY_ERROR:
209 return State.TERMINATED_SECURITY_ERROR;
210 case FAILED_APPLICATION:
211 case UNSUPPORTED_TRANSPORTS:
212 case UNSUPPORTED_APPLICATIONS:
213 return State.TERMINATED_APPLICATION_FAILURE;
214 default:
215 return State.TERMINATED_CONNECTIVITY_ERROR;
216 }
217 }
218
219 @Override
220 synchronized void deliverPacket(final JinglePacket jinglePacket) {
221 switch (jinglePacket.getAction()) {
222 case SESSION_INITIATE:
223 receiveSessionInitiate(jinglePacket);
224 break;
225 case TRANSPORT_INFO:
226 receiveTransportInfo(jinglePacket);
227 break;
228 case SESSION_ACCEPT:
229 receiveSessionAccept(jinglePacket);
230 break;
231 case SESSION_TERMINATE:
232 receiveSessionTerminate(jinglePacket);
233 break;
234 case CONTENT_ADD:
235 receiveContentAdd(jinglePacket);
236 break;
237 case CONTENT_ACCEPT:
238 receiveContentAccept(jinglePacket);
239 break;
240 case CONTENT_REJECT:
241 receiveContentReject(jinglePacket);
242 break;
243 case CONTENT_REMOVE:
244 receiveContentRemove(jinglePacket);
245 break;
246 case CONTENT_MODIFY:
247 receiveContentModify(jinglePacket);
248 break;
249 default:
250 respondOk(jinglePacket);
251 Log.d(
252 Config.LOGTAG,
253 String.format(
254 "%s: received unhandled jingle action %s",
255 id.account.getJid().asBareJid(), jinglePacket.getAction()));
256 break;
257 }
258 }
259
260 @Override
261 synchronized void notifyRebound() {
262 if (isTerminated()) {
263 return;
264 }
265 webRTCWrapper.close();
266 if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) {
267 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
268 }
269 if (isInState(
270 State.SESSION_INITIALIZED,
271 State.SESSION_INITIALIZED_PRE_APPROVED,
272 State.SESSION_ACCEPTED)) {
273 // we might have already changed resources (full jid) at this point; so this might not
274 // even reach the other party
275 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
276 } else {
277 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
278 finish();
279 }
280 }
281
282 public boolean applyDtmfTone(String tone) {
283 return webRTCWrapper.applyDtmfTone(tone);
284 }
285
286 private void receiveSessionTerminate(final JinglePacket jinglePacket) {
287 respondOk(jinglePacket);
288 final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason();
289 final State previous = this.state;
290 Log.d(
291 Config.LOGTAG,
292 id.account.getJid().asBareJid()
293 + ": received session terminate reason="
294 + wrapper.reason
295 + "("
296 + Strings.nullToEmpty(wrapper.text)
297 + ") while in state "
298 + previous);
299 if (TERMINATED.contains(previous)) {
300 Log.d(
301 Config.LOGTAG,
302 id.account.getJid().asBareJid()
303 + ": ignoring session terminate because already in "
304 + previous);
305 return;
306 }
307 webRTCWrapper.close();
308 final State target = reasonToState(wrapper.reason);
309 transitionOrThrow(target);
310 writeLogMessage(target);
311 if (previous == State.PROPOSED || previous == State.SESSION_INITIALIZED) {
312 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
313 }
314 finish();
315 }
316
317 private void receiveTransportInfo(final JinglePacket jinglePacket) {
318 // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to
319 // INITIALIZED only after transport-info has been received
320 if (isInState(
321 State.NULL,
322 State.PROCEED,
323 State.SESSION_INITIALIZED,
324 State.SESSION_INITIALIZED_PRE_APPROVED,
325 State.SESSION_ACCEPTED)) {
326 final RtpContentMap contentMap;
327 try {
328 contentMap = RtpContentMap.of(jinglePacket);
329 } catch (final IllegalArgumentException | NullPointerException e) {
330 Log.d(
331 Config.LOGTAG,
332 id.account.getJid().asBareJid()
333 + ": improperly formatted contents; ignoring",
334 e);
335 respondOk(jinglePacket);
336 return;
337 }
338 receiveTransportInfo(jinglePacket, contentMap);
339 } else {
340 if (isTerminated()) {
341 respondOk(jinglePacket);
342 Log.d(
343 Config.LOGTAG,
344 id.account.getJid().asBareJid()
345 + ": ignoring out-of-order transport info; we where already terminated");
346 } else {
347 Log.d(
348 Config.LOGTAG,
349 id.account.getJid().asBareJid()
350 + ": received transport info while in state="
351 + this.state);
352 terminateWithOutOfOrder(jinglePacket);
353 }
354 }
355 }
356
357 private void receiveTransportInfo(
358 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
359 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates =
360 contentMap.contents.entrySet();
361 if (this.state == State.SESSION_ACCEPTED) {
362 // zero candidates + modified credentials are an ICE restart offer
363 if (checkForIceRestart(jinglePacket, contentMap)) {
364 return;
365 }
366 respondOk(jinglePacket);
367 try {
368 processCandidates(candidates);
369 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
370 Log.w(
371 Config.LOGTAG,
372 id.account.getJid().asBareJid()
373 + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
374 }
375 } else {
376 respondOk(jinglePacket);
377 pendingIceCandidates.addAll(candidates);
378 }
379 }
380
381 private void receiveContentAdd(final JinglePacket jinglePacket) {
382 final RtpContentMap modification;
383 try {
384 modification = RtpContentMap.of(jinglePacket);
385 modification.requireContentDescriptions();
386 } catch (final RuntimeException e) {
387 Log.d(
388 Config.LOGTAG,
389 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
390 Throwables.getRootCause(e));
391 respondOk(jinglePacket);
392 webRTCWrapper.close();
393 sendSessionTerminate(Reason.of(e), e.getMessage());
394 return;
395 }
396 if (isInState(State.SESSION_ACCEPTED)) {
397 receiveContentAdd(jinglePacket, modification);
398 } else {
399 terminateWithOutOfOrder(jinglePacket);
400 }
401 }
402
403 private void receiveContentAdd(
404 final JinglePacket jinglePacket, final RtpContentMap modification) {
405 final RtpContentMap remote = getRemoteContentMap();
406 if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
407 respondOk(jinglePacket);
408 this.webRTCWrapper.close();
409 sendSessionTerminate(
410 Reason.FAILED_APPLICATION,
411 String.format(
412 "contents with names %s already exists",
413 Joiner.on(", ").join(modification.getNames())));
414 return;
415 }
416 final ContentAddition contentAddition =
417 ContentAddition.of(ContentAddition.Direction.INCOMING, modification);
418
419 final RtpContentMap outgoing = this.outgoingContentAdd;
420 final Set<ContentAddition.Summary> outgoingContentAddSummary =
421 outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);
422
423 if (outgoingContentAddSummary.equals(contentAddition.summary)) {
424 if (isInitiator()) {
425 Log.d(
426 Config.LOGTAG,
427 id.getAccount().getJid().asBareJid()
428 + ": respond with tie break to matching content-add offer");
429 respondWithTieBreak(jinglePacket);
430 } else {
431 Log.d(
432 Config.LOGTAG,
433 id.getAccount().getJid().asBareJid()
434 + ": automatically accept matching content-add offer");
435 acceptContentAdd(contentAddition.summary, modification);
436 }
437 return;
438 }
439
440 // once we can display multiple video tracks we can be more loose with this condition
441 // theoretically it should also be fine to automatically accept audio only contents
442 if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
443 Log.d(
444 Config.LOGTAG,
445 id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
446 this.incomingContentAdd = modification;
447 respondOk(jinglePacket);
448 updateEndUserState();
449 } else {
450 respondOk(jinglePacket);
451 // TODO do we want to add a reason?
452 rejectContentAdd(modification);
453 }
454 }
455
456 private void receiveContentAccept(final JinglePacket jinglePacket) {
457 final RtpContentMap receivedContentAccept;
458 try {
459 receivedContentAccept = RtpContentMap.of(jinglePacket);
460 receivedContentAccept.requireContentDescriptions();
461 } catch (final RuntimeException e) {
462 Log.d(
463 Config.LOGTAG,
464 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
465 Throwables.getRootCause(e));
466 respondOk(jinglePacket);
467 webRTCWrapper.close();
468 sendSessionTerminate(Reason.of(e), e.getMessage());
469 return;
470 }
471
472 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
473 if (outgoingContentAdd == null) {
474 Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
475 terminateWithOutOfOrder(jinglePacket);
476 return;
477 }
478 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
479 if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
480 this.outgoingContentAdd = null;
481 respondOk(jinglePacket);
482 receiveContentAccept(receivedContentAccept);
483 } else {
484 Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
485 terminateWithOutOfOrder(jinglePacket);
486 }
487 }
488
489 private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
490 final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
491 final RtpContentMap modifiedContentMap =
492 getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);
493
494 setRemoteContentMap(modifiedContentMap);
495
496 final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());
497
498 final org.webrtc.SessionDescription sdp =
499 new org.webrtc.SessionDescription(
500 org.webrtc.SessionDescription.Type.ANSWER, answer.toString());
501
502 try {
503 this.webRTCWrapper.setRemoteDescription(sdp).get();
504 } catch (final Exception e) {
505 final Throwable cause = Throwables.getRootCause(e);
506 Log.d(
507 Config.LOGTAG,
508 id.getAccount().getJid().asBareJid()
509 + ": unable to set remote description after receiving content-accept",
510 cause);
511 webRTCWrapper.close();
512 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
513 return;
514 }
515 updateEndUserState();
516 Log.d(
517 Config.LOGTAG,
518 id.getAccount().getJid().asBareJid()
519 + ": remote has accepted content-add "
520 + ContentAddition.summary(receivedContentAccept));
521 }
522
523 private void receiveContentModify(final JinglePacket jinglePacket) {
524 final Map<String, Content.Senders> modification =
525 Maps.transformEntries(
526 jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
527 respondOk(jinglePacket);
528 Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
529 }
530
531 private void receiveContentReject(final JinglePacket jinglePacket) {
532 final RtpContentMap receivedContentReject;
533 try {
534 receivedContentReject = RtpContentMap.of(jinglePacket);
535 } catch (final RuntimeException e) {
536 Log.d(
537 Config.LOGTAG,
538 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
539 Throwables.getRootCause(e));
540 respondOk(jinglePacket);
541 this.webRTCWrapper.close();
542 sendSessionTerminate(Reason.of(e), e.getMessage());
543 return;
544 }
545
546 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
547 if (outgoingContentAdd == null) {
548 Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
549 terminateWithOutOfOrder(jinglePacket);
550 return;
551 }
552 final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
553 if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
554 this.outgoingContentAdd = null;
555 respondOk(jinglePacket);
556 Log.d(Config.LOGTAG,jinglePacket.toString());
557 receiveContentReject(ourSummary);
558 } else {
559 Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
560 terminateWithOutOfOrder(jinglePacket);
561 }
562 }
563
564 private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
565 try {
566 this.webRTCWrapper.removeTrack(Media.VIDEO);
567 final RtpContentMap localContentMap = customRollback();
568 modifyLocalContentMap(localContentMap);
569 } catch (final Exception e) {
570 final Throwable cause = Throwables.getRootCause(e);
571 Log.d(
572 Config.LOGTAG,
573 id.getAccount().getJid().asBareJid()
574 + ": unable to rollback local description after receiving content-reject",
575 cause);
576 webRTCWrapper.close();
577 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
578 return;
579 }
580 Log.d(
581 Config.LOGTAG,
582 id.getAccount().getJid().asBareJid()
583 + ": remote has rejected our content-add "
584 + summary);
585 }
586
587 private void receiveContentRemove(final JinglePacket jinglePacket) {
588 final RtpContentMap receivedContentRemove;
589 try {
590 receivedContentRemove = RtpContentMap.of(jinglePacket);
591 receivedContentRemove.requireContentDescriptions();
592 } catch (final RuntimeException e) {
593 Log.d(
594 Config.LOGTAG,
595 id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
596 Throwables.getRootCause(e));
597 respondOk(jinglePacket);
598 this.webRTCWrapper.close();
599 sendSessionTerminate(Reason.of(e), e.getMessage());
600 return;
601 }
602 respondOk(jinglePacket);
603 receiveContentRemove(receivedContentRemove);
604 }
605
606 private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
607 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
608 final Set<ContentAddition.Summary> contentAddSummary =
609 incomingContentAdd == null
610 ? Collections.emptySet()
611 : ContentAddition.summary(incomingContentAdd);
612 final Set<ContentAddition.Summary> removeSummary =
613 ContentAddition.summary(receivedContentRemove);
614 if (contentAddSummary.equals(removeSummary)) {
615 this.incomingContentAdd = null;
616 updateEndUserState();
617 } else {
618 webRTCWrapper.close();
619 sendSessionTerminate(
620 Reason.FAILED_APPLICATION,
621 String.format(
622 "%s only supports %s as a means to retract a not yet accepted %s",
623 BuildConfig.APP_NAME,
624 JinglePacket.Action.CONTENT_REMOVE,
625 JinglePacket.Action.CONTENT_ACCEPT));
626 }
627 }
628
629 public synchronized void retractContentAdd() {
630 final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
631 if (outgoingContentAdd == null) {
632 throw new IllegalStateException("Not outgoing content add");
633 }
634 try {
635 webRTCWrapper.removeTrack(Media.VIDEO);
636 final RtpContentMap localContentMap = customRollback();
637 modifyLocalContentMap(localContentMap);
638 } catch (final Exception e) {
639 final Throwable cause = Throwables.getRootCause(e);
640 Log.d(
641 Config.LOGTAG,
642 id.getAccount().getJid().asBareJid()
643 + ": unable to rollback local description after trying to retract content-add",
644 cause);
645 webRTCWrapper.close();
646 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
647 return;
648 }
649 this.outgoingContentAdd = null;
650 final JinglePacket retract =
651 outgoingContentAdd
652 .toStub()
653 .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
654 this.send(retract);
655 Log.d(
656 Config.LOGTAG,
657 id.getAccount().getJid()
658 + ": retract content-add "
659 + ContentAddition.summary(outgoingContentAdd));
660 }
661
662 private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
663 final SessionDescription sdp = setLocalSessionDescription();
664 final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
665 final SessionDescription answer = generateFakeResponse(localRtpContentMap);
666 this.webRTCWrapper
667 .setRemoteDescription(
668 new org.webrtc.SessionDescription(
669 org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
670 .get();
671 return localRtpContentMap;
672 }
673
674 private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
675 final RtpContentMap currentRemote = getRemoteContentMap();
676 final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
677 if (diff.isEmpty()) {
678 throw new IllegalStateException(
679 "Unexpected rollback condition. No difference between local and remote");
680 }
681 final RtpContentMap patch = localContentMap.toContentModification(diff.added);
682 if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
683 final RtpContentMap nextRemote =
684 currentRemote.addContent(
685 patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
686 return SessionDescription.of(nextRemote, !isInitiator());
687 }
688 throw new IllegalStateException(
689 "Unexpected rollback condition. Senders were not uniformly none");
690 }
691
692 public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
693 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
694 if (incomingContentAdd == null) {
695 throw new IllegalStateException("No incoming content add");
696 }
697
698 if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
699 this.incomingContentAdd = null;
700 acceptContentAdd(contentAddition, incomingContentAdd);
701 } else {
702 throw new IllegalStateException("Accepted content add does not match pending content-add");
703 }
704 }
705
706 private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
707 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
708 final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
709 this.setRemoteContentMap(modifiedContentMap);
710
711 final SessionDescription offer;
712 try {
713 offer = SessionDescription.of(modifiedContentMap, !isInitiator());
714 } catch (final IllegalArgumentException | NullPointerException e) {
715 Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
716 webRTCWrapper.close();
717 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
718 return;
719 }
720 this.incomingContentAdd = null;
721 acceptContentAdd(contentAddition, offer);
722 }
723
724 private void acceptContentAdd(
725 final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
726 final org.webrtc.SessionDescription sdp =
727 new org.webrtc.SessionDescription(
728 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
729 try {
730 this.webRTCWrapper.setRemoteDescription(sdp).get();
731
732 // TODO add tracks for 'media' where contentAddition.senders matches
733
734 // TODO if senders.sending(isInitiator())
735
736 this.webRTCWrapper.addTrack(Media.VIDEO);
737
738 // TODO add additional transceivers for recv only cases
739
740 final SessionDescription answer = setLocalSessionDescription();
741 final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
742
743 final RtpContentMap contentAcceptMap =
744 rtpContentMap.toContentModification(
745 Collections2.transform(contentAddition, ca -> ca.name));
746 Log.d(
747 Config.LOGTAG,
748 id.getAccount().getJid().asBareJid()
749 + ": sending content-accept "
750 + ContentAddition.summary(contentAcceptMap));
751 modifyLocalContentMap(rtpContentMap);
752 sendContentAccept(contentAcceptMap);
753 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
754 } catch (final Exception e) {
755 Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
756 webRTCWrapper.close();
757 sendSessionTerminate(Reason.FAILED_APPLICATION);
758 }
759 }
760
761 private void sendContentAccept(final RtpContentMap contentAcceptMap) {
762 final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
763 send(jinglePacket);
764 }
765
766 public synchronized void rejectContentAdd() {
767 final RtpContentMap incomingContentAdd = this.incomingContentAdd;
768 if (incomingContentAdd == null) {
769 throw new IllegalStateException("No incoming content add");
770 }
771 this.incomingContentAdd = null;
772 updateEndUserState();
773 rejectContentAdd(incomingContentAdd);
774 }
775
776 private void rejectContentAdd(final RtpContentMap contentMap) {
777 final JinglePacket jinglePacket =
778 contentMap
779 .toStub()
780 .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
781 Log.d(
782 Config.LOGTAG,
783 id.getAccount().getJid().asBareJid()
784 + ": rejecting content "
785 + ContentAddition.summary(contentMap));
786 send(jinglePacket);
787 }
788
789 private boolean checkForIceRestart(
790 final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
791 final RtpContentMap existing = getRemoteContentMap();
792 final Set<IceUdpTransportInfo.Credentials> existingCredentials;
793 final IceUdpTransportInfo.Credentials newCredentials;
794 try {
795 existingCredentials = existing.getCredentials();
796 newCredentials = rtpContentMap.getDistinctCredentials();
797 } catch (final IllegalStateException e) {
798 Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e);
799 return false;
800 }
801 if (existingCredentials.contains(newCredentials)) {
802 return false;
803 }
804 // TODO an alternative approach is to check if we already got an iq result to our
805 // ICE-restart
806 // and if that's the case we are seeing an answer.
807 // This might be more spec compliant but also more error prone potentially
808 final boolean isOffer = rtpContentMap.emptyCandidates();
809 final RtpContentMap restartContentMap;
810 try {
811 if (isOffer) {
812 Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials);
813 restartContentMap =
814 existing.modifiedCredentials(
815 newCredentials, IceUdpTransportInfo.Setup.ACTPASS);
816 } else {
817 final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
818 Log.d(
819 Config.LOGTAG,
820 "received confirmation of ICE restart"
821 + newCredentials
822 + " peer_setup="
823 + setup);
824 // DTLS setup attribute needs to be rewritten to reflect current peer state
825 // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM
826 restartContentMap = existing.modifiedCredentials(newCredentials, setup);
827 }
828 if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) {
829 return isOffer;
830 } else {
831 Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break");
832 respondWithTieBreak(jinglePacket);
833 return true;
834 }
835 } catch (final Exception exception) {
836 respondOk(jinglePacket);
837 final Throwable rootCause = Throwables.getRootCause(exception);
838 if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) {
839 // If this happens a termination is already in progress
840 Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart");
841 return true;
842 }
843 Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause);
844 webRTCWrapper.close();
845 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
846 return true;
847 }
848 }
849
850 private IceUdpTransportInfo.Setup getPeerDtlsSetup() {
851 final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup;
852 if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) {
853 throw new IllegalStateException("Invalid peer setup");
854 }
855 return peerSetup;
856 }
857
858 private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) {
859 if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) {
860 throw new IllegalArgumentException("Trying to store invalid peer dtls setup");
861 }
862 this.peerDtlsSetup = setup;
863 }
864
865 private boolean applyIceRestart(
866 final JinglePacket jinglePacket,
867 final RtpContentMap restartContentMap,
868 final boolean isOffer)
869 throws ExecutionException, InterruptedException {
870 final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator());
871 final org.webrtc.SessionDescription.Type type =
872 isOffer
873 ? org.webrtc.SessionDescription.Type.OFFER
874 : org.webrtc.SessionDescription.Type.ANSWER;
875 org.webrtc.SessionDescription sdp =
876 new org.webrtc.SessionDescription(type, sessionDescription.toString());
877 if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
878 if (isInitiator()) {
879 // We ignore the offer and respond with tie-break. This will clause the responder
880 // not to apply the content map
881 return false;
882 }
883 }
884 webRTCWrapper.setRemoteDescription(sdp).get();
885 setRemoteContentMap(restartContentMap);
886 if (isOffer) {
887 final SessionDescription localSessionDescription = setLocalSessionDescription();
888 setLocalContentMap(RtpContentMap.of(localSessionDescription, isInitiator()));
889 // We need to respond OK before sending any candidates
890 respondOk(jinglePacket);
891 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
892 } else {
893 storePeerDtlsSetup(restartContentMap.getDtlsSetup());
894 }
895 return true;
896 }
897
898 private void processCandidates(
899 final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
900 for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
901 processCandidate(content);
902 }
903 }
904
905 private void processCandidate(
906 final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
907 final RtpContentMap rtpContentMap = getRemoteContentMap();
908 final List<String> indices = toIdentificationTags(rtpContentMap);
909 final String sdpMid = content.getKey(); // aka content name
910 final IceUdpTransportInfo transport = content.getValue().transport;
911 final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
912
913 // TODO check that credentials remained the same
914
915 for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
916 final String sdp;
917 try {
918 sdp = candidate.toSdpAttribute(credentials.ufrag);
919 } catch (final IllegalArgumentException e) {
920 Log.d(
921 Config.LOGTAG,
922 id.account.getJid().asBareJid()
923 + ": ignoring invalid ICE candidate "
924 + e.getMessage());
925 continue;
926 }
927 final int mLineIndex = indices.indexOf(sdpMid);
928 if (mLineIndex < 0) {
929 Log.w(
930 Config.LOGTAG,
931 "mLineIndex not found for " + sdpMid + ". available indices " + indices);
932 }
933 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
934 Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
935 this.webRTCWrapper.addIceCandidate(iceCandidate);
936 }
937 }
938
939 private RtpContentMap getRemoteContentMap() {
940 return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
941 }
942
943 private RtpContentMap getLocalContentMap() {
944 return isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
945 }
946
947 private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
948 final Group originalGroup = rtpContentMap.group;
949 final List<String> identificationTags =
950 originalGroup == null
951 ? rtpContentMap.getNames()
952 : originalGroup.getIdentificationTags();
953 if (identificationTags.size() == 0) {
954 Log.w(
955 Config.LOGTAG,
956 id.account.getJid().asBareJid()
957 + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
958 }
959 return identificationTags;
960 }
961
962 private ListenableFuture<RtpContentMap> receiveRtpContentMap(
963 final JinglePacket jinglePacket, final boolean expectVerification) {
964 final RtpContentMap receivedContentMap;
965 try {
966 receivedContentMap = RtpContentMap.of(jinglePacket);
967 } catch (final Exception e) {
968 return Futures.immediateFailedFuture(e);
969 }
970 if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) {
971 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<RtpContentMap>> future =
972 id.account
973 .getAxolotlService()
974 .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with);
975 return Futures.transform(
976 future,
977 omemoVerifiedPayload -> {
978 // TODO test if an exception here triggers a correct abort
979 omemoVerification.setOrEnsureEqual(omemoVerifiedPayload);
980 Log.d(
981 Config.LOGTAG,
982 id.account.getJid().asBareJid()
983 + ": received verifiable DTLS fingerprint via "
984 + omemoVerification);
985 return omemoVerifiedPayload.getPayload();
986 },
987 MoreExecutors.directExecutor());
988 } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) {
989 return Futures.immediateFailedFuture(
990 new SecurityException("DTLS fingerprint was unexpectedly not verifiable"));
991 } else {
992 return Futures.immediateFuture(receivedContentMap);
993 }
994 }
995
996 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
997 if (isInitiator()) {
998 Log.d(
999 Config.LOGTAG,
1000 String.format(
1001 "%s: received session-initiate even though we were initiating",
1002 id.account.getJid().asBareJid()));
1003 if (isTerminated()) {
1004 Log.d(
1005 Config.LOGTAG,
1006 String.format(
1007 "%s: got a reason to terminate with out-of-order. but already in state %s",
1008 id.account.getJid().asBareJid(), getState()));
1009 respondWithOutOfOrder(jinglePacket);
1010 } else {
1011 terminateWithOutOfOrder(jinglePacket);
1012 }
1013 return;
1014 }
1015 final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
1016 Futures.addCallback(
1017 future,
1018 new FutureCallback<RtpContentMap>() {
1019 @Override
1020 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1021 receiveSessionInitiate(jinglePacket, rtpContentMap);
1022 }
1023
1024 @Override
1025 public void onFailure(@NonNull final Throwable throwable) {
1026 respondOk(jinglePacket);
1027 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1028 }
1029 },
1030 MoreExecutors.directExecutor());
1031 }
1032
1033 private void receiveSessionInitiate(
1034 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1035 try {
1036 contentMap.requireContentDescriptions();
1037 contentMap.requireDTLSFingerprint(true);
1038 } catch (final RuntimeException e) {
1039 Log.d(
1040 Config.LOGTAG,
1041 id.account.getJid().asBareJid() + ": improperly formatted contents",
1042 Throwables.getRootCause(e));
1043 respondOk(jinglePacket);
1044 sendSessionTerminate(Reason.of(e), e.getMessage());
1045 return;
1046 }
1047 Log.d(
1048 Config.LOGTAG,
1049 "processing session-init with " + contentMap.contents.size() + " contents");
1050 final State target;
1051 if (this.state == State.PROCEED) {
1052 Preconditions.checkState(
1053 proposedMedia != null && proposedMedia.size() > 0,
1054 "proposed media must be set when processing pre-approved session-initiate");
1055 if (!this.proposedMedia.equals(contentMap.getMedia())) {
1056 sendSessionTerminate(
1057 Reason.SECURITY_ERROR,
1058 String.format(
1059 "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s",
1060 this.proposedMedia, contentMap.getMedia()));
1061 return;
1062 }
1063 target = State.SESSION_INITIALIZED_PRE_APPROVED;
1064 } else {
1065 target = State.SESSION_INITIALIZED;
1066 }
1067 if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
1068 respondOk(jinglePacket);
1069 pendingIceCandidates.addAll(contentMap.contents.entrySet());
1070 if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
1071 Log.d(
1072 Config.LOGTAG,
1073 id.account.getJid().asBareJid()
1074 + ": automatically accepting session-initiate");
1075 sendSessionAccept();
1076 } else {
1077 Log.d(
1078 Config.LOGTAG,
1079 id.account.getJid().asBareJid()
1080 + ": received not pre-approved session-initiate. start ringing");
1081 startRinging();
1082 }
1083 } else {
1084 Log.d(
1085 Config.LOGTAG,
1086 String.format(
1087 "%s: received session-initiate while in state %s",
1088 id.account.getJid().asBareJid(), state));
1089 terminateWithOutOfOrder(jinglePacket);
1090 }
1091 }
1092
1093 private void receiveSessionAccept(final JinglePacket jinglePacket) {
1094 if (!isInitiator()) {
1095 Log.d(
1096 Config.LOGTAG,
1097 String.format(
1098 "%s: received session-accept even though we were responding",
1099 id.account.getJid().asBareJid()));
1100 terminateWithOutOfOrder(jinglePacket);
1101 return;
1102 }
1103 final ListenableFuture<RtpContentMap> future =
1104 receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint());
1105 Futures.addCallback(
1106 future,
1107 new FutureCallback<RtpContentMap>() {
1108 @Override
1109 public void onSuccess(@Nullable RtpContentMap rtpContentMap) {
1110 receiveSessionAccept(jinglePacket, rtpContentMap);
1111 }
1112
1113 @Override
1114 public void onFailure(@NonNull final Throwable throwable) {
1115 respondOk(jinglePacket);
1116 Log.d(
1117 Config.LOGTAG,
1118 id.account.getJid().asBareJid()
1119 + ": improperly formatted contents in session-accept",
1120 throwable);
1121 webRTCWrapper.close();
1122 sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage());
1123 }
1124 },
1125 MoreExecutors.directExecutor());
1126 }
1127
1128 private void receiveSessionAccept(
1129 final JinglePacket jinglePacket, final RtpContentMap contentMap) {
1130 try {
1131 contentMap.requireContentDescriptions();
1132 contentMap.requireDTLSFingerprint();
1133 } catch (final RuntimeException e) {
1134 respondOk(jinglePacket);
1135 Log.d(
1136 Config.LOGTAG,
1137 id.account.getJid().asBareJid()
1138 + ": improperly formatted contents in session-accept",
1139 e);
1140 webRTCWrapper.close();
1141 sendSessionTerminate(Reason.of(e), e.getMessage());
1142 return;
1143 }
1144 final Set<Media> initiatorMedia = this.initiatorRtpContentMap.getMedia();
1145 if (!initiatorMedia.equals(contentMap.getMedia())) {
1146 sendSessionTerminate(
1147 Reason.SECURITY_ERROR,
1148 String.format(
1149 "Your session-included included media %s but our session-initiate was %s",
1150 this.proposedMedia, contentMap.getMedia()));
1151 return;
1152 }
1153 Log.d(
1154 Config.LOGTAG,
1155 "processing session-accept with " + contentMap.contents.size() + " contents");
1156 if (transition(State.SESSION_ACCEPTED)) {
1157 respondOk(jinglePacket);
1158 receiveSessionAccept(contentMap);
1159 } else {
1160 Log.d(
1161 Config.LOGTAG,
1162 String.format(
1163 "%s: received session-accept while in state %s",
1164 id.account.getJid().asBareJid(), state));
1165 respondOk(jinglePacket);
1166 }
1167 }
1168
1169 private void receiveSessionAccept(final RtpContentMap contentMap) {
1170 this.responderRtpContentMap = contentMap;
1171 this.storePeerDtlsSetup(contentMap.getDtlsSetup());
1172 final SessionDescription sessionDescription;
1173 try {
1174 sessionDescription = SessionDescription.of(contentMap, false);
1175 } catch (final IllegalArgumentException | NullPointerException e) {
1176 Log.d(
1177 Config.LOGTAG,
1178 id.account.getJid().asBareJid()
1179 + ": unable convert offer from session-accept to SDP",
1180 e);
1181 webRTCWrapper.close();
1182 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1183 return;
1184 }
1185 final org.webrtc.SessionDescription answer =
1186 new org.webrtc.SessionDescription(
1187 org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString());
1188 try {
1189 this.webRTCWrapper.setRemoteDescription(answer).get();
1190 } catch (final Exception e) {
1191 Log.d(
1192 Config.LOGTAG,
1193 id.account.getJid().asBareJid()
1194 + ": unable to set remote description after receiving session-accept",
1195 Throwables.getRootCause(e));
1196 webRTCWrapper.close();
1197 sendSessionTerminate(
1198 Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage());
1199 return;
1200 }
1201 processCandidates(contentMap.contents.entrySet());
1202 }
1203
1204 private void sendSessionAccept() {
1205 final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
1206 if (rtpContentMap == null) {
1207 throw new IllegalStateException("initiator RTP Content Map has not been set");
1208 }
1209 final SessionDescription offer;
1210 try {
1211 offer = SessionDescription.of(rtpContentMap, true);
1212 } catch (final IllegalArgumentException | NullPointerException e) {
1213 Log.d(
1214 Config.LOGTAG,
1215 id.account.getJid().asBareJid()
1216 + ": unable convert offer from session-initiate to SDP",
1217 e);
1218 webRTCWrapper.close();
1219 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1220 return;
1221 }
1222 sendSessionAccept(rtpContentMap.getMedia(), offer);
1223 }
1224
1225 private void sendSessionAccept(final Set<Media> media, final SessionDescription offer) {
1226 discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers));
1227 }
1228
1229 private synchronized void sendSessionAccept(
1230 final Set<Media> media,
1231 final SessionDescription offer,
1232 final List<PeerConnection.IceServer> iceServers) {
1233 if (isTerminated()) {
1234 Log.w(
1235 Config.LOGTAG,
1236 id.account.getJid().asBareJid()
1237 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1238 return;
1239 }
1240 try {
1241 setupWebRTC(media, iceServers);
1242 } catch (final WebRTCWrapper.InitializationException e) {
1243 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1244 webRTCWrapper.close();
1245 sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
1246 return;
1247 }
1248 final org.webrtc.SessionDescription sdp =
1249 new org.webrtc.SessionDescription(
1250 org.webrtc.SessionDescription.Type.OFFER, offer.toString());
1251 try {
1252 this.webRTCWrapper.setRemoteDescription(sdp).get();
1253 addIceCandidatesFromBlackLog();
1254 org.webrtc.SessionDescription webRTCSessionDescription =
1255 this.webRTCWrapper.setLocalDescription().get();
1256 prepareSessionAccept(webRTCSessionDescription);
1257 } catch (final Exception e) {
1258 failureToAcceptSession(e);
1259 }
1260 }
1261
1262 private void failureToAcceptSession(final Throwable throwable) {
1263 if (isTerminated()) {
1264 return;
1265 }
1266 final Throwable rootCause = Throwables.getRootCause(throwable);
1267 Log.d(Config.LOGTAG, "unable to send session accept", rootCause);
1268 webRTCWrapper.close();
1269 sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage());
1270 }
1271
1272 private void addIceCandidatesFromBlackLog() {
1273 Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
1274 while ((foo = this.pendingIceCandidates.poll()) != null) {
1275 processCandidate(foo);
1276 Log.d(
1277 Config.LOGTAG,
1278 id.account.getJid().asBareJid() + ": added candidate from back log");
1279 }
1280 }
1281
1282 private void prepareSessionAccept(
1283 final org.webrtc.SessionDescription webRTCSessionDescription) {
1284 final SessionDescription sessionDescription =
1285 SessionDescription.parse(webRTCSessionDescription.description);
1286 final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
1287 this.responderRtpContentMap = respondingRtpContentMap;
1288 storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
1289 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1290 prepareOutgoingContentMap(respondingRtpContentMap);
1291 Futures.addCallback(
1292 outgoingContentMapFuture,
1293 new FutureCallback<RtpContentMap>() {
1294 @Override
1295 public void onSuccess(final RtpContentMap outgoingContentMap) {
1296 sendSessionAccept(outgoingContentMap);
1297 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1298 }
1299
1300 @Override
1301 public void onFailure(@NonNull Throwable throwable) {
1302 failureToAcceptSession(throwable);
1303 }
1304 },
1305 MoreExecutors.directExecutor());
1306 }
1307
1308 private void sendSessionAccept(final RtpContentMap rtpContentMap) {
1309 if (isTerminated()) {
1310 Log.w(
1311 Config.LOGTAG,
1312 id.account.getJid().asBareJid()
1313 + ": preparing session accept was too slow. already terminated. nothing to do.");
1314 return;
1315 }
1316 transitionOrThrow(State.SESSION_ACCEPTED);
1317 final JinglePacket sessionAccept =
1318 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
1319 send(sessionAccept);
1320 }
1321
1322 private ListenableFuture<RtpContentMap> prepareOutgoingContentMap(
1323 final RtpContentMap rtpContentMap) {
1324 if (this.omemoVerification.hasDeviceId()) {
1325 ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1326 verifiedPayloadFuture =
1327 id.account
1328 .getAxolotlService()
1329 .encrypt(
1330 rtpContentMap,
1331 id.with,
1332 omemoVerification.getDeviceId());
1333 return Futures.transform(
1334 verifiedPayloadFuture,
1335 verifiedPayload -> {
1336 omemoVerification.setOrEnsureEqual(verifiedPayload);
1337 return verifiedPayload.getPayload();
1338 },
1339 MoreExecutors.directExecutor());
1340 } else {
1341 return Futures.immediateFuture(rtpContentMap);
1342 }
1343 }
1344
1345 synchronized void deliveryMessage(
1346 final Jid from,
1347 final Element message,
1348 final String serverMessageId,
1349 final long timestamp) {
1350 Log.d(
1351 Config.LOGTAG,
1352 id.account.getJid().asBareJid()
1353 + ": delivered message to JingleRtpConnection "
1354 + message);
1355 switch (message.getName()) {
1356 case "propose":
1357 receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp);
1358 break;
1359 case "proceed":
1360 receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp);
1361 break;
1362 case "retract":
1363 receiveRetract(from, serverMessageId, timestamp);
1364 break;
1365 case "reject":
1366 receiveReject(from, serverMessageId, timestamp);
1367 break;
1368 case "accept":
1369 receiveAccept(from, serverMessageId, timestamp);
1370 break;
1371 default:
1372 break;
1373 }
1374 }
1375
1376 void deliverFailedProceed(final String message) {
1377 Log.d(
1378 Config.LOGTAG,
1379 id.account.getJid().asBareJid() + ": receive message error for proceed message ("+Strings.nullToEmpty(message)+")");
1380 if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) {
1381 webRTCWrapper.close();
1382 Log.d(
1383 Config.LOGTAG,
1384 id.account.getJid().asBareJid() + ": transitioned into connectivity error");
1385 this.finish();
1386 }
1387 }
1388
1389 private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) {
1390 final boolean originatedFromMyself =
1391 from.asBareJid().equals(id.account.getJid().asBareJid());
1392 if (originatedFromMyself) {
1393 if (transition(State.ACCEPTED)) {
1394 acceptedOnOtherDevice(serverMsgId, timestamp);
1395 } else {
1396 Log.d(
1397 Config.LOGTAG,
1398 id.account.getJid().asBareJid()
1399 + ": unable to transition to accept because already in state="
1400 + this.state);
1401 Log.d(Config.LOGTAG, id.account.getJid() + ": received accept from " + from);
1402 }
1403 } else {
1404 Log.d(
1405 Config.LOGTAG,
1406 id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from);
1407 }
1408 }
1409
1410 private void acceptedOnOtherDevice(final String serverMsgId, final long timestamp) {
1411 if (serverMsgId != null) {
1412 this.message.setServerMsgId(serverMsgId);
1413 }
1414 this.message.setTime(timestamp);
1415 this.message.setCarbon(true); // indicate that call was accepted on other device
1416 this.writeLogMessageSuccess(0);
1417 this.xmppConnectionService
1418 .getNotificationService()
1419 .cancelIncomingCallNotification();
1420 this.finish();
1421 }
1422
1423 private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) {
1424 final boolean originatedFromMyself =
1425 from.asBareJid().equals(id.account.getJid().asBareJid());
1426 // reject from another one of my clients
1427 if (originatedFromMyself) {
1428 receiveRejectFromMyself(serverMsgId, timestamp);
1429 } else if (isInitiator()) {
1430 if (from.equals(id.with)) {
1431 receiveRejectFromResponder();
1432 } else {
1433 Log.d(
1434 Config.LOGTAG,
1435 id.account.getJid()
1436 + ": ignoring reject from "
1437 + from
1438 + " for session with "
1439 + id.with);
1440 }
1441 } else {
1442 Log.d(
1443 Config.LOGTAG,
1444 id.account.getJid()
1445 + ": ignoring reject from "
1446 + from
1447 + " for session with "
1448 + id.with);
1449 }
1450 }
1451
1452 private void receiveRejectFromMyself(String serverMsgId, long timestamp) {
1453 if (transition(State.REJECTED)) {
1454 this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1455 this.finish();
1456 if (serverMsgId != null) {
1457 this.message.setServerMsgId(serverMsgId);
1458 }
1459 this.message.setTime(timestamp);
1460 this.message.setCarbon(true); // indicate that call was rejected on other device
1461 writeLogMessageMissed();
1462 } else {
1463 Log.d(
1464 Config.LOGTAG,
1465 "not able to transition into REJECTED because already in " + this.state);
1466 }
1467 }
1468
1469 private void receiveRejectFromResponder() {
1470 if (isInState(State.PROCEED)) {
1471 Log.d(
1472 Config.LOGTAG,
1473 id.account.getJid()
1474 + ": received reject while still in proceed. callee reconsidered");
1475 closeTransitionLogFinish(State.REJECTED_RACED);
1476 return;
1477 }
1478 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) {
1479 Log.d(
1480 Config.LOGTAG,
1481 id.account.getJid()
1482 + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init");
1483 closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY);
1484 return;
1485 }
1486 Log.d(
1487 Config.LOGTAG,
1488 id.account.getJid()
1489 + ": ignoring reject from responder because already in state "
1490 + this.state);
1491 }
1492
1493 private void receivePropose(
1494 final Jid from, final Propose propose, final String serverMsgId, final long timestamp) {
1495 final boolean originatedFromMyself =
1496 from.asBareJid().equals(id.account.getJid().asBareJid());
1497 if (originatedFromMyself) {
1498 Log.d(
1499 Config.LOGTAG,
1500 id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring");
1501 } else if (transition(
1502 State.PROPOSED,
1503 () -> {
1504 final Collection<RtpDescription> descriptions =
1505 Collections2.transform(
1506 Collections2.filter(
1507 propose.getDescriptions(),
1508 d -> d instanceof RtpDescription),
1509 input -> (RtpDescription) input);
1510 final Collection<Media> media =
1511 Collections2.transform(descriptions, RtpDescription::getMedia);
1512 Preconditions.checkState(
1513 !media.contains(Media.UNKNOWN),
1514 "RTP descriptions contain unknown media");
1515 Log.d(
1516 Config.LOGTAG,
1517 id.account.getJid().asBareJid()
1518 + ": received session proposal from "
1519 + from
1520 + " for "
1521 + media);
1522 this.proposedMedia = Sets.newHashSet(media);
1523 })) {
1524 if (serverMsgId != null) {
1525 this.message.setServerMsgId(serverMsgId);
1526 }
1527 this.message.setTime(timestamp);
1528 startRinging();
1529 } else {
1530 Log.d(
1531 Config.LOGTAG,
1532 id.account.getJid()
1533 + ": ignoring session proposal because already in "
1534 + state);
1535 }
1536 }
1537
1538 private void startRinging() {
1539 Log.d(
1540 Config.LOGTAG,
1541 id.account.getJid().asBareJid()
1542 + ": received call from "
1543 + id.with
1544 + ". start ringing");
1545 ringingTimeoutFuture =
1546 jingleConnectionManager.schedule(
1547 this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS);
1548 xmppConnectionService.getNotificationService().startRinging(id, getMedia());
1549 }
1550
1551 private synchronized void ringingTimeout() {
1552 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing");
1553 switch (this.state) {
1554 case PROPOSED:
1555 message.markUnread();
1556 rejectCallFromProposed();
1557 break;
1558 case SESSION_INITIALIZED:
1559 message.markUnread();
1560 rejectCallFromSessionInitiate();
1561 break;
1562 }
1563 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1564 }
1565
1566 private void cancelRingingTimeout() {
1567 final ScheduledFuture<?> future = this.ringingTimeoutFuture;
1568 if (future != null && !future.isCancelled()) {
1569 future.cancel(false);
1570 }
1571 }
1572
1573 private void receiveProceed(
1574 final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) {
1575 final Set<Media> media =
1576 Preconditions.checkNotNull(
1577 this.proposedMedia, "Proposed media has to be set before handling proceed");
1578 Preconditions.checkState(media.size() > 0, "Proposed media should not be empty");
1579 if (from.equals(id.with)) {
1580 if (isInitiator()) {
1581 if (transition(State.PROCEED)) {
1582 if (serverMsgId != null) {
1583 this.message.setServerMsgId(serverMsgId);
1584 }
1585 this.message.setTime(timestamp);
1586 final Integer remoteDeviceId = proceed.getDeviceId();
1587 if (isOmemoEnabled()) {
1588 this.omemoVerification.setDeviceId(remoteDeviceId);
1589 } else {
1590 if (remoteDeviceId != null) {
1591 Log.d(
1592 Config.LOGTAG,
1593 id.account.getJid().asBareJid()
1594 + ": remote party signaled support for OMEMO verification but we have OMEMO disabled");
1595 }
1596 this.omemoVerification.setDeviceId(null);
1597 }
1598 this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED);
1599 } else {
1600 Log.d(
1601 Config.LOGTAG,
1602 String.format(
1603 "%s: ignoring proceed because already in %s",
1604 id.account.getJid().asBareJid(), this.state));
1605 }
1606 } else {
1607 Log.d(
1608 Config.LOGTAG,
1609 String.format(
1610 "%s: ignoring proceed because we were not initializing",
1611 id.account.getJid().asBareJid()));
1612 }
1613 } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) {
1614 if (transition(State.ACCEPTED)) {
1615 Log.d(
1616 Config.LOGTAG,
1617 id.account.getJid().asBareJid()
1618 + ": moved session with "
1619 + id.with
1620 + " into state accepted after received carbon copied proceed");
1621 acceptedOnOtherDevice(serverMsgId, timestamp);
1622 }
1623 } else {
1624 Log.d(
1625 Config.LOGTAG,
1626 String.format(
1627 "%s: ignoring proceed from %s. was expected from %s",
1628 id.account.getJid().asBareJid(), from, id.with));
1629 }
1630 }
1631
1632 private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) {
1633 if (from.equals(id.with)) {
1634 final State target =
1635 this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED;
1636 if (transition(target)) {
1637 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
1638 xmppConnectionService.getNotificationService().pushMissedCallNow(message);
1639 Log.d(
1640 Config.LOGTAG,
1641 id.account.getJid().asBareJid()
1642 + ": session with "
1643 + id.with
1644 + " has been retracted (serverMsgId="
1645 + serverMsgId
1646 + ")");
1647 if (serverMsgId != null) {
1648 this.message.setServerMsgId(serverMsgId);
1649 }
1650 this.message.setTime(timestamp);
1651 if (target == State.RETRACTED) {
1652 this.message.markUnread();
1653 }
1654 writeLogMessageMissed();
1655 finish();
1656 } else {
1657 Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state);
1658 }
1659 } else {
1660 // TODO parse retract from self
1661 Log.d(
1662 Config.LOGTAG,
1663 id.account.getJid().asBareJid()
1664 + ": received retract from "
1665 + from
1666 + ". expected retract from"
1667 + id.with
1668 + ". ignoring");
1669 }
1670 }
1671
1672 public void sendSessionInitiate() {
1673 sendSessionInitiate(this.proposedMedia, State.SESSION_INITIALIZED);
1674 }
1675
1676 private void sendSessionInitiate(final Set<Media> media, final State targetState) {
1677 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
1678 discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers));
1679 }
1680
1681 private synchronized void sendSessionInitiate(
1682 final Set<Media> media,
1683 final State targetState,
1684 final List<PeerConnection.IceServer> iceServers) {
1685 if (isTerminated()) {
1686 Log.w(
1687 Config.LOGTAG,
1688 id.account.getJid().asBareJid()
1689 + ": ICE servers got discovered when session was already terminated. nothing to do.");
1690 return;
1691 }
1692 try {
1693 setupWebRTC(media, iceServers);
1694 } catch (final WebRTCWrapper.InitializationException e) {
1695 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
1696 webRTCWrapper.close();
1697 sendRetract(Reason.ofThrowable(e));
1698 return;
1699 }
1700 try {
1701 org.webrtc.SessionDescription webRTCSessionDescription =
1702 this.webRTCWrapper.setLocalDescription().get();
1703 prepareSessionInitiate(webRTCSessionDescription, targetState);
1704 } catch (final Exception e) {
1705 // TODO sending the error text is worthwhile as well. Especially for FailureToSet
1706 // exceptions
1707 failureToInitiateSession(e, targetState);
1708 }
1709 }
1710
1711 private void failureToInitiateSession(final Throwable throwable, final State targetState) {
1712 if (isTerminated()) {
1713 return;
1714 }
1715 Log.d(
1716 Config.LOGTAG,
1717 id.account.getJid().asBareJid() + ": unable to sendSessionInitiate",
1718 Throwables.getRootCause(throwable));
1719 webRTCWrapper.close();
1720 final Reason reason = Reason.ofThrowable(throwable);
1721 if (isInState(targetState)) {
1722 sendSessionTerminate(reason, throwable.getMessage());
1723 } else {
1724 sendRetract(reason);
1725 }
1726 }
1727
1728 private void sendRetract(final Reason reason) {
1729 // TODO embed reason into retract
1730 sendJingleMessage("retract", id.with.asBareJid());
1731 transitionOrThrow(reasonToState(reason));
1732 this.finish();
1733 }
1734
1735 private void prepareSessionInitiate(
1736 final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
1737 final SessionDescription sessionDescription =
1738 SessionDescription.parse(webRTCSessionDescription.description);
1739 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
1740 this.initiatorRtpContentMap = rtpContentMap;
1741 final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
1742 encryptSessionInitiate(rtpContentMap);
1743 Futures.addCallback(
1744 outgoingContentMapFuture,
1745 new FutureCallback<RtpContentMap>() {
1746 @Override
1747 public void onSuccess(final RtpContentMap outgoingContentMap) {
1748 sendSessionInitiate(outgoingContentMap, targetState);
1749 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
1750 }
1751
1752 @Override
1753 public void onFailure(@NonNull final Throwable throwable) {
1754 failureToInitiateSession(throwable, targetState);
1755 }
1756 },
1757 MoreExecutors.directExecutor());
1758 }
1759
1760 private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {
1761 if (isTerminated()) {
1762 Log.w(
1763 Config.LOGTAG,
1764 id.account.getJid().asBareJid()
1765 + ": preparing session was too slow. already terminated. nothing to do.");
1766 return;
1767 }
1768 this.transitionOrThrow(targetState);
1769 final JinglePacket sessionInitiate =
1770 rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
1771 send(sessionInitiate);
1772 }
1773
1774 private ListenableFuture<RtpContentMap> encryptSessionInitiate(
1775 final RtpContentMap rtpContentMap) {
1776 if (this.omemoVerification.hasDeviceId()) {
1777 final ListenableFuture<AxolotlService.OmemoVerifiedPayload<OmemoVerifiedRtpContentMap>>
1778 verifiedPayloadFuture =
1779 id.account
1780 .getAxolotlService()
1781 .encrypt(
1782 rtpContentMap,
1783 id.with,
1784 omemoVerification.getDeviceId());
1785 final ListenableFuture<RtpContentMap> future =
1786 Futures.transform(
1787 verifiedPayloadFuture,
1788 verifiedPayload -> {
1789 omemoVerification.setSessionFingerprint(
1790 verifiedPayload.getFingerprint());
1791 return verifiedPayload.getPayload();
1792 },
1793 MoreExecutors.directExecutor());
1794 if (Config.REQUIRE_RTP_VERIFICATION) {
1795 return future;
1796 }
1797 return Futures.catching(
1798 future,
1799 CryptoFailedException.class,
1800 e -> {
1801 Log.w(
1802 Config.LOGTAG,
1803 id.account.getJid().asBareJid()
1804 + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back",
1805 e);
1806 return rtpContentMap;
1807 },
1808 MoreExecutors.directExecutor());
1809 } else {
1810 return Futures.immediateFuture(rtpContentMap);
1811 }
1812 }
1813
1814 private void sendSessionTerminate(final Reason reason) {
1815 sendSessionTerminate(reason, null);
1816 }
1817
1818 private void sendSessionTerminate(final Reason reason, final String text) {
1819 final State previous = this.state;
1820 final State target = reasonToState(reason);
1821 transitionOrThrow(target);
1822 if (previous != State.NULL) {
1823 writeLogMessage(target);
1824 }
1825 final JinglePacket jinglePacket =
1826 new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId);
1827 jinglePacket.setReason(reason, text);
1828 Log.d(Config.LOGTAG, jinglePacket.toString());
1829 send(jinglePacket);
1830 finish();
1831 }
1832
1833 private void sendTransportInfo(
1834 final String contentName, IceUdpTransportInfo.Candidate candidate) {
1835 final RtpContentMap transportInfo;
1836 try {
1837 final RtpContentMap rtpContentMap =
1838 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
1839 transportInfo = rtpContentMap.transportInfo(contentName, candidate);
1840 } catch (final Exception e) {
1841 Log.d(
1842 Config.LOGTAG,
1843 id.account.getJid().asBareJid()
1844 + ": unable to prepare transport-info from candidate for content="
1845 + contentName);
1846 return;
1847 }
1848 final JinglePacket jinglePacket =
1849 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
1850 send(jinglePacket);
1851 }
1852
1853 private void send(final JinglePacket jinglePacket) {
1854 jinglePacket.setTo(id.with);
1855 xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
1856 }
1857
1858 private synchronized void handleIqResponse(final Account account, final IqPacket response) {
1859 if (response.getType() == IqPacket.TYPE.ERROR) {
1860 handleIqErrorResponse(response);
1861 return;
1862 }
1863 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
1864 handleIqTimeoutResponse(response);
1865 }
1866 }
1867
1868 private void handleIqErrorResponse(final IqPacket response) {
1869 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR);
1870 final String errorCondition = response.getErrorCondition();
1871 Log.d(
1872 Config.LOGTAG,
1873 id.account.getJid().asBareJid()
1874 + ": received IQ-error from "
1875 + response.getFrom()
1876 + " in RTP session. "
1877 + errorCondition);
1878 if (isTerminated()) {
1879 Log.i(
1880 Config.LOGTAG,
1881 id.account.getJid().asBareJid()
1882 + ": ignoring error because session was already terminated");
1883 return;
1884 }
1885 this.webRTCWrapper.close();
1886 final State target;
1887 if (Arrays.asList(
1888 "service-unavailable",
1889 "recipient-unavailable",
1890 "remote-server-not-found",
1891 "remote-server-timeout")
1892 .contains(errorCondition)) {
1893 target = State.TERMINATED_CONNECTIVITY_ERROR;
1894 } else {
1895 target = State.TERMINATED_APPLICATION_FAILURE;
1896 }
1897 transitionOrThrow(target);
1898 this.finish();
1899 }
1900
1901 private void handleIqTimeoutResponse(final IqPacket response) {
1902 Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT);
1903 Log.d(
1904 Config.LOGTAG,
1905 id.account.getJid().asBareJid()
1906 + ": received IQ timeout in RTP session with "
1907 + id.with
1908 + ". terminating with connectivity error");
1909 if (isTerminated()) {
1910 Log.i(
1911 Config.LOGTAG,
1912 id.account.getJid().asBareJid()
1913 + ": ignoring error because session was already terminated");
1914 return;
1915 }
1916 this.webRTCWrapper.close();
1917 transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR);
1918 this.finish();
1919 }
1920
1921 private void terminateWithOutOfOrder(final JinglePacket jinglePacket) {
1922 Log.d(
1923 Config.LOGTAG,
1924 id.account.getJid().asBareJid() + ": terminating session with out-of-order");
1925 this.webRTCWrapper.close();
1926 transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE);
1927 respondWithOutOfOrder(jinglePacket);
1928 this.finish();
1929 }
1930
1931 private void respondWithTieBreak(final JinglePacket jinglePacket) {
1932 respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel");
1933 }
1934
1935 private void respondWithOutOfOrder(final JinglePacket jinglePacket) {
1936 respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
1937 }
1938
1939 void respondWithJingleError(
1940 final IqPacket original,
1941 String jingleCondition,
1942 String condition,
1943 String conditionType) {
1944 jingleConnectionManager.respondWithJingleError(
1945 id.account, original, jingleCondition, condition, conditionType);
1946 }
1947
1948 private void respondOk(final JinglePacket jinglePacket) {
1949 xmppConnectionService.sendIqPacket(
1950 id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null);
1951 }
1952
1953 public RtpEndUserState getEndUserState() {
1954 switch (this.state) {
1955 case NULL:
1956 case PROPOSED:
1957 case SESSION_INITIALIZED:
1958 if (isInitiator()) {
1959 return RtpEndUserState.RINGING;
1960 } else {
1961 return RtpEndUserState.INCOMING_CALL;
1962 }
1963 case PROCEED:
1964 if (isInitiator()) {
1965 return RtpEndUserState.RINGING;
1966 } else {
1967 return RtpEndUserState.ACCEPTING_CALL;
1968 }
1969 case SESSION_INITIALIZED_PRE_APPROVED:
1970 if (isInitiator()) {
1971 return RtpEndUserState.RINGING;
1972 } else {
1973 return RtpEndUserState.CONNECTING;
1974 }
1975 case SESSION_ACCEPTED:
1976 final ContentAddition ca = getPendingContentAddition();
1977 if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
1978 return RtpEndUserState.INCOMING_CONTENT_ADD;
1979 }
1980 return getPeerConnectionStateAsEndUserState();
1981 case REJECTED:
1982 case REJECTED_RACED:
1983 case TERMINATED_DECLINED_OR_BUSY:
1984 if (isInitiator()) {
1985 return RtpEndUserState.DECLINED_OR_BUSY;
1986 } else {
1987 return RtpEndUserState.ENDED;
1988 }
1989 case TERMINATED_SUCCESS:
1990 case ACCEPTED:
1991 case RETRACTED:
1992 case TERMINATED_CANCEL_OR_TIMEOUT:
1993 return RtpEndUserState.ENDED;
1994 case RETRACTED_RACED:
1995 if (isInitiator()) {
1996 return RtpEndUserState.ENDED;
1997 } else {
1998 return RtpEndUserState.RETRACTED;
1999 }
2000 case TERMINATED_CONNECTIVITY_ERROR:
2001 return zeroDuration()
2002 ? RtpEndUserState.CONNECTIVITY_ERROR
2003 : RtpEndUserState.CONNECTIVITY_LOST_ERROR;
2004 case TERMINATED_APPLICATION_FAILURE:
2005 return RtpEndUserState.APPLICATION_ERROR;
2006 case TERMINATED_SECURITY_ERROR:
2007 return RtpEndUserState.SECURITY_ERROR;
2008 }
2009 throw new IllegalStateException(
2010 String.format("%s has no equivalent EndUserState", this.state));
2011 }
2012
2013 private RtpEndUserState getPeerConnectionStateAsEndUserState() {
2014 final PeerConnection.PeerConnectionState state;
2015 try {
2016 state = webRTCWrapper.getState();
2017 } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
2018 // We usually close the WebRTCWrapper *before* transitioning so we might still
2019 // be in SESSION_ACCEPTED even though the peerConnection has been torn down
2020 return RtpEndUserState.ENDING_CALL;
2021 }
2022 switch (state) {
2023 case CONNECTED:
2024 return RtpEndUserState.CONNECTED;
2025 case NEW:
2026 case CONNECTING:
2027 return RtpEndUserState.CONNECTING;
2028 case CLOSED:
2029 return RtpEndUserState.ENDING_CALL;
2030 default:
2031 return zeroDuration()
2032 ? RtpEndUserState.CONNECTIVITY_ERROR
2033 : RtpEndUserState.RECONNECTING;
2034 }
2035 }
2036
2037 public ContentAddition getPendingContentAddition() {
2038 final RtpContentMap in = this.incomingContentAdd;
2039 final RtpContentMap out = this.outgoingContentAdd;
2040 if (out != null) {
2041 return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
2042 } else if (in != null) {
2043 return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
2044 } else {
2045 return null;
2046 }
2047 }
2048
2049 public Set<Media> getMedia() {
2050 final State current = getState();
2051 if (current == State.NULL) {
2052 if (isInitiator()) {
2053 return Preconditions.checkNotNull(
2054 this.proposedMedia, "RTP connection has not been initialized properly");
2055 }
2056 throw new IllegalStateException("RTP connection has not been initialized yet");
2057 }
2058 if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) {
2059 return Preconditions.checkNotNull(
2060 this.proposedMedia, "RTP connection has not been initialized properly");
2061 }
2062 final RtpContentMap localContentMap = getLocalContentMap();
2063 final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
2064 if (localContentMap != null) {
2065 return localContentMap.getMedia();
2066 } else if (initiatorContentMap != null) {
2067 return initiatorContentMap.getMedia();
2068 } else if (isTerminated()) {
2069 return Collections.emptySet(); //we might fail before we ever got a chance to set media
2070 } else {
2071 return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
2072 }
2073 }
2074
2075 public boolean isVerified() {
2076 final String fingerprint = this.omemoVerification.getFingerprint();
2077 if (fingerprint == null) {
2078 return false;
2079 }
2080 final FingerprintStatus status =
2081 id.account.getAxolotlService().getFingerprintTrust(fingerprint);
2082 return status != null && status.isVerified();
2083 }
2084
2085 public boolean addMedia(final Media media) {
2086 final Set<Media> currentMedia = getMedia();
2087 if (currentMedia.contains(media)) {
2088 throw new IllegalStateException(String.format("%s has already been proposed", media));
2089 }
2090 // TODO add state protection - can only add while ACCEPTED or so
2091 Log.d(Config.LOGTAG,"adding media: "+media);
2092 return webRTCWrapper.addTrack(media);
2093 }
2094
2095 public synchronized void acceptCall() {
2096 switch (this.state) {
2097 case PROPOSED:
2098 cancelRingingTimeout();
2099 acceptCallFromProposed();
2100 break;
2101 case SESSION_INITIALIZED:
2102 cancelRingingTimeout();
2103 acceptCallFromSessionInitialized();
2104 break;
2105 case ACCEPTED:
2106 Log.w(
2107 Config.LOGTAG,
2108 id.account.getJid().asBareJid()
2109 + ": the call has already been accepted with another client. UI was just lagging behind");
2110 break;
2111 case PROCEED:
2112 case SESSION_ACCEPTED:
2113 Log.w(
2114 Config.LOGTAG,
2115 id.account.getJid().asBareJid()
2116 + ": the call has already been accepted. user probably double tapped the UI");
2117 break;
2118 default:
2119 throw new IllegalStateException("Can not accept call from " + this.state);
2120 }
2121 }
2122
2123 public void notifyPhoneCall() {
2124 Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections");
2125 if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) {
2126 rejectCall();
2127 } else {
2128 endCall();
2129 }
2130 }
2131
2132 public synchronized void rejectCall() {
2133 if (isTerminated()) {
2134 Log.w(
2135 Config.LOGTAG,
2136 id.account.getJid().asBareJid()
2137 + ": received rejectCall() when session has already been terminated. nothing to do");
2138 return;
2139 }
2140 switch (this.state) {
2141 case PROPOSED:
2142 rejectCallFromProposed();
2143 break;
2144 case SESSION_INITIALIZED:
2145 rejectCallFromSessionInitiate();
2146 break;
2147 default:
2148 throw new IllegalStateException("Can not reject call from " + this.state);
2149 }
2150 }
2151
2152 public synchronized void endCall() {
2153 if (isTerminated()) {
2154 Log.w(
2155 Config.LOGTAG,
2156 id.account.getJid().asBareJid()
2157 + ": received endCall() when session has already been terminated. nothing to do");
2158 return;
2159 }
2160 if (isInState(State.PROPOSED) && !isInitiator()) {
2161 rejectCallFromProposed();
2162 return;
2163 }
2164 if (isInState(State.PROCEED)) {
2165 if (isInitiator()) {
2166 retractFromProceed();
2167 } else {
2168 rejectCallFromProceed();
2169 }
2170 return;
2171 }
2172 if (isInitiator()
2173 && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) {
2174 this.webRTCWrapper.close();
2175 sendSessionTerminate(Reason.CANCEL);
2176 return;
2177 }
2178 if (isInState(State.SESSION_INITIALIZED)) {
2179 rejectCallFromSessionInitiate();
2180 return;
2181 }
2182 if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
2183 this.webRTCWrapper.close();
2184 sendSessionTerminate(Reason.SUCCESS);
2185 return;
2186 }
2187 if (isInState(
2188 State.TERMINATED_APPLICATION_FAILURE,
2189 State.TERMINATED_CONNECTIVITY_ERROR,
2190 State.TERMINATED_DECLINED_OR_BUSY)) {
2191 Log.d(
2192 Config.LOGTAG,
2193 "ignoring request to end call because already in state " + this.state);
2194 return;
2195 }
2196 throw new IllegalStateException(
2197 "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator());
2198 }
2199
2200 private void retractFromProceed() {
2201 Log.d(Config.LOGTAG, "retract from proceed");
2202 this.sendJingleMessage("retract");
2203 closeTransitionLogFinish(State.RETRACTED_RACED);
2204 }
2205
2206 private void closeTransitionLogFinish(final State state) {
2207 this.webRTCWrapper.close();
2208 transitionOrThrow(state);
2209 writeLogMessage(state);
2210 finish();
2211 }
2212
2213 private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
2214 this.jingleConnectionManager.ensureConnectionIsRegistered(this);
2215 this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
2216 this.webRTCWrapper.initializePeerConnection(media, iceServers);
2217 }
2218
2219 private void acceptCallFromProposed() {
2220 transitionOrThrow(State.PROCEED);
2221 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2222 this.sendJingleMessage("accept", id.account.getJid().asBareJid());
2223 this.sendJingleMessage("proceed");
2224 }
2225
2226 private void rejectCallFromProposed() {
2227 transitionOrThrow(State.REJECTED);
2228 writeLogMessageMissed();
2229 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2230 this.sendJingleMessage("reject");
2231 finish();
2232 }
2233
2234 private void rejectCallFromProceed() {
2235 this.sendJingleMessage("reject");
2236 closeTransitionLogFinish(State.REJECTED_RACED);
2237 }
2238
2239 private void rejectCallFromSessionInitiate() {
2240 webRTCWrapper.close();
2241 sendSessionTerminate(Reason.DECLINE);
2242 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2243 }
2244
2245 private void sendJingleMessage(final String action) {
2246 sendJingleMessage(action, id.with);
2247 }
2248
2249 private void sendJingleMessage(final String action, final Jid to) {
2250 final MessagePacket messagePacket = new MessagePacket();
2251 messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those
2252 messagePacket.setTo(to);
2253 final Element intent =
2254 messagePacket
2255 .addChild(action, Namespace.JINGLE_MESSAGE)
2256 .setAttribute("id", id.sessionId);
2257 if ("proceed".equals(action)) {
2258 messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId);
2259 if (isOmemoEnabled()) {
2260 final int deviceId = id.account.getAxolotlService().getOwnDeviceId();
2261 final Element device =
2262 intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION);
2263 device.setAttribute("id", deviceId);
2264 }
2265 }
2266 messagePacket.addChild("store", "urn:xmpp:hints");
2267 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
2268 }
2269
2270 private boolean isOmemoEnabled() {
2271 final Conversational conversational = message.getConversation();
2272 if (conversational instanceof Conversation) {
2273 return ((Conversation) conversational).getNextEncryption()
2274 == Message.ENCRYPTION_AXOLOTL;
2275 }
2276 return false;
2277 }
2278
2279 private void acceptCallFromSessionInitialized() {
2280 xmppConnectionService.getNotificationService().cancelIncomingCallNotification();
2281 sendSessionAccept();
2282 }
2283
2284 private synchronized boolean isInState(State... state) {
2285 return Arrays.asList(state).contains(this.state);
2286 }
2287
2288 private boolean transition(final State target) {
2289 return transition(target, null);
2290 }
2291
2292 private synchronized boolean transition(final State target, final Runnable runnable) {
2293 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
2294 if (validTransitions != null && validTransitions.contains(target)) {
2295 this.state = target;
2296 if (runnable != null) {
2297 runnable.run();
2298 }
2299 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
2300 updateEndUserState();
2301 updateOngoingCallNotification();
2302 return true;
2303 } else {
2304 return false;
2305 }
2306 }
2307
2308 void transitionOrThrow(final State target) {
2309 if (!transition(target)) {
2310 throw new IllegalStateException(
2311 String.format("Unable to transition from %s to %s", this.state, target));
2312 }
2313 }
2314
2315 @Override
2316 public void onIceCandidate(final IceCandidate iceCandidate) {
2317 final RtpContentMap rtpContentMap =
2318 isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
2319 final IceUdpTransportInfo.Credentials credentials;
2320 try {
2321 credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid);
2322 } catch (final IllegalArgumentException e) {
2323 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e);
2324 return;
2325 }
2326 final String uFrag = credentials.ufrag;
2327 final IceUdpTransportInfo.Candidate candidate =
2328 IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag);
2329 if (candidate == null) {
2330 Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate);
2331 return;
2332 }
2333 Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate);
2334 sendTransportInfo(iceCandidate.sdpMid, candidate);
2335 }
2336
2337 @Override
2338 public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
2339 Log.d(
2340 Config.LOGTAG,
2341 id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState);
2342 this.stateHistory.add(newState);
2343 if (newState == PeerConnection.PeerConnectionState.CONNECTED) {
2344 this.sessionDuration.start();
2345 updateOngoingCallNotification();
2346 } else if (this.sessionDuration.isRunning()) {
2347 this.sessionDuration.stop();
2348 updateOngoingCallNotification();
2349 }
2350
2351 final boolean neverConnected =
2352 !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED);
2353
2354 if (newState == PeerConnection.PeerConnectionState.FAILED) {
2355 if (neverConnected) {
2356 if (isTerminated()) {
2357 Log.d(
2358 Config.LOGTAG,
2359 id.account.getJid().asBareJid()
2360 + ": not sending session-terminate after connectivity error because session is already in state "
2361 + this.state);
2362 return;
2363 }
2364 webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
2365 return;
2366 } else {
2367 this.restartIce();
2368 }
2369 }
2370 updateEndUserState();
2371 }
2372
2373 private void restartIce() {
2374 this.stateHistory.clear();
2375 this.webRTCWrapper.restartIceAsync();
2376 }
2377
2378 @Override
2379 public void onRenegotiationNeeded() {
2380 this.webRTCWrapper.execute(this::renegotiate);
2381 }
2382
2383 private void renegotiate() {
2384 final SessionDescription sessionDescription;
2385 try {
2386 sessionDescription = setLocalSessionDescription();
2387 } catch (final Exception e) {
2388 final Throwable cause = Throwables.getRootCause(e);
2389 Log.d(Config.LOGTAG, "failed to renegotiate", cause);
2390 webRTCWrapper.close();
2391 sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
2392 return;
2393 }
2394 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, isInitiator());
2395 final RtpContentMap currentContentMap = getLocalContentMap();
2396 final boolean iceRestart = currentContentMap.iceRestart(rtpContentMap);
2397 final RtpContentMap.Diff diff = currentContentMap.diff(rtpContentMap);
2398
2399 Log.d(
2400 Config.LOGTAG,
2401 id.getAccount().getJid().asBareJid()
2402 + ": renegotiate. iceRestart="
2403 + iceRestart
2404 + " content id diff="
2405 + diff);
2406
2407 if (diff.hasModifications() && iceRestart) {
2408 webRTCWrapper.close();
2409 sendSessionTerminate(
2410 Reason.FAILED_APPLICATION,
2411 "WebRTC unexpectedly tried to modify content and transport at once");
2412 return;
2413 }
2414
2415 if (iceRestart) {
2416 initiateIceRestart(rtpContentMap);
2417 return;
2418 } else if (diff.isEmpty()) {
2419 Log.d(
2420 Config.LOGTAG,
2421 "renegotiation. nothing to do. SignalingState="
2422 + this.webRTCWrapper.getSignalingState());
2423 }
2424
2425 if (diff.added.size() > 0) {
2426 modifyLocalContentMap(rtpContentMap);
2427 sendContentAdd(rtpContentMap, diff.added);
2428 }
2429 }
2430
2431 private void initiateIceRestart(final RtpContentMap rtpContentMap) {
2432 final RtpContentMap transportInfo = rtpContentMap.transportInfo();
2433 final JinglePacket jinglePacket =
2434 transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
2435 Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
2436 jinglePacket.setTo(id.with);
2437 xmppConnectionService.sendIqPacket(
2438 id.account,
2439 jinglePacket,
2440 (account, response) -> {
2441 if (response.getType() == IqPacket.TYPE.RESULT) {
2442 Log.d(Config.LOGTAG, "received success to our ice restart");
2443 setLocalContentMap(rtpContentMap);
2444 webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2445 return;
2446 }
2447 if (response.getType() == IqPacket.TYPE.ERROR) {
2448 if (isTieBreak(response)) {
2449 Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
2450 return;
2451 }
2452 handleIqErrorResponse(response);
2453 }
2454 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2455 handleIqTimeoutResponse(response);
2456 }
2457 });
2458 }
2459
2460 private boolean isTieBreak(final IqPacket response) {
2461 final Element error = response.findChild("error");
2462 return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
2463 }
2464
2465 private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
2466 final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
2467 this.outgoingContentAdd = contentAdd;
2468 final JinglePacket jinglePacket =
2469 contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
2470 jinglePacket.setTo(id.with);
2471 xmppConnectionService.sendIqPacket(
2472 id.account,
2473 jinglePacket,
2474 (connection, response) -> {
2475 if (response.getType() == IqPacket.TYPE.RESULT) {
2476 Log.d(
2477 Config.LOGTAG,
2478 id.getAccount().getJid().asBareJid()
2479 + ": received ACK to our content-add");
2480 return;
2481 }
2482 if (response.getType() == IqPacket.TYPE.ERROR) {
2483 if (isTieBreak(response)) {
2484 this.outgoingContentAdd = null;
2485 Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
2486 return;
2487 }
2488 handleIqErrorResponse(response);
2489 }
2490 if (response.getType() == IqPacket.TYPE.TIMEOUT) {
2491 handleIqTimeoutResponse(response);
2492 }
2493 });
2494 this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
2495 }
2496
2497 private void setLocalContentMap(final RtpContentMap rtpContentMap) {
2498 if (isInitiator()) {
2499 this.initiatorRtpContentMap = rtpContentMap;
2500 } else {
2501 this.responderRtpContentMap = rtpContentMap;
2502 }
2503 }
2504
2505 private void setRemoteContentMap(final RtpContentMap rtpContentMap) {
2506 if (isInitiator()) {
2507 this.responderRtpContentMap = rtpContentMap;
2508 } else {
2509 this.initiatorRtpContentMap = rtpContentMap;
2510 }
2511 }
2512
2513 // this method is to be used for content map modifications that modify media
2514 private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
2515 final RtpContentMap activeContents = rtpContentMap.activeContents();
2516 setLocalContentMap(activeContents);
2517 this.webRTCWrapper.switchSpeakerPhonePreference(
2518 AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
2519 updateEndUserState();
2520 }
2521
2522 private SessionDescription setLocalSessionDescription()
2523 throws ExecutionException, InterruptedException {
2524 final org.webrtc.SessionDescription sessionDescription =
2525 this.webRTCWrapper.setLocalDescription().get();
2526 return SessionDescription.parse(sessionDescription.description);
2527 }
2528
2529 private void closeWebRTCSessionAfterFailedConnection() {
2530 this.webRTCWrapper.close();
2531 synchronized (this) {
2532 if (isTerminated()) {
2533 Log.d(
2534 Config.LOGTAG,
2535 id.account.getJid().asBareJid()
2536 + ": no need to send session-terminate after failed connection. Other party already did");
2537 return;
2538 }
2539 sendSessionTerminate(Reason.CONNECTIVITY_ERROR);
2540 }
2541 }
2542
2543 public boolean zeroDuration() {
2544 return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0;
2545 }
2546
2547 public long getCallDuration() {
2548 return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS);
2549 }
2550
2551 public AppRTCAudioManager getAudioManager() {
2552 return webRTCWrapper.getAudioManager();
2553 }
2554
2555 public boolean isMicrophoneEnabled() {
2556 return webRTCWrapper.isMicrophoneEnabled();
2557 }
2558
2559 public boolean setMicrophoneEnabled(final boolean enabled) {
2560 return webRTCWrapper.setMicrophoneEnabled(enabled);
2561 }
2562
2563 public boolean isVideoEnabled() {
2564 return webRTCWrapper.isVideoEnabled();
2565 }
2566
2567 public void setVideoEnabled(final boolean enabled) {
2568 webRTCWrapper.setVideoEnabled(enabled);
2569 }
2570
2571 public boolean isCameraSwitchable() {
2572 return webRTCWrapper.isCameraSwitchable();
2573 }
2574
2575 public boolean isFrontCamera() {
2576 return webRTCWrapper.isFrontCamera();
2577 }
2578
2579 public ListenableFuture<Boolean> switchCamera() {
2580 return webRTCWrapper.switchCamera();
2581 }
2582
2583 @Override
2584 public void onAudioDeviceChanged(
2585 AppRTCAudioManager.AudioDevice selectedAudioDevice,
2586 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices) {
2587 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2588 selectedAudioDevice, availableAudioDevices);
2589 }
2590
2591 private void updateEndUserState() {
2592 final RtpEndUserState endUserState = getEndUserState();
2593 jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia());
2594 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2595 id.account, id.with, id.sessionId, endUserState);
2596 }
2597
2598 private void updateOngoingCallNotification() {
2599 final State state = this.state;
2600 if (STATES_SHOWING_ONGOING_CALL.contains(state)) {
2601 final boolean reconnecting;
2602 if (state == State.SESSION_ACCEPTED) {
2603 reconnecting =
2604 getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING;
2605 } else {
2606 reconnecting = false;
2607 }
2608 xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting);
2609 } else {
2610 xmppConnectionService.removeOngoingCall();
2611 }
2612 }
2613
2614 private void discoverIceServers(final OnIceServersDiscovered onIceServersDiscovered) {
2615 if (id.account.getXmppConnection().getFeatures().externalServiceDiscovery()) {
2616 final IqPacket request = new IqPacket(IqPacket.TYPE.GET);
2617 request.setTo(id.account.getDomain());
2618 request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2619 xmppConnectionService.sendIqPacket(
2620 id.account,
2621 request,
2622 (account, response) -> {
2623 ImmutableList.Builder<PeerConnection.IceServer> listBuilder =
2624 new ImmutableList.Builder<>();
2625 if (response.getType() == IqPacket.TYPE.RESULT) {
2626 final Element services =
2627 response.findChild(
2628 "services", Namespace.EXTERNAL_SERVICE_DISCOVERY);
2629 final List<Element> children =
2630 services == null
2631 ? Collections.emptyList()
2632 : services.getChildren();
2633 for (final Element child : children) {
2634 if ("service".equals(child.getName())) {
2635 final String type = child.getAttribute("type");
2636 final String host = child.getAttribute("host");
2637 final String sport = child.getAttribute("port");
2638 final Integer port =
2639 sport == null ? null : Ints.tryParse(sport);
2640 final String transport = child.getAttribute("transport");
2641 final String username = child.getAttribute("username");
2642 final String password = child.getAttribute("password");
2643 if (Strings.isNullOrEmpty(host) || port == null) {
2644 continue;
2645 }
2646 if (port < 0 || port > 65535) {
2647 continue;
2648 }
2649
2650
2651
2652
2653 if (Arrays.asList("stun", "stuns", "turn", "turns")
2654 .contains(type)
2655 && Arrays.asList("udp", "tcp").contains(transport)) {
2656 if (Arrays.asList("stuns", "turns").contains(type)
2657 && "udp".equals(transport)) {
2658 Log.d(
2659 Config.LOGTAG,
2660 id.account.getJid().asBareJid()
2661 + ": skipping invalid combination of udp/tls in external services");
2662 continue;
2663 }
2664
2665 // STUN URLs do not support a query section since M110
2666 final String uri;
2667 if (Arrays.asList("stun","stuns").contains(type)) {
2668 uri = String.format("%s:%s%s", type, IP.wrapIPv6(host),port);
2669 } else {
2670 uri = String.format(
2671 "%s:%s:%s?transport=%s",
2672 type,
2673 IP.wrapIPv6(host),
2674 port,
2675 transport);
2676 }
2677
2678 final PeerConnection.IceServer.Builder iceServerBuilder =
2679 PeerConnection.IceServer.builder(uri);
2680 iceServerBuilder.setTlsCertPolicy(
2681 PeerConnection.TlsCertPolicy
2682 .TLS_CERT_POLICY_INSECURE_NO_CHECK);
2683 if (username != null && password != null) {
2684 iceServerBuilder.setUsername(username);
2685 iceServerBuilder.setPassword(password);
2686 } else if (Arrays.asList("turn", "turns").contains(type)) {
2687 // The WebRTC spec requires throwing an
2688 // InvalidAccessError when username (from libwebrtc
2689 // source coder)
2690 // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc
2691 Log.d(
2692 Config.LOGTAG,
2693 id.account.getJid().asBareJid()
2694 + ": skipping "
2695 + type
2696 + "/"
2697 + transport
2698 + " without username and password");
2699 continue;
2700 }
2701 final PeerConnection.IceServer iceServer =
2702 iceServerBuilder.createIceServer();
2703 Log.d(
2704 Config.LOGTAG,
2705 id.account.getJid().asBareJid()
2706 + ": discovered ICE Server: "
2707 + iceServer);
2708 listBuilder.add(iceServer);
2709 }
2710 }
2711 }
2712 }
2713 final List<PeerConnection.IceServer> iceServers = listBuilder.build();
2714 if (iceServers.size() == 0) {
2715 Log.w(
2716 Config.LOGTAG,
2717 id.account.getJid().asBareJid()
2718 + ": no ICE server found "
2719 + response);
2720 }
2721 onIceServersDiscovered.onIceServersDiscovered(iceServers);
2722 });
2723 } else {
2724 Log.w(
2725 Config.LOGTAG,
2726 id.account.getJid().asBareJid() + ": has no external service discovery");
2727 onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList());
2728 }
2729 }
2730
2731 private void finish() {
2732 if (isTerminated()) {
2733 this.cancelRingingTimeout();
2734 this.webRTCWrapper.verifyClosed();
2735 this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia());
2736 this.jingleConnectionManager.finishConnectionOrThrow(this);
2737 try {
2738 File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log");
2739 log.getParentFile().mkdirs();
2740 Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()});
2741 } catch (final IOException e) { }
2742 } else {
2743 throw new IllegalStateException(
2744 String.format("Unable to call finish from %s", this.state));
2745 }
2746 }
2747
2748 private void writeLogMessage(final State state) {
2749 final long duration = getCallDuration();
2750 if (state == State.TERMINATED_SUCCESS
2751 || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) {
2752 writeLogMessageSuccess(duration);
2753 } else {
2754 writeLogMessageMissed();
2755 }
2756 }
2757
2758 private void writeLogMessageSuccess(final long duration) {
2759 this.message.setBody(new RtpSessionStatus(true, duration).toString());
2760 this.writeMessage();
2761 }
2762
2763 private void writeLogMessageMissed() {
2764 this.message.setBody(new RtpSessionStatus(false, 0).toString());
2765 this.writeMessage();
2766 }
2767
2768 private void writeMessage() {
2769 final Conversational conversational = message.getConversation();
2770 if (conversational instanceof Conversation) {
2771 ((Conversation) conversational).add(this.message);
2772 xmppConnectionService.createMessageAsync(message);
2773 xmppConnectionService.updateConversationUi();
2774 } else {
2775 throw new IllegalStateException("Somehow the conversation in a message was a stub");
2776 }
2777 }
2778
2779 public State getState() {
2780 return this.state;
2781 }
2782
2783 boolean isTerminated() {
2784 return TERMINATED.contains(this.state);
2785 }
2786
2787 public Optional<VideoTrack> getLocalVideoTrack() {
2788 return webRTCWrapper.getLocalVideoTrack();
2789 }
2790
2791 public Optional<VideoTrack> getRemoteVideoTrack() {
2792 return webRTCWrapper.getRemoteVideoTrack();
2793 }
2794
2795 public EglBase.Context getEglBaseContext() {
2796 return webRTCWrapper.getEglBaseContext();
2797 }
2798
2799 void setProposedMedia(final Set<Media> media) {
2800 this.proposedMedia = media;
2801 }
2802
2803 public void fireStateUpdate() {
2804 final RtpEndUserState endUserState = getEndUserState();
2805 xmppConnectionService.notifyJingleRtpConnectionUpdate(
2806 id.account, id.with, id.sessionId, endUserState);
2807 }
2808
2809 public boolean isSwitchToVideoAvailable() {
2810 final boolean prerequisite =
2811 Media.audioOnly(getMedia())
2812 && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING)
2813 .contains(getEndUserState());
2814 return prerequisite && remoteHasVideoFeature();
2815 }
2816
2817 private boolean remoteHasVideoFeature() {
2818 final Contact contact = id.getContact();
2819 final Presence presence =
2820 contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
2821 final ServiceDiscoveryResult serviceDiscoveryResult =
2822 presence == null ? null : presence.getServiceDiscoveryResult();
2823 final List<String> features =
2824 serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
2825 return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO);
2826 }
2827
2828 private interface OnIceServersDiscovered {
2829 void onIceServersDiscovered(List<PeerConnection.IceServer> iceServers);
2830 }
2831}