1package eu.siacs.conversations.xmpp.jingle;
2
3import android.util.Log;
4
5import com.google.common.collect.ImmutableList;
6import com.google.common.collect.ImmutableMap;
7
8import org.webrtc.AudioSource;
9import org.webrtc.AudioTrack;
10import org.webrtc.DataChannel;
11import org.webrtc.IceCandidate;
12import org.webrtc.MediaConstraints;
13import org.webrtc.MediaStream;
14import org.webrtc.PeerConnection;
15import org.webrtc.PeerConnectionFactory;
16import org.webrtc.RtpReceiver;
17import org.webrtc.SdpObserver;
18
19import java.util.ArrayDeque;
20import java.util.Arrays;
21import java.util.Collection;
22import java.util.Collections;
23import java.util.List;
24import java.util.Map;
25
26import eu.siacs.conversations.Config;
27import eu.siacs.conversations.xml.Element;
28import eu.siacs.conversations.xml.Namespace;
29import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
30import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
31import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
32import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
33import rocks.xmpp.addr.Jid;
34
35public class JingleRtpConnection extends AbstractJingleConnection {
36
37 private static final Map<State, Collection<State>> VALID_TRANSITIONS;
38
39 static {
40 final ImmutableMap.Builder<State, Collection<State>> transitionBuilder = new ImmutableMap.Builder<>();
41 transitionBuilder.put(State.NULL, ImmutableList.of(State.PROPOSED, State.SESSION_INITIALIZED));
42 transitionBuilder.put(State.PROPOSED, ImmutableList.of(State.ACCEPTED, State.PROCEED));
43 transitionBuilder.put(State.PROCEED, ImmutableList.of(State.SESSION_INITIALIZED));
44 VALID_TRANSITIONS = transitionBuilder.build();
45 }
46
47 private State state = State.NULL;
48 private RtpContentMap initialRtpContentMap;
49 private PeerConnection peerConnection;
50
51 private final ArrayDeque<IceCandidate> pendingIceCandidates = new ArrayDeque<>();
52
53
54 public JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
55 super(jingleConnectionManager, id, initiator);
56 }
57
58 @Override
59 void deliverPacket(final JinglePacket jinglePacket) {
60 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
61 switch (jinglePacket.getAction()) {
62 case SESSION_INITIATE:
63 receiveSessionInitiate(jinglePacket);
64 break;
65 case TRANSPORT_INFO:
66 receiveTransportInfo(jinglePacket);
67 break;
68 default:
69 Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction()));
70 break;
71 }
72 }
73
74 private void receiveTransportInfo(final JinglePacket jinglePacket) {
75 if (isInState(State.SESSION_INITIALIZED, State.SESSION_ACCEPTED)) {
76 final RtpContentMap contentMap;
77 try {
78 contentMap = RtpContentMap.of(jinglePacket);
79 } catch (IllegalArgumentException | NullPointerException e) {
80 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e);
81 return;
82 }
83 final Group originalGroup = this.initialRtpContentMap != null ? this.initialRtpContentMap.group : null;
84 final List<String> identificationTags = originalGroup == null ? Collections.emptyList() : originalGroup.getIdentificationTags();
85 if (identificationTags.size() == 0) {
86 Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
87 }
88 for(final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contentMap.contents.entrySet()) {
89 final String ufrag = content.getValue().transport.getAttribute("ufrag");
90 for(final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
91 final String sdp = candidate.toSdpAttribute(ufrag);
92 final String sdpMid = content.getKey();
93 final int mLineIndex = identificationTags.indexOf(sdpMid);
94 final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
95 Log.d(Config.LOGTAG,"received candidate: "+iceCandidate);
96 if (isInState(State.SESSION_ACCEPTED)) {
97 this.peerConnection.addIceCandidate(iceCandidate);
98 } else {
99 this.pendingIceCandidates.push(iceCandidate);
100 }
101 }
102 }
103 } else {
104 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state);
105 }
106 }
107
108 private void receiveSessionInitiate(final JinglePacket jinglePacket) {
109 Log.d(Config.LOGTAG, jinglePacket.toString());
110 if (isInitiator()) {
111 Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
112 //TODO respond with out-of-order
113 return;
114 }
115 final RtpContentMap contentMap;
116 try {
117 contentMap = RtpContentMap.of(jinglePacket);
118 contentMap.requireContentDescriptions();
119 } catch (IllegalArgumentException | IllegalStateException | NullPointerException e) {
120 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e);
121 return;
122 }
123 Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
124 final State oldState = this.state;
125 if (transition(State.SESSION_INITIALIZED)) {
126 this.initialRtpContentMap = contentMap;
127 if (oldState == State.PROCEED) {
128 processContents(contentMap);
129 sendSessionAccept();
130 } else {
131 //TODO start ringing
132 }
133 } else {
134 Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state));
135 }
136 }
137
138 private void processContents(final RtpContentMap contentMap) {
139 setupWebRTC();
140 org.webrtc.SessionDescription sessionDescription = new org.webrtc.SessionDescription(org.webrtc.SessionDescription.Type.OFFER, SessionDescription.of(contentMap).toString());
141 Log.d(Config.LOGTAG, "debug print for sessionDescription:" + sessionDescription.description);
142 this.peerConnection.setRemoteDescription(new SdpObserver() {
143 @Override
144 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
145
146 }
147
148 @Override
149 public void onSetSuccess() {
150 Log.d(Config.LOGTAG, "onSetSuccess() for setRemoteDescription");
151 }
152
153 @Override
154 public void onCreateFailure(String s) {
155
156 }
157
158 @Override
159 public void onSetFailure(String s) {
160 Log.d(Config.LOGTAG, "onSetFailure() for setRemoteDescription. " + s);
161
162 }
163 }, sessionDescription);
164 }
165
166 void deliveryMessage(final Jid from, final Element message) {
167 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message);
168 switch (message.getName()) {
169 case "propose":
170 receivePropose(from, message);
171 break;
172 case "proceed":
173 receiveProceed(from, message);
174 default:
175 break;
176 }
177 }
178
179 private void receivePropose(final Jid from, final Element propose) {
180 final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
181 if (originatedFromMyself) {
182 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from mysql. ignoring");
183 } else if (transition(State.PROPOSED)) {
184 //TODO start ringing or something
185 pickUpCall();
186 } else {
187 Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state);
188 }
189 }
190
191 private void receiveProceed(final Jid from, final Element proceed) {
192 if (from.equals(id.with)) {
193 if (isInitiator()) {
194 if (transition(State.PROCEED)) {
195 this.sendSessionInitiate();
196 } else {
197 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state));
198 }
199 } else {
200 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid()));
201 }
202 } else {
203 Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with));
204 }
205 }
206
207 private void sendSessionInitiate() {
208 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
209 setupWebRTC();
210 createOffer();
211 }
212
213 private void sendSessionInitiate(RtpContentMap rtpContentMap) {
214 this.initialRtpContentMap = rtpContentMap;
215 final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
216 Log.d(Config.LOGTAG, sessionInitiate.toString());
217 Log.d(Config.LOGTAG, "here is what we think the sdp looks like" + SessionDescription.of(rtpContentMap).toString());
218 send(sessionInitiate);
219 }
220
221 private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
222 final RtpContentMap transportInfo;
223 try {
224 transportInfo = this.initialRtpContentMap.transportInfo(contentName, candidate);
225 } catch (Exception e) {
226 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
227 return;
228 }
229 final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
230 Log.d(Config.LOGTAG, jinglePacket.toString());
231 send(jinglePacket);
232 }
233
234 private void send(final JinglePacket jinglePacket) {
235 jinglePacket.setTo(id.with);
236 //TODO track errors
237 xmppConnectionService.sendIqPacket(id.account, jinglePacket, null);
238 }
239
240
241 private void sendSessionAccept() {
242 Log.d(Config.LOGTAG, "sending session-accept");
243 }
244
245 public void pickUpCall() {
246 switch (this.state) {
247 case PROPOSED:
248 pickupCallFromProposed();
249 break;
250 case SESSION_INITIALIZED:
251 pickupCallFromSessionInitialized();
252 break;
253 default:
254 throw new IllegalStateException("Can not pick up call from " + this.state);
255 }
256 }
257
258 private void setupWebRTC() {
259 PeerConnectionFactory.initialize(
260 PeerConnectionFactory.InitializationOptions.builder(xmppConnectionService).createInitializationOptions()
261 );
262 final PeerConnectionFactory.Options options = new PeerConnectionFactory.Options();
263 PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder().createPeerConnectionFactory();
264
265 final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
266
267 final AudioTrack audioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
268 final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
269 stream.addTrack(audioTrack);
270
271
272 final List<PeerConnection.IceServer> iceServers = ImmutableList.of(
273 PeerConnection.IceServer.builder("stun:xmpp.conversations.im:3478").createIceServer()
274 );
275 this.peerConnection = peerConnectionFactory.createPeerConnection(iceServers, new PeerConnection.Observer() {
276 @Override
277 public void onSignalingChange(PeerConnection.SignalingState signalingState) {
278
279 }
280
281 @Override
282 public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
283
284 }
285
286 @Override
287 public void onIceConnectionReceivingChange(boolean b) {
288
289 }
290
291 @Override
292 public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
293 Log.d(Config.LOGTAG, "onIceGatheringChange() " + iceGatheringState);
294 }
295
296 @Override
297 public void onIceCandidate(IceCandidate iceCandidate) {
298 IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
299 Log.d(Config.LOGTAG, "onIceCandidate: " + iceCandidate.sdp + " mLineIndex=" + iceCandidate.sdpMLineIndex);
300 sendTransportInfo(iceCandidate.sdpMid, candidate);
301
302 }
303
304 @Override
305 public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
306
307 }
308
309 @Override
310 public void onAddStream(MediaStream mediaStream) {
311
312 }
313
314 @Override
315 public void onRemoveStream(MediaStream mediaStream) {
316
317 }
318
319 @Override
320 public void onDataChannel(DataChannel dataChannel) {
321
322 }
323
324 @Override
325 public void onRenegotiationNeeded() {
326
327 }
328
329 @Override
330 public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
331
332 }
333 });
334
335 peerConnection.addStream(stream);
336 }
337
338 private void createOffer() {
339 Log.d(Config.LOGTAG, "createOffer()");
340 peerConnection.createOffer(new SdpObserver() {
341
342 @Override
343 public void onCreateSuccess(org.webrtc.SessionDescription description) {
344 final SessionDescription sessionDescription = SessionDescription.parse(description.description);
345 Log.d(Config.LOGTAG, "description: " + description.description);
346 final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
347 sendSessionInitiate(rtpContentMap);
348 peerConnection.setLocalDescription(new SdpObserver() {
349 @Override
350 public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
351
352 }
353
354 @Override
355 public void onSetSuccess() {
356 Log.d(Config.LOGTAG, "onSetSuccess()");
357 }
358
359 @Override
360 public void onCreateFailure(String s) {
361
362 }
363
364 @Override
365 public void onSetFailure(String s) {
366
367 }
368 }, description);
369 }
370
371 @Override
372 public void onSetSuccess() {
373
374 }
375
376 @Override
377 public void onCreateFailure(String s) {
378
379 }
380
381 @Override
382 public void onSetFailure(String s) {
383
384 }
385 }, new MediaConstraints());
386 }
387
388 private void pickupCallFromProposed() {
389 transitionOrThrow(State.PROCEED);
390 final MessagePacket messagePacket = new MessagePacket();
391 messagePacket.setTo(id.with);
392 //Note that Movim needs 'accept', correct is 'proceed' https://github.com/movim/movim/issues/916
393 messagePacket.addChild("proceed", Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId);
394 Log.d(Config.LOGTAG, messagePacket.toString());
395 xmppConnectionService.sendMessagePacket(id.account, messagePacket);
396 }
397
398 private void pickupCallFromSessionInitialized() {
399
400 }
401
402 private synchronized boolean isInState(State... state) {
403 return Arrays.asList(state).contains(this.state);
404 }
405
406 private synchronized boolean transition(final State target) {
407 final Collection<State> validTransitions = VALID_TRANSITIONS.get(this.state);
408 if (validTransitions != null && validTransitions.contains(target)) {
409 this.state = target;
410 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into " + target);
411 return true;
412 } else {
413 return false;
414 }
415 }
416
417 public void transitionOrThrow(final State target) {
418 if (!transition(target)) {
419 throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
420 }
421 }
422}