From 3c9fef4af23491f4d1dc6e8629ad2f57c33b897d Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Mon, 12 Dec 2022 22:05:26 -0500 Subject: [PATCH 01/13] Parse all SIMS and OOBs on a message Use a set that keys on URL so that we don't get duplicates. Still only store the first one for now. --- .../siacs/conversations/entities/Message.java | 10 +++++++ .../conversations/parser/MessageParser.java | 27 ++++++++++++------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/main/java/eu/siacs/conversations/entities/Message.java b/src/main/java/eu/siacs/conversations/entities/Message.java index 8234408b31c58116fd82c6bd1159f89d441245ae..2f2564acab60b68ac89387f7844ac240470315ed 100644 --- a/src/main/java/eu/siacs/conversations/entities/Message.java +++ b/src/main/java/eu/siacs/conversations/entities/Message.java @@ -1125,6 +1125,16 @@ public class Message extends AbstractEntity implements AvatarService.Avatarable if (runtime > 0) builder.append('|').append(runtime); return builder.toString(); } + + public boolean equals(Object o) { + if (!(o instanceof FileParams)) return false; + + return url.equals(((FileParams) o).url); + } + + public int hashCode() { + return url.hashCode(); + } } public void setFingerprint(String fingerprint) { diff --git a/src/main/java/eu/siacs/conversations/parser/MessageParser.java b/src/main/java/eu/siacs/conversations/parser/MessageParser.java index a6f924b28cb7e903f872c88285dfb18395fc14e0..f4030e437d4fa0136838326340ea97872f3c700c 100644 --- a/src/main/java/eu/siacs/conversations/parser/MessageParser.java +++ b/src/main/java/eu/siacs/conversations/parser/MessageParser.java @@ -11,6 +11,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -410,13 +411,19 @@ public class MessageParser extends AbstractParser implements OnMessagePacketRece final Element mucUserElement = packet.findChild("x", Namespace.MUC_USER); final String pgpEncrypted = packet.findChildContent("x", "jabber:x:encrypted"); final Element replaceElement = packet.findChild("replace", "urn:xmpp:message-correct:0"); - Element oob = packet.findChild("x", Namespace.OOB); - if (oob != null && oob.findChildContent("url") == null) { - oob = null; + Set attachments = new LinkedHashSet<>(); + for (Element child : packet.getChildren()) { + // SIMS first so they get preference in the set + if (child.getName().equals("reference") && child.getNamespace().equals("urn:xmpp:reference:0")) { + if (child.findChild("media-sharing", "urn:xmpp:sims:1") != null) { + attachments.add(new Message.FileParams(child)); + } + } } - final Element reference = packet.findChild("reference", "urn:xmpp:reference:0"); - if (reference != null && reference.findChild("media-sharing", "urn:xmpp:sims:1") != null) { - oob = reference; + for (Element child : packet.getChildren()) { + if (child.getName().equals("x") && child.getNamespace().equals(Namespace.OOB)) { + attachments.add(new Message.FileParams(child)); + } } String replacementId = replaceElement == null ? null : replaceElement.getAttribute("id"); if (replacementId == null) { @@ -485,7 +492,7 @@ public class MessageParser extends AbstractParser implements OnMessagePacketRece } } - if ((body != null || pgpEncrypted != null || (axolotlEncrypted != null && axolotlEncrypted.hasChild("payload")) || oob != null || html != null) && !isMucStatusMessage) { + if ((body != null || pgpEncrypted != null || (axolotlEncrypted != null && axolotlEncrypted.hasChild("payload")) || !attachments.isEmpty() || html != null) && !isMucStatusMessage) { final boolean conversationIsProbablyMuc = isTypeGroupChat || mucUserElement != null || account.getXmppConnection().getMucServersWithholdAccount().contains(counterpart.getDomain().toEscapedString()); final Conversation conversation = mXmppConnectionService.findOrCreateConversation(account, counterpart.asBareJid(), conversationIsProbablyMuc, false, query, false); final boolean conversationMultiMode = conversation.getMode() == Conversation.MODE_MULTI; @@ -583,7 +590,7 @@ public class MessageParser extends AbstractParser implements OnMessagePacketRece if (conversationMultiMode) { message.setTrueCounterpart(origin); } - } else if (body == null && oob != null) { + } else if (body == null && !attachments.isEmpty()) { message = new Message(conversation, "", Message.ENCRYPTION_NONE, status); } else { message = new Message(conversation, body == null ? "HTML-only message" : body.content, Message.ENCRYPTION_NONE, status); @@ -599,8 +606,8 @@ public class MessageParser extends AbstractParser implements OnMessagePacketRece message.setServerMsgId(serverMsgId); message.setCarbon(isCarbon); message.setTime(timestamp); - if (oob != null) { - message.setFileParams(new Message.FileParams(oob)); + if (!attachments.isEmpty()) { + message.setFileParams(attachments.iterator().next()); if (CryptoHelper.isPgpEncryptedUrl(message.getFileParams().url)) { message.setEncryption(Message.ENCRYPTION_DECRYPTED); } From 5e414dd7b99ebc8cb6c17e6d99beb62190d2725d Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Tue, 13 Dec 2022 01:39:03 -0500 Subject: [PATCH 02/13] Prevent accidental mutation of Elements If you want to mutate, use bindTo and mutate the Element, don't getChildren() and mutate that or it will get out of sync with childNodes --- src/main/java/eu/siacs/conversations/xml/Element.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/eu/siacs/conversations/xml/Element.java b/src/main/java/eu/siacs/conversations/xml/Element.java index 0b572a64318d191cff15d2e5a25ca234f97c233c..269cd3aaaa410b7759a31e23cb7df9385574c69a 100644 --- a/src/main/java/eu/siacs/conversations/xml/Element.java +++ b/src/main/java/eu/siacs/conversations/xml/Element.java @@ -1,6 +1,7 @@ package eu.siacs.conversations.xml; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; import org.jetbrains.annotations.NotNull; @@ -134,13 +135,13 @@ public class Element implements Node { } public final List getChildren() { - return this.children; + return ImmutableList.copyOf(this.children); } // Deprecated: you probably want bindTo or replaceChildren public Element setChildren(List children) { this.childNodes = new ArrayList(children); - this.children = children; + this.children = new ArrayList(children); return this; } From 3944ff152899aecd35bfbf9398b6ccd8bd5b21ba Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Tue, 13 Dec 2022 01:40:45 -0500 Subject: [PATCH 03/13] Store FileParams as SIMS in payloads We no longer use the old fileParams column or serialization format for new messages, instead storing and loading to/from a SIMS element in payloads. If the incoming message had a SIMS element, we preserve it as much as possible and only add newly discovered information to it. If the incoming message had a different kind of attachment (such as OOB) then we synthesize a SIMS element from the extracted data. SIMS has no spec for storing width/heigh/duration of the final media, so use RDF namespace from schema.org for those data. --- .../siacs/conversations/entities/Message.java | 73 +++++++++++++++++-- .../eu/siacs/conversations/xml/Element.java | 2 + 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/main/java/eu/siacs/conversations/entities/Message.java b/src/main/java/eu/siacs/conversations/entities/Message.java index 2f2564acab60b68ac89387f7844ac240470315ed..2e73357a9ceb49502cce027f5afb2273838d2bac 100644 --- a/src/main/java/eu/siacs/conversations/entities/Message.java +++ b/src/main/java/eu/siacs/conversations/entities/Message.java @@ -23,6 +23,7 @@ import java.lang.ref.WeakReference; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -242,8 +243,8 @@ public class Message extends AbstractEntity implements AvatarService.Avatarable this.bodyLanguage = bodyLanguage; this.timeReceived = timeReceived; this.subject = subject; - if (fileParams != null) this.fileParams = new FileParams(fileParams); if (payloads != null) this.payloads = payloads; + if (fileParams != null && getSims().isEmpty()) this.fileParams = new FileParams(fileParams); } public static Message fromCursor(Cursor cursor, Conversation conversation) throws IOException { @@ -319,6 +320,14 @@ public class Message extends AbstractEntity implements AvatarService.Avatarable values.put(UUID, uuid); values.put("subject", subject); values.put("fileParams", fileParams == null ? null : fileParams.toString()); + if (fileParams != null) { + List sims = getSims(); + if (sims.isEmpty()) { + addPayload(fileParams.toSims()); + } else { + sims.get(0).replaceChildren(fileParams.toSims().getChildren()); + } + } values.put("payloads", payloads.size() < 1 ? null : payloads.stream().map(Object::toString).collect(Collectors.joining())); return values; } @@ -996,21 +1005,33 @@ public class Message extends AbstractEntity implements AvatarService.Avatarable return isGeoUri; } + protected List getSims() { + return payloads.stream().filter(el -> + el.getName().equals("reference") && el.getNamespace().equals("urn:xmpp:reference:0") && + el.findChild("media-sharing", "urn:xmpp:sims:1") != null + ).collect(Collectors.toList()); + } + public synchronized void resetFileParams() { this.fileParams = null; } public synchronized void setFileParams(FileParams fileParams) { + if (this.fileParams != null && this.fileParams.sims != null && fileParams.sims == null) { + fileParams.sims = this.fileParams.sims; + } this.fileParams = fileParams; } public synchronized FileParams getFileParams() { if (fileParams == null) { - fileParams = new FileParams(oob ? this.body : ""); + List sims = getSims(); + fileParams = sims.isEmpty() ? new FileParams(oob ? this.body : "") : new FileParams(sims.get(0)); if (this.transferable != null) { fileParams.size = this.transferable.getFileSize(); } } + return fileParams; } @@ -1054,6 +1075,7 @@ public class Message extends AbstractEntity implements AvatarService.Avatarable public int width = 0; public int height = 0; public int runtime = 0; + public Element sims = null; public FileParams() { } @@ -1062,6 +1084,7 @@ public class Message extends AbstractEntity implements AvatarService.Avatarable this.url = el.findChildContent("url", Namespace.OOB); } if (el.getName().equals("reference") && el.getNamespace().equals("urn:xmpp:reference:0")) { + sims = el; final String refUri = el.getAttribute("uri"); if (refUri != null) url = refUri; final Element mediaSharing = el.findChild("media-sharing", "urn:xmpp:sims:1"); @@ -1070,10 +1093,14 @@ public class Message extends AbstractEntity implements AvatarService.Avatarable if (file == null) file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:4"); if (file == null) file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:3"); if (file != null) { - String sizeS = file.findChildContent("size", "urn:xmpp:jingle:apps:file-transfer:5"); - if (sizeS == null) sizeS = file.findChildContent("size", "urn:xmpp:jingle:apps:file-transfer:4"); - if (sizeS == null) sizeS = file.findChildContent("size", "urn:xmpp:jingle:apps:file-transfer:3"); + String sizeS = file.findChildContent("size", file.getNamespace()); if (sizeS != null) size = new Long(sizeS); + String widthS = file.findChildContent("width", "https://schema.org/"); + if (widthS != null) width = parseInt(widthS); + String heightS = file.findChildContent("height", "https://schema.org/"); + if (heightS != null) height = parseInt(heightS); + String durationS = file.findChildContent("duration", "https://schema.org/"); + if (durationS != null) runtime = (int)(Duration.parse(durationS).toMillis() / 1000L); } final Element sources = mediaSharing.findChild("sources", "urn:xmpp:sims:1"); @@ -1116,6 +1143,42 @@ public class Message extends AbstractEntity implements AvatarService.Avatarable return size == null ? 0 : size; } + public Element toSims() { + if (sims == null) sims = new Element("reference", "urn:xmpp:reference:0"); + sims.setAttribute("type", "data"); + Element mediaSharing = sims.findChild("media-sharing", "urn:xmpp:sims:1"); + if (mediaSharing == null) mediaSharing = sims.addChild("media-sharing", "urn:xmpp:sims:1"); + + Element file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:5"); + if (file == null) file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:4"); + if (file == null) file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:3"); + if (file == null) file = mediaSharing.addChild("file", "urn:xmpp:jingle:apps:file-transfer:5"); + + file.removeChild(file.findChild("size", file.getNamespace())); + if (size != null) file.addChild("size", file.getNamespace()).setContent(size.toString()); + + file.removeChild(file.findChild("width", "https://schema.org/")); + if (width > 0) file.addChild("width", "https://schema.org/").setContent(String.valueOf(width)); + + file.removeChild(file.findChild("height", "https://schema.org/")); + if (height > 0) file.addChild("height", "https://schema.org/").setContent(String.valueOf(height)); + + file.removeChild(file.findChild("duration", "https://schema.org/")); + if (runtime > 0) file.addChild("duration", "https://schema.org/").setContent("PT" + runtime + "S"); + + if (url != null) { + Element sources = mediaSharing.findChild("sources", mediaSharing.getNamespace()); + if (sources == null) sources = mediaSharing.addChild("sources", mediaSharing.getNamespace()); + + Element source = sources.findChild("reference", "urn:xmpp:reference:0"); + if (source == null) source = sources.addChild("reference", "urn:xmpp:reference:0"); + source.setAttribute("type", "data"); + source.setAttribute("uri", url); + } + + return sims; + } + public String toString() { final StringBuilder builder = new StringBuilder(); if (url != null) builder.append(url); diff --git a/src/main/java/eu/siacs/conversations/xml/Element.java b/src/main/java/eu/siacs/conversations/xml/Element.java index 269cd3aaaa410b7759a31e23cb7df9385574c69a..83228b2da15580026fc8c97fe61a792c49b85e6b 100644 --- a/src/main/java/eu/siacs/conversations/xml/Element.java +++ b/src/main/java/eu/siacs/conversations/xml/Element.java @@ -71,6 +71,8 @@ public class Element implements Node { } public void removeChild(Node child) { + if (child == null) return; + this.childNodes.remove(child); if (child instanceof Element) this.children.remove(child); } From 2b6197a97a2d15f9c652201af4ce3a583cdb7a20 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Tue, 13 Dec 2022 22:06:35 -0500 Subject: [PATCH 04/13] Helper to get thumbnails from SIMS element --- .../siacs/conversations/entities/Message.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/eu/siacs/conversations/entities/Message.java b/src/main/java/eu/siacs/conversations/entities/Message.java index 2e73357a9ceb49502cce027f5afb2273838d2bac..890523c854e559f5815d8e5550113ad6def7354d 100644 --- a/src/main/java/eu/siacs/conversations/entities/Message.java +++ b/src/main/java/eu/siacs/conversations/entities/Message.java @@ -1179,6 +1179,26 @@ public class Message extends AbstractEntity implements AvatarService.Avatarable return sims; } + public List getThumbnails() { + List thumbs = new ArrayList<>(); + if (sims == null) return thumbs; + + Element mediaSharing = sims.findChild("media-sharing", "urn:xmpp:sims:1"); + if (mediaSharing == null) return thumbs; + Element file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:5"); + if (file == null) file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:4"); + if (file == null) file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:3"); + if (file == null) return thumbs; + + for (Element child : file.getChildren()) { + if (child.getName().equals("thumbnail") && child.getNamespace().equals("urn:xmpp:thumbs:1")) { + thumbs.add(child); + } + } + + return thumbs; + } + public String toString() { final StringBuilder builder = new StringBuilder(); if (url != null) builder.append(url); From b9d8b9373c4ce28d11472f0cfcc58cea0940de35 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Tue, 13 Dec 2022 22:21:42 -0500 Subject: [PATCH 05/13] Show fallback thumbnail if there is no image Some thumbnails are just as good as scaling down the real image, but some (especially blurhash) are not. So if nothing that's "good enough" to go in the main cache is present, try the fallback. If there is a thumbnail we can render right now (as of this commit, just blurhash) then render that alongside the download button. Tapping the image starts the download just like the button. --- build.gradle | 1 + .../persistance/FileBackend.java | 38 +++++++++++++ .../siacs/conversations/ui/XmppActivity.java | 10 +++- .../ui/adapter/MessageAdapter.java | 56 +++++++++++++++---- 4 files changed, 91 insertions(+), 14 deletions(-) diff --git a/build.gradle b/build.gradle index 79c4737acb03a5bfed8adcf642c1de5a634333c9..5a7b41a48febd0617335a793610d608ca4693400 100644 --- a/build.gradle +++ b/build.gradle @@ -100,6 +100,7 @@ dependencies { implementation 'me.saket:better-link-movement-method:2.2.0' implementation 'com.github.singpolyma:android-identicons:master-SNAPSHOT' implementation 'org.snikket:webrtc-android:107.0.0' + implementation 'com.github.woltapp:blurhash:master' // INSERT } diff --git a/src/main/java/eu/siacs/conversations/persistance/FileBackend.java b/src/main/java/eu/siacs/conversations/persistance/FileBackend.java index 067bd3ed6e1d6184da9775519486689ba34e0a94..7dd89d718bfe58c37760affb3c53678df12d3aa5 100644 --- a/src/main/java/eu/siacs/conversations/persistance/FileBackend.java +++ b/src/main/java/eu/siacs/conversations/persistance/FileBackend.java @@ -41,6 +41,8 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.io.ByteStreams; +import com.wolt.blurhashkt.BlurHashDecoder; + import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.File; @@ -78,6 +80,7 @@ import eu.siacs.conversations.utils.FileUtils; import eu.siacs.conversations.utils.FileWriterException; import eu.siacs.conversations.utils.MimeUtils; import eu.siacs.conversations.xmpp.pep.Avatar; +import eu.siacs.conversations.xml.Element; public class FileBackend { @@ -1000,6 +1003,41 @@ public class FileBackend { } } + public BitmapDrawable getFallbackThumbnail(final Message message, int size) { + List thumbs = message.getFileParams() != null ? message.getFileParams().getThumbnails() : null; + if (thumbs != null && !thumbs.isEmpty()) { + for (Element thumb : thumbs) { + Uri uri = Uri.parse(thumb.getAttribute("uri")); + if (uri.getScheme().equals("data")) { + String[] parts = uri.getSchemeSpecificPart().split(",", 2); + if (parts[0].equals("image/blurhash")) { + final LruCache cache = mXmppConnectionService.getDrawableCache(); + BitmapDrawable cached = (BitmapDrawable) cache.get(parts[1]); + if (cached != null) return cached; + + int width = message.getFileParams().width; + if (width < 1 && thumb.getAttribute("width") != null) width = Integer.parseInt(thumb.getAttribute("width")); + if (width < 1) width = 1920; + + int height = message.getFileParams().height; + if (height < 1 && thumb.getAttribute("height") != null) height = Integer.parseInt(thumb.getAttribute("height")); + if (height < 1) height = 1080; + Rect r = rectForSize(width, height, size); + + Bitmap blurhash = BlurHashDecoder.INSTANCE.decode(parts[1], r.width(), r.height(), 1.0f, false); + if (blurhash != null) { + cached = new BitmapDrawable(blurhash); + cache.put(parts[1], cached); + return cached; + } + } + } + } + } + + return null; + } + public Drawable getThumbnail(Message message, Resources res, int size, boolean cacheOnly) throws IOException { return getThumbnail(getFile(message), res, size, cacheOnly); } diff --git a/src/main/java/eu/siacs/conversations/ui/XmppActivity.java b/src/main/java/eu/siacs/conversations/ui/XmppActivity.java index d828101eee0ca317edf97b14bbb0794d1c55b3c4..4490a41757643af940b6a738a27b7c63df600b77 100644 --- a/src/main/java/eu/siacs/conversations/ui/XmppActivity.java +++ b/src/main/java/eu/siacs/conversations/ui/XmppActivity.java @@ -916,8 +916,9 @@ public abstract class XmppActivity extends ActionBarActivity { imageView.setBackgroundColor(0xff333333); imageView.setImageDrawable(null); final BitmapWorkerTask task = new BitmapWorkerTask(imageView); + final BitmapDrawable fallbackThumb = xmppConnectionService.getFileBackend().getFallbackThumbnail(message, (int) (metrics.density * 288)); final AsyncDrawable asyncDrawable = new AsyncDrawable( - getResources(), null, task); + getResources(), fallbackThumb != null ? fallbackThumb.getBitmap() : null, task); imageView.setImageDrawable(asyncDrawable); try { task.execute(message); @@ -995,7 +996,12 @@ public abstract class XmppActivity extends ActionBarActivity { if (!isCancelled()) { final ImageView imageView = imageViewReference.get(); if (imageView != null) { - imageView.setImageDrawable(drawable); + Drawable old = imageView.getDrawable(); + if (drawable == null && old instanceof AsyncDrawable) { + imageView.setImageDrawable(new BitmapDrawable(((AsyncDrawable) old).getBitmap())); + } else { + imageView.setImageDrawable(drawable); + } imageView.setBackgroundColor(drawable == null ? 0xff333333 : 0x00000000); if (Build.VERSION.SDK_INT >= 28 && drawable instanceof AnimatedImageDrawable) { ((AnimatedImageDrawable) drawable).start(); diff --git a/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java b/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java index 1e44e08373324f97beea339a7e92eb8fd1d048b8..a92591937ecba73b6335fc9c0087f6b13171ae7c 100644 --- a/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java +++ b/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java @@ -586,6 +586,34 @@ public class MessageAdapter extends ArrayAdapter { private void displayDownloadableMessage(ViewHolder viewHolder, final Message message, String text, final boolean darkBackground, final int type) { displayTextMessage(viewHolder, message, darkBackground, type); viewHolder.image.setVisibility(View.GONE); + List thumbs = message.getFileParams() != null ? message.getFileParams().getThumbnails() : null; + if (thumbs != null && !thumbs.isEmpty()) { + for (Element thumb : thumbs) { + Uri uri = Uri.parse(thumb.getAttribute("uri")); + if (uri.getScheme().equals("data")) { + String[] parts = uri.getSchemeSpecificPart().split(",", 2); + parts = parts[0].split(";"); + if (!parts[0].equals("image/blurhash")) continue; + } else { + continue; + } + + int width = message.getFileParams().width; + if (width < 1 && thumb.getAttribute("width") != null) width = Integer.parseInt(thumb.getAttribute("width")); + if (width < 1) width = 1920; + + int height = message.getFileParams().height; + if (height < 1 && thumb.getAttribute("height") != null) height = Integer.parseInt(thumb.getAttribute("height")); + if (height < 1) height = 1080; + + viewHolder.image.setVisibility(View.VISIBLE); + imagePreviewLayout(width, height, viewHolder.image); + activity.loadBitmap(message, viewHolder.image); + viewHolder.image.setOnClickListener(v -> ConversationFragment.downloadFile(activity, message)); + + break; + } + } viewHolder.audioPlayer.setVisibility(View.GONE); viewHolder.download_button.setVisibility(View.VISIBLE); viewHolder.download_button.setText(text); @@ -626,27 +654,31 @@ public class MessageAdapter extends ArrayAdapter { viewHolder.audioPlayer.setVisibility(View.GONE); viewHolder.image.setVisibility(View.VISIBLE); final FileParams params = message.getFileParams(); + imagePreviewLayout(params.width, params.height, viewHolder.image); + activity.loadBitmap(message, viewHolder.image); + viewHolder.image.setOnClickListener(v -> openDownloadable(message)); + } + + private void imagePreviewLayout(int w, int h, ImageView image) { final float target = activity.getResources().getDimension(R.dimen.image_preview_width); final int scaledW; final int scaledH; - if (Math.max(params.height, params.width) * metrics.density <= target) { - scaledW = (int) (params.width * metrics.density); - scaledH = (int) (params.height * metrics.density); - } else if (Math.max(params.height, params.width) <= target) { - scaledW = params.width; - scaledH = params.height; - } else if (params.width <= params.height) { - scaledW = (int) (params.width / ((double) params.height / target)); + if (Math.max(h, w) * metrics.density <= target) { + scaledW = (int) (w * metrics.density); + scaledH = (int) (h * metrics.density); + } else if (Math.max(h, w) <= target) { + scaledW = w; + scaledH = h; + } else if (w <= h) { + scaledW = (int) (w / ((double) h / target)); scaledH = (int) target; } else { scaledW = (int) target; - scaledH = (int) (params.height / ((double) params.width / target)); + scaledH = (int) (h / ((double) w / target)); } final LinearLayout.LayoutParams layoutParams = new LinearLayout.LayoutParams(scaledW, scaledH); layoutParams.setMargins(0, (int) (metrics.density * 4), 0, (int) (metrics.density * 4)); - viewHolder.image.setLayoutParams(layoutParams); - activity.loadBitmap(message, viewHolder.image); - viewHolder.image.setOnClickListener(v -> openDownloadable(message)); + image.setLayoutParams(layoutParams); } private void toggleWhisperInfo(ViewHolder viewHolder, final Message message, final boolean darkBackground) { From 944d70a5912362490696537e0d9e0ca37cfe9f89 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Tue, 13 Dec 2022 22:24:18 -0500 Subject: [PATCH 06/13] Show thumbnail during download, if relevant --- .../siacs/conversations/ui/adapter/MessageAdapter.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java b/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java index a92591937ecba73b6335fc9c0087f6b13171ae7c..ebe612059c228fcef8d61115fef7570236d531ad 100644 --- a/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java +++ b/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java @@ -341,6 +341,13 @@ public class MessageAdapter extends ArrayAdapter { } } + private void displayInfoMessage(ViewHolder viewHolder, CharSequence text, boolean darkBackground, final Message message, int type) { + displayDownloadableMessage(viewHolder, message, "", darkBackground, type); + int imageVisibility = viewHolder.image.getVisibility(); + displayInfoMessage(viewHolder, text, darkBackground); + viewHolder.image.setVisibility(imageVisibility); + } + private void displayInfoMessage(ViewHolder viewHolder, CharSequence text, boolean darkBackground) { viewHolder.download_button.setVisibility(View.GONE); viewHolder.audioPlayer.setVisibility(View.GONE); @@ -913,7 +920,7 @@ public class MessageAdapter extends ArrayAdapter { } else if (transferable != null && transferable.getStatus() == Transferable.STATUS_OFFER_CHECK_FILESIZE) { displayDownloadableMessage(viewHolder, message, activity.getString(R.string.check_x_filesize, UIHelper.getFileDescriptionString(activity, message)), darkBackground, type); } else { - displayInfoMessage(viewHolder, UIHelper.getMessagePreview(activity, message).first, darkBackground); + displayInfoMessage(viewHolder, UIHelper.getMessagePreview(activity, message).first, darkBackground, message, type); } } else if (message.isFileOrImage() && message.getEncryption() != Message.ENCRYPTION_PGP && message.getEncryption() != Message.ENCRYPTION_DECRYPTION_FAILED) { if (message.getFileParams().width > 0 && message.getFileParams().height > 0) { From 0bcbf0dcc166c9980d1a834cff52c9f9cfb1de4f Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Tue, 13 Dec 2022 22:57:17 -0500 Subject: [PATCH 07/13] Support data-uri thumbnails --- .../persistance/FileBackend.java | 43 ++++++++++++++++++- .../ui/adapter/MessageAdapter.java | 2 +- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/main/java/eu/siacs/conversations/persistance/FileBackend.java b/src/main/java/eu/siacs/conversations/persistance/FileBackend.java index 7dd89d718bfe58c37760affb3c53678df12d3aa5..3063b6b033581be7995bc5d593182e788fb7ae3f 100644 --- a/src/main/java/eu/siacs/conversations/persistance/FileBackend.java +++ b/src/main/java/eu/siacs/conversations/persistance/FileBackend.java @@ -55,11 +55,13 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.ServerSocket; import java.net.Socket; +import java.nio.ByteBuffer; import java.security.DigestOutputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Locale; @@ -1039,7 +1041,46 @@ public class FileBackend { } public Drawable getThumbnail(Message message, Resources res, int size, boolean cacheOnly) throws IOException { - return getThumbnail(getFile(message), res, size, cacheOnly); + final LruCache cache = mXmppConnectionService.getDrawableCache(); + DownloadableFile file = getFile(message); + Drawable thumbnail = cache.get(file.getAbsolutePath()); + if (thumbnail != null) return thumbnail; + + if ((thumbnail == null) && (!cacheOnly)) { + synchronized (THUMBNAIL_LOCK) { + List thumbs = message.getFileParams() != null ? message.getFileParams().getThumbnails() : null; + if (thumbs != null && !thumbs.isEmpty()) { + for (Element thumb : thumbs) { + Uri uri = Uri.parse(thumb.getAttribute("uri")); + if (uri.getScheme().equals("data")) { + if (android.os.Build.VERSION.SDK_INT < 28) continue; + String[] parts = uri.getSchemeSpecificPart().split(",", 2); + byte[] data; + if (Arrays.asList(parts[0].split(";")).contains("base64")) { + data = Base64.decode(parts[1], 0); + } else { + data = parts[1].getBytes("UTF-8"); + } + + ImageDecoder.Source source = ImageDecoder.createSource(ByteBuffer.wrap(data)); + thumbnail = ImageDecoder.decodeDrawable(source, (decoder, info, src) -> { + int w = info.getSize().getWidth(); + int h = info.getSize().getHeight(); + Rect r = rectForSize(w, h, size); + decoder.setTargetSize(r.width(), r.height()); + }); + + if (thumbnail != null) { + cache.put(file.getAbsolutePath(), thumbnail); + return thumbnail; + } + } + } + } + } + } + + return getThumbnail(file, res, size, cacheOnly); } public Drawable getThumbnail(DownloadableFile file, Resources res, int size, boolean cacheOnly) throws IOException { diff --git a/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java b/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java index ebe612059c228fcef8d61115fef7570236d531ad..54f2c7ad07e2431814760f8b0c1db59dc3156709 100644 --- a/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java +++ b/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java @@ -600,7 +600,7 @@ public class MessageAdapter extends ArrayAdapter { if (uri.getScheme().equals("data")) { String[] parts = uri.getSchemeSpecificPart().split(",", 2); parts = parts[0].split(";"); - if (!parts[0].equals("image/blurhash")) continue; + if (!parts[0].equals("image/blurhash") && !parts[0].equals("image/jpeg") && !parts[0].equals("image/png") && !parts[0].equals("image/webp") && !parts[0].equals("image/gif")) continue; } else { continue; } From 8e30c8181093748d9afcb4246c78bc9563e1f06c Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Tue, 13 Dec 2022 23:21:12 -0500 Subject: [PATCH 08/13] Support android Uri as well as java URI --- src/cheogram/java/com/cheogram/android/BobTransfer.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cheogram/java/com/cheogram/android/BobTransfer.java b/src/cheogram/java/com/cheogram/android/BobTransfer.java index 11ef425c9e860907fd59ddb5f60fcb0dfcb5475c..5ce721c5e9ad41a34803f66ae988a68417d28075 100644 --- a/src/cheogram/java/com/cheogram/android/BobTransfer.java +++ b/src/cheogram/java/com/cheogram/android/BobTransfer.java @@ -1,5 +1,6 @@ package com.cheogram.android; +import android.net.Uri; import android.util.Base64; import android.util.Log; @@ -35,6 +36,11 @@ public class BobTransfer implements Transferable { protected Jid to; protected XmppConnectionService xmppConnectionService; + public static Cid cid(Uri uri) { + if (!uri.getScheme().equals("cid")) return null; + return cid(uri.getSchemeSpecificPart()); + } + public static Cid cid(URI uri) { if (!uri.getScheme().equals("cid")) return null; return cid(uri.getSchemeSpecificPart()); From 1eb5b6bd69c323eaba03bca20a66f7bab3374cd4 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Tue, 13 Dec 2022 23:21:55 -0500 Subject: [PATCH 09/13] Support BoB thumbnails --- .../siacs/conversations/persistance/FileBackend.java | 9 +++++++++ .../conversations/ui/adapter/MessageAdapter.java | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/main/java/eu/siacs/conversations/persistance/FileBackend.java b/src/main/java/eu/siacs/conversations/persistance/FileBackend.java index 3063b6b033581be7995bc5d593182e788fb7ae3f..abba72fa86f3536e13e164ddf20f7573e6f15ed0 100644 --- a/src/main/java/eu/siacs/conversations/persistance/FileBackend.java +++ b/src/main/java/eu/siacs/conversations/persistance/FileBackend.java @@ -37,6 +37,8 @@ import androidx.annotation.StringRes; import androidx.core.content.FileProvider; import androidx.exifinterface.media.ExifInterface; +import com.cheogram.android.BobTransfer; + import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.io.ByteStreams; @@ -1074,6 +1076,13 @@ public class FileBackend { cache.put(file.getAbsolutePath(), thumbnail); return thumbnail; } + } else if (uri.getScheme().equals("cid")) { + Cid cid = BobTransfer.cid(uri); + if (cid == null) continue; + DownloadableFile f = mXmppConnectionService.getFileForCid(cid); + if (f != null && f.canRead()) { + return getThumbnail(f, res, size, cacheOnly); + } } } } diff --git a/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java b/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java index 54f2c7ad07e2431814760f8b0c1db59dc3156709..7194f5c80ec1f7a890a454bd0fcfbc02ffde8a00 100644 --- a/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java +++ b/src/main/java/eu/siacs/conversations/ui/adapter/MessageAdapter.java @@ -601,6 +601,18 @@ public class MessageAdapter extends ArrayAdapter { String[] parts = uri.getSchemeSpecificPart().split(",", 2); parts = parts[0].split(";"); if (!parts[0].equals("image/blurhash") && !parts[0].equals("image/jpeg") && !parts[0].equals("image/png") && !parts[0].equals("image/webp") && !parts[0].equals("image/gif")) continue; + } else if (uri.getScheme().equals("cid")) { + Cid cid = BobTransfer.cid(uri); + if (cid == null) continue; + DownloadableFile f = activity.xmppConnectionService.getFileForCid(cid); + if (f == null || !f.canRead()) { + if (!message.trusted() && !message.getConversation().canInferPresence()) continue; + + try { + new BobTransfer(BobTransfer.uri(cid), message.getConversation().getAccount(), message.getCounterpart(), activity.xmppConnectionService).start(); + } catch (final NoSuchAlgorithmException | URISyntaxException e) { } + continue; + } } else { continue; } From 7a1ee8049bd9b380138c77fd12b1ff0ef6ea7777 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Sat, 17 Dec 2022 22:29:21 -0500 Subject: [PATCH 10/13] Don't just start from blank if there are some FileParams we can update --- .../java/eu/siacs/conversations/persistance/FileBackend.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/eu/siacs/conversations/persistance/FileBackend.java b/src/main/java/eu/siacs/conversations/persistance/FileBackend.java index abba72fa86f3536e13e164ddf20f7573e6f15ed0..5378d2da8c496c4c6ab549e13dcda7fcb57911b0 100644 --- a/src/main/java/eu/siacs/conversations/persistance/FileBackend.java +++ b/src/main/java/eu/siacs/conversations/persistance/FileBackend.java @@ -1693,7 +1693,8 @@ public class FileBackend { final boolean image = message.getType() == Message.TYPE_IMAGE || (mime != null && mime.startsWith("image/")); - Message.FileParams fileParams = new Message.FileParams(); + Message.FileParams fileParams = message.getFileParams(); + if (fileParams == null) fileParams = new Message.FileParams(); if (url != null) { fileParams.url = url; } From 115576a96fc696f2084ec7a9f5819f9d13a3b999 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Sat, 17 Dec 2022 22:30:15 -0500 Subject: [PATCH 11/13] Helper to get file element, where thumnails etc live --- .../eu/siacs/conversations/entities/Message.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/java/eu/siacs/conversations/entities/Message.java b/src/main/java/eu/siacs/conversations/entities/Message.java index 890523c854e559f5815d8e5550113ad6def7354d..f94401268795f300b86c1c064963ec170f0e0851 100644 --- a/src/main/java/eu/siacs/conversations/entities/Message.java +++ b/src/main/java/eu/siacs/conversations/entities/Message.java @@ -1179,15 +1179,21 @@ public class Message extends AbstractEntity implements AvatarService.Avatarable return sims; } - public List getThumbnails() { - List thumbs = new ArrayList<>(); - if (sims == null) return thumbs; + protected Element getFileElement() { + Element file = null; + if (sims == null) return file; Element mediaSharing = sims.findChild("media-sharing", "urn:xmpp:sims:1"); - if (mediaSharing == null) return thumbs; - Element file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:5"); + if (mediaSharing == null) return file; + file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:5"); if (file == null) file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:4"); if (file == null) file = mediaSharing.findChild("file", "urn:xmpp:jingle:apps:file-transfer:3"); + return file; + } + + public List getThumbnails() { + List thumbs = new ArrayList<>(); + Element file = getFileElement(); if (file == null) return thumbs; for (Element child : file.getChildren()) { From 1a6ee8eae43cdf2da6553fa52ad1573e056f27eb Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Sat, 17 Dec 2022 22:30:36 -0500 Subject: [PATCH 12/13] Helper to get Cids for the elements --- .../siacs/conversations/entities/Message.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/eu/siacs/conversations/entities/Message.java b/src/main/java/eu/siacs/conversations/entities/Message.java index f94401268795f300b86c1c064963ec170f0e0851..04175632ef46035b175a40da6cf09727de0b2c14 100644 --- a/src/main/java/eu/siacs/conversations/entities/Message.java +++ b/src/main/java/eu/siacs/conversations/entities/Message.java @@ -7,6 +7,7 @@ import android.graphics.Color; import android.os.Build; import android.text.Html; import android.text.SpannableStringBuilder; +import android.util.Base64; import android.util.Log; import com.cheogram.android.BobTransfer; @@ -24,6 +25,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.time.Duration; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -1191,6 +1193,24 @@ public class Message extends AbstractEntity implements AvatarService.Avatarable return file; } + public List getCids() { + List cids = new ArrayList<>(); + Element file = getFileElement(); + if (file == null) return cids; + + for (Element child : file.getChildren()) { + if (child.getName().equals("hash") && child.getNamespace().equals("urn:xmpp:hashes:2")) { + try { + cids.add(CryptoHelper.cid(Base64.decode(child.getContent(), Base64.DEFAULT), child.getAttribute("algo"))); + } catch (final NoSuchAlgorithmException | IllegalStateException e) { } + } + } + + cids.sort((x, y) -> y.getType().compareTo(x.getType())); + + return cids; + } + public List getThumbnails() { List thumbs = new ArrayList<>(); Element file = getFileElement(); From 86dbeaa892ec4111e3347429f97647c8c352d5b9 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Sat, 17 Dec 2022 22:30:56 -0500 Subject: [PATCH 13/13] If we already have a file by this hash, let's just use it Was already the case for BobTransfer, but here we do it before looking to see what kind of transfers are supported. Even if there are no sources at all, if we have this has already it'll work. --- .../conversations/parser/MessageParser.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/eu/siacs/conversations/parser/MessageParser.java b/src/main/java/eu/siacs/conversations/parser/MessageParser.java index f4030e437d4fa0136838326340ea97872f3c700c..6ccfe69b4cde5c2b07cb84656c6b2bd5354a7a17 100644 --- a/src/main/java/eu/siacs/conversations/parser/MessageParser.java +++ b/src/main/java/eu/siacs/conversations/parser/MessageParser.java @@ -5,6 +5,7 @@ import android.util.Pair; import com.cheogram.android.BobTransfer; +import java.io.File; import java.net.URISyntaxException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -18,6 +19,8 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import io.ipfs.cid.Cid; + import eu.siacs.conversations.Config; import eu.siacs.conversations.R; import eu.siacs.conversations.crypto.axolotl.AxolotlService; @@ -775,9 +778,21 @@ public class MessageParser extends AbstractParser implements OnMessagePacketRece processMessageReceipts(account, packet, remoteMsgId, query); } + if (message.getFileParams() != null) { + for (Cid cid : message.getFileParams().getCids()) { + File f = mXmppConnectionService.getFileForCid(cid); + if (f != null && f.canRead()) { + message.setRelativeFilePath(f.getAbsolutePath()); + mXmppConnectionService.getFileBackend().updateFileParams(message, null, false); + break; + } + } + } + mXmppConnectionService.databaseBackend.createMessage(message); + final HttpConnectionManager manager = this.mXmppConnectionService.getHttpConnectionManager(); - if (message.trusted() && message.treatAsDownloadable() && manager.getAutoAcceptFileSize() > 0) { + if (message.getRelativeFilePath() == null && message.trusted() && message.treatAsDownloadable() && manager.getAutoAcceptFileSize() > 0) { if (message.getOob() != null && message.getOob().getScheme().equalsIgnoreCase("cid")) { try { BobTransfer transfer = new BobTransfer.ForMessage(message, mXmppConnectionService);