From e822c0a7341c42263e1b2d7b8ce87ddb0af44566 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Wed, 28 Sep 2022 15:34:43 -0500 Subject: [PATCH] Content Addressable Media Storage Downloads are stored at a filename that is the CID computed from a SHA256 hash of their contents. Where necessary (such as to stream a large file down) they are stored at the name based on the message UUID temporarily and then renamed. Uploads, when they must be copied into our storage, are stored in the same way. This results in de-duplicated storage, so sending the same file back and forth many times even in different conversations results in only one copy being stored. Since Android does not play nice with symlinks, but we want to be able to look up media by multiple possible hashes, the CIDs for SHA1, SHA256, and SHA512 are all stored in a new database table whenever a file is downloaded or uploaded, even if the file is never copied into our storage at all (so then the db table contains a path to, say, the image in the camera folder). When bits-of-binary cid transfer is attempted, first check if we have that content locally already and just use it if we already have it instead of requesting it again. --- build.gradle | 2 + .../com/cheogram/android/BobTransfer.java | 65 ++++++++++++------- .../http/HttpDownloadConnection.java | 13 +++- .../persistance/DatabaseBackend.java | 31 +++++++++ .../persistance/FileBackend.java | 60 ++++++++++++++++- .../services/XmppConnectionService.java | 10 +++ .../conversations/utils/CryptoHelper.java | 42 ++++++++++++ .../jingle/JingleFileTransferConnection.java | 23 +++++-- 8 files changed, 213 insertions(+), 33 deletions(-) diff --git a/build.gradle b/build.gradle index 5e429cb647041020fa39acf74c679ff4f25cfe12..a89af52f7c680c46218b69b2efa48dce9d7fe680 100644 --- a/build.gradle +++ b/build.gradle @@ -20,6 +20,7 @@ repositories { google() mavenCentral() jcenter() + maven { url 'https://jitpack.io' } } // https://stackoverflow.com/a/38105112/8611 @@ -103,6 +104,7 @@ dependencies { implementation 'io.michaelrocks:libphonenumber-android:8.12.49' implementation 'io.github.nishkarsh:android-permissions:2.1.6' implementation 'androidx.recyclerview:recyclerview:1.1.0' + implementation 'com.github.ipld:java-cid:v1.3.1' implementation urlFile('https://gateway.pinata.cloud/ipfs/QmeqMiLxHi8AAjXobxr3QTfa1bSSLyAu86YviAqQnjxCjM/libwebrtc.aar', 'libwebrtc.aar') // INSERT } diff --git a/src/cheogram/java/com/cheogram/android/BobTransfer.java b/src/cheogram/java/com/cheogram/android/BobTransfer.java index dc8aefc0c4c44a42bd7d31e7c73fa4be17204806..1099b143081ea1f5dfa7c6a58e1c3d5fbe4ed60a 100644 --- a/src/cheogram/java/com/cheogram/android/BobTransfer.java +++ b/src/cheogram/java/com/cheogram/android/BobTransfer.java @@ -3,10 +3,15 @@ package com.cheogram.android; import android.util.Base64; import android.util.Log; +import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; +import java.security.NoSuchAlgorithmException; + +import io.ipfs.cid.Cid; import eu.siacs.conversations.Config; import eu.siacs.conversations.R; @@ -26,29 +31,36 @@ public class BobTransfer implements Transferable { protected Message message; protected URI uri; protected XmppConnectionService xmppConnectionService; - protected DownloadableFile file; + + public static Cid cid(URI uri) { + String bobCid = uri.getSchemeSpecificPart(); + if (!bobCid.contains("@") || !bobCid.contains("+")) return null; + String[] cidParts = bobCid.split("@")[0].split("\\+"); + try { + return CryptoHelper.cid(CryptoHelper.hexToBytes(cidParts[1]), cidParts[0]); + } catch (final NoSuchAlgorithmException e) { + return null; + } + } public BobTransfer(Message message, XmppConnectionService xmppConnectionService) throws URISyntaxException { this.message = message; this.xmppConnectionService = xmppConnectionService; this.uri = new URI(message.getFileParams().url); - setupFile(); - } - - private void setupFile() { - final String reference = uri.getFragment(); - if (reference != null && AesGcmURL.IV_KEY.matcher(reference).matches()) { - this.file = new DownloadableFile(xmppConnectionService.getCacheDir(), message.getUuid()); - this.file.setKeyAndIv(CryptoHelper.hexToBytes(reference)); - Log.d(Config.LOGTAG, "create temporary OMEMO encrypted file: " + this.file.getAbsolutePath() + "(" + message.getMimeType() + ")"); - } else { - this.file = xmppConnectionService.getFileBackend().getFile(message, false); - } } @Override public boolean start() { if (status == Transferable.STATUS_DOWNLOADING) return true; + File f = xmppConnectionService.getFileForCid(cid(uri)); + + if (f != null && f.canRead()) { + message.setRelativeFilePath(f.getAbsolutePath()); + finish(); + message.setTransferable(null); + xmppConnectionService.updateConversationUi(); + return true; + } if (xmppConnectionService.hasInternetConnection()) { changeStatus(Transferable.STATUS_DOWNLOADING); @@ -64,29 +76,27 @@ public class BobTransfer implements Transferable { xmppConnectionService.showErrorToastInUi(R.string.download_failed_file_not_found); } else { final String contentType = data.getAttribute("type"); + String fileExtension = "dat"; if (contentType != null) { - final String fileExtension = MimeUtils.guessExtensionFromMimeType(contentType); - if (fileExtension != null) { - xmppConnectionService.getFileBackend().setupRelativeFilePath(message, String.format("%s.%s", message.getUuid(), fileExtension), contentType); - Log.d(Config.LOGTAG, "rewriting name for bob based on content type"); - setupFile(); - } + fileExtension = MimeUtils.guessExtensionFromMimeType(contentType); } try { + final byte[] bytes = Base64.decode(data.getContent(), Base64.DEFAULT); + + xmppConnectionService.getFileBackend().setupRelativeFilePath(message, new ByteArrayInputStream(bytes), fileExtension); + DownloadableFile file = xmppConnectionService.getFileBackend().getFile(message); file.getParentFile().mkdirs(); if (!file.exists() && !file.createNewFile()) { throw new IOException(file.getAbsolutePath()); } + final OutputStream outputStream = AbstractConnectionManager.createOutputStream(file, false, false); - outputStream.write(Base64.decode(data.getContent(), Base64.DEFAULT)); + outputStream.write(bytes); outputStream.flush(); outputStream.close(); - final boolean privateMessage = message.isPrivateMessage(); - message.setType(privateMessage ? Message.TYPE_PRIVATE_FILE : Message.TYPE_FILE); - xmppConnectionService.getFileBackend().updateFileParams(message, uri.toString()); - xmppConnectionService.updateMessage(message); + finish(); } catch (IOException e) { xmppConnectionService.showErrorToastInUi(R.string.download_failed_could_not_write_file); } @@ -126,4 +136,11 @@ public class BobTransfer implements Transferable { status = newStatus; xmppConnectionService.updateConversationUi(); } + + protected void finish() { + final boolean privateMessage = message.isPrivateMessage(); + message.setType(privateMessage ? Message.TYPE_PRIVATE_FILE : Message.TYPE_FILE); + xmppConnectionService.getFileBackend().updateFileParams(message, uri.toString(), false); + xmppConnectionService.updateMessage(message); + } } diff --git a/src/main/java/eu/siacs/conversations/http/HttpDownloadConnection.java b/src/main/java/eu/siacs/conversations/http/HttpDownloadConnection.java index d0c6583683aecc8abaa86da620be12093ebfa2cc..13d10a7a1d921e89e4111ad42c334ef282ad5df8 100644 --- a/src/main/java/eu/siacs/conversations/http/HttpDownloadConnection.java +++ b/src/main/java/eu/siacs/conversations/http/HttpDownloadConnection.java @@ -192,9 +192,18 @@ public class HttpDownloadConnection implements Transferable { if (message.getEncryption() == Message.ENCRYPTION_PGP) { notify = message.getConversation().getAccount().getPgpDecryptionService().decrypt(message, notify); } - mHttpConnectionManager.updateConversationUi(true); + DownloadableFile file; + final DownloadableFile tmp = mXmppConnectionService.getFileBackend().getFile(message); + final String extension = MimeUtils.extractRelevantExtension(tmp.getName()); + try { + mXmppConnectionService.getFileBackend().setupRelativeFilePath(message, new FileInputStream(tmp), extension); + file = mXmppConnectionService.getFileBackend().getFile(message); + tmp.renameTo(file); + } catch (final IOException e) { + file = tmp; + } + mXmppConnectionService.updateMessage(message); final boolean notifyAfterScan = notify; - final DownloadableFile file = mXmppConnectionService.getFileBackend().getFile(message, true); mXmppConnectionService.getFileBackend().updateMediaScanner(file, () -> { if (notifyAfterScan) { mXmppConnectionService.getNotificationService().push(message); diff --git a/src/main/java/eu/siacs/conversations/persistance/DatabaseBackend.java b/src/main/java/eu/siacs/conversations/persistance/DatabaseBackend.java index 9c9f9e086c4d7481d10bce4289e1d7307944beda..bf69e87613035244c94d62fdf3c827fa7690d924 100644 --- a/src/main/java/eu/siacs/conversations/persistance/DatabaseBackend.java +++ b/src/main/java/eu/siacs/conversations/persistance/DatabaseBackend.java @@ -39,6 +39,8 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CopyOnWriteArrayList; +import io.ipfs.cid.Cid; + import eu.siacs.conversations.Config; import eu.siacs.conversations.crypto.axolotl.AxolotlService; import eu.siacs.conversations.crypto.axolotl.FingerprintStatus; @@ -252,6 +254,16 @@ public class DatabaseBackend extends SQLiteOpenHelper { db.execSQL("PRAGMA cheogram.user_version = 3"); } + if(cheogramVersion < 4) { + db.execSQL( + "CREATE TABLE cheogram.cids (" + + "cid TEXT NOT NULL PRIMARY KEY," + + "path TEXT NOT NULL" + + ")" + ); + db.execSQL("PRAGMA cheogram.user_version = 4"); + } + db.setTransactionSuccessful(); } finally { db.endTransaction(); @@ -721,6 +733,25 @@ public class DatabaseBackend extends SQLiteOpenHelper { cursor.close(); } + public File getFileForCid(Cid cid) { + SQLiteDatabase db = this.getReadableDatabase(); + Cursor cursor = db.query("cheogram.cids", new String[]{"path"}, "cid=?", new String[]{cid.toString()}, null, null, null); + File f = null; + if (cursor.moveToNext()) { + f = new File(cursor.getString(0)); + } + cursor.close(); + return f; + } + + public void saveCid(Cid cid, File file) { + SQLiteDatabase db = this.getWritableDatabase(); + ContentValues cv = new ContentValues(); + cv.put("cid", cid.toString()); + cv.put("path", file.getAbsolutePath()); + db.insertWithOnConflict("cheogram.cids", null, cv, SQLiteDatabase.CONFLICT_REPLACE); + } + public void createConversation(Conversation conversation) { SQLiteDatabase db = this.getWritableDatabase(); db.insert(Conversation.TABLENAME, null, conversation.getContentValues()); diff --git a/src/main/java/eu/siacs/conversations/persistance/FileBackend.java b/src/main/java/eu/siacs/conversations/persistance/FileBackend.java index 73c0b83510c31c77339582bfbd70ce146602999b..c2edaa3a15da9b4155e162762c37283e99f8402d 100644 --- a/src/main/java/eu/siacs/conversations/persistance/FileBackend.java +++ b/src/main/java/eu/siacs/conversations/persistance/FileBackend.java @@ -62,6 +62,8 @@ import java.util.List; import java.util.Locale; import java.util.UUID; +import io.ipfs.cid.Cid; + import eu.siacs.conversations.Config; import eu.siacs.conversations.R; import eu.siacs.conversations.entities.DownloadableFile; @@ -712,7 +714,8 @@ public class FileBackend { if ("ogg".equals(extension) && type != null && type.startsWith("audio/")) { extension = "oga"; } - setupRelativeFilePath(message, String.format("%s.%s", message.getUuid(), extension)); + + setupRelativeFilePath(message, uri, extension); copyFileToPrivateStorage(mXmppConnectionService.getFileBackend().getFile(message), uri); } @@ -850,8 +853,46 @@ public class FileBackend { throw new IllegalStateException("Unknown image format"); } setupRelativeFilePath(message, filename); - copyImageToPrivateStorage(getFile(message), image); - updateFileParams(message); + final File tmp = getFile(message); + copyImageToPrivateStorage(tmp, image); + final String extension = MimeUtils.extractRelevantExtension(filename); + try { + setupRelativeFilePath(message, new FileInputStream(tmp), extension); + } catch (final FileNotFoundException e) { + throw new FileCopyException(R.string.error_file_not_found); + } catch (final IOException e) { + throw new FileCopyException(R.string.error_io_exception); + } + tmp.renameTo(getFile(message)); + updateFileParams(message, null, false); + } + + public void setupRelativeFilePath(final Message message, final Uri uri, final String extension) throws FileCopyException { + try { + setupRelativeFilePath(message, mXmppConnectionService.getContentResolver().openInputStream(uri), extension); + } catch (final FileNotFoundException e) { + throw new FileCopyException(R.string.error_file_not_found); + } catch (final IOException e) { + throw new FileCopyException(R.string.error_io_exception); + } + } + + public Cid[] calculateCids(final InputStream is) throws IOException { + try { + return CryptoHelper.cid(is, new String[]{"SHA-256", "SHA-1", "SHA-512"}); + } catch (final NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } + + public void setupRelativeFilePath(final Message message, final InputStream is, final String extension) throws IOException { + Cid[] cids = calculateCids(is); + + setupRelativeFilePath(message, String.format("%s.%s", cids[0], extension)); + File file = getFile(message); + for (int i = 0; i < cids.length; i++) { + mXmppConnectionService.saveCid(cids[i], file); + } } public void setupRelativeFilePath(final Message message, final String filename) { @@ -1533,6 +1574,10 @@ public class FileBackend { } public void updateFileParams(final Message message, final String url) { + updateFileParams(message, url, true); + } + + public void updateFileParams(final Message message, final String url, boolean updateCids) { final boolean encrypted = message.getEncryption() == Message.ENCRYPTION_PGP || message.getEncryption() == Message.ENCRYPTION_DECRYPTED; @@ -1602,6 +1647,15 @@ public class FileBackend { privateMessage ? Message.TYPE_PRIVATE_FILE : (image ? Message.TYPE_IMAGE : Message.TYPE_FILE)); + + if (updateCids) { + try { + Cid[] cids = calculateCids(new FileInputStream(getFile(message))); + for (int i = 0; i < cids.length; i++) { + mXmppConnectionService.saveCid(cids[i], file); + } + } catch (final IOException e) { } + } } private int getMediaRuntime(final File file) { diff --git a/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java b/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java index b2aa59129d238cbfdf84492d49b47e14eee20374..802a834d3b6865aac6b1bc8c6c0ca306678b15d8 100644 --- a/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java +++ b/src/main/java/eu/siacs/conversations/services/XmppConnectionService.java @@ -86,6 +86,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import io.ipfs.cid.Cid; + import eu.siacs.conversations.Config; import eu.siacs.conversations.R; import eu.siacs.conversations.android.JabberIdContact; @@ -547,6 +549,14 @@ public class XmppConnectionService extends Service { return this.fileBackend; } + public File getFileForCid(Cid cid) { + return this.databaseBackend.getFileForCid(cid); + } + + public void saveCid(Cid cid, File file) { + this.databaseBackend.saveCid(cid, file); + } + public AvatarService getAvatarService() { return this.mAvatarService; } diff --git a/src/main/java/eu/siacs/conversations/utils/CryptoHelper.java b/src/main/java/eu/siacs/conversations/utils/CryptoHelper.java index a92d48825e70cf48b7c135b8e122c7c46d3e3070..c2d47ebe3c8ddbbe0092eaf285b56fa8676e3bdd 100644 --- a/src/main/java/eu/siacs/conversations/utils/CryptoHelper.java +++ b/src/main/java/eu/siacs/conversations/utils/CryptoHelper.java @@ -9,6 +9,8 @@ import org.bouncycastle.asn1.x500.style.BCStyle; import org.bouncycastle.asn1.x500.style.IETFUtils; import org.bouncycastle.cert.jcajce.JcaX509CertificateHolder; +import java.io.InputStream; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -25,6 +27,9 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.regex.Pattern; +import io.ipfs.cid.Cid; +import io.ipfs.multihash.Multihash; + import eu.siacs.conversations.Config; import eu.siacs.conversations.R; import eu.siacs.conversations.entities.Account; @@ -282,4 +287,41 @@ public final class CryptoHelper { final String u = url.toLowerCase(); return !u.contains(" ") && (u.startsWith("https://") || u.startsWith("http://") || u.startsWith("p1s3://")) && u.endsWith(".pgp"); } + + public static Multihash.Type multihashType(String algo) throws NoSuchAlgorithmException { + if (algo.equals("SHA-1") || algo.equals("sha-1") || algo.equals("sha1")) { + return Multihash.Type.sha1; + } else if (algo.equals("SHA-256") || algo.equals("sha-256")) { + return Multihash.Type.sha2_256; + } else if (algo.equals("SHA-512") | algo.equals("sha-512")) { + return Multihash.Type.sha2_512; + } else { + throw new NoSuchAlgorithmException(algo); + } + } + + public static Cid cid(byte[] digest, String algo) throws NoSuchAlgorithmException { + return Cid.buildCidV1(Cid.Codec.Raw, multihashType(algo), digest); + } + + public static Cid[] cid(InputStream in, String[] algo) throws NoSuchAlgorithmException, IOException { + byte[] buf = new byte[4096]; + int len; + MessageDigest[] md = new MessageDigest[algo.length]; + for (int i = 0; i < md.length; i++) { + md[i] = MessageDigest.getInstance(algo[i]); + } + while ((len = in.read(buf)) != -1) { + for (int i = 0; i < md.length; i++) { + md[i].update(buf, 0, len); + } + } + + Cid[] cid = new Cid[md.length]; + for (int i = 0; i < cid.length; i++) { + cid[i] = cid(md[i].digest(), algo[i]); + } + + return cid; + } } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleFileTransferConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleFileTransferConnection.java index 2411170c667727aad291310be1e3ab8abd759dd2..f14cb07c3cace5ae1ad5e102f71ac73fe8ee90d1 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleFileTransferConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleFileTransferConnection.java @@ -39,6 +39,7 @@ import eu.siacs.conversations.parser.IqParser; import eu.siacs.conversations.persistance.FileBackend; import eu.siacs.conversations.services.AbstractConnectionManager; import eu.siacs.conversations.utils.CryptoHelper; +import eu.siacs.conversations.utils.MimeUtils; import eu.siacs.conversations.xml.Element; import eu.siacs.conversations.xml.Namespace; import eu.siacs.conversations.xmpp.Jid; @@ -114,6 +115,8 @@ public class JingleFileTransferConnection extends AbstractJingleConnection imple @Override public void onFileTransmitted(DownloadableFile file) { + DownloadableFile finalFile; + if (responding()) { if (expectedHash.length > 0) { if (Arrays.equals(expectedHash, file.getSha1Sum())) { @@ -125,7 +128,17 @@ public class JingleFileTransferConnection extends AbstractJingleConnection imple Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": other party did not include file hash in file transfer"); } sendSuccess(); - xmppConnectionService.getFileBackend().updateFileParams(message); + + final String extension = MimeUtils.extractRelevantExtension(file.getName()); + try { + xmppConnectionService.getFileBackend().setupRelativeFilePath(message, new FileInputStream(file), extension); + finalFile = xmppConnectionService.getFileBackend().getFile(message); + file.renameTo(finalFile); + } catch (final IOException e) { + finalFile = file; + } + + xmppConnectionService.getFileBackend().updateFileParams(message, null, false); xmppConnectionService.databaseBackend.createMessage(message); xmppConnectionService.markMessage(message, Message.STATUS_RECEIVED); if (acceptedAutomatically) { @@ -142,6 +155,8 @@ public class JingleFileTransferConnection extends AbstractJingleConnection imple id.account.getPgpDecryptionService().decrypt(message, true); } } else { + finalFile = file; + if (description.getVersion() == FileTransferDescription.Version.FT_5) { //older Conversations will break when receiving a session-info sendHash(); } @@ -149,13 +164,13 @@ public class JingleFileTransferConnection extends AbstractJingleConnection imple id.account.getPgpDecryptionService().decrypt(message, false); } if (message.getEncryption() == Message.ENCRYPTION_PGP || message.getEncryption() == Message.ENCRYPTION_DECRYPTED) { - file.delete(); + finalFile.delete(); } disconnectSocks5Connections(); } - Log.d(Config.LOGTAG, "successfully transmitted file:" + file.getAbsolutePath() + " (" + CryptoHelper.bytesToHex(file.getSha1Sum()) + ")"); + Log.d(Config.LOGTAG, "successfully transmitted file:" + finalFile.getAbsolutePath() + " (" + CryptoHelper.bytesToHex(file.getSha1Sum()) + ")"); if (message.getEncryption() != Message.ENCRYPTION_PGP) { - xmppConnectionService.getFileBackend().updateMediaScanner(file); + xmppConnectionService.getFileBackend().updateMediaScanner(finalFile); } }