diff --git a/src/main/java/eu/siacs/conversations/xmpp/manager/AvatarManager.java b/src/main/java/eu/siacs/conversations/xmpp/manager/AvatarManager.java
index 73db1a9abff60c83fd4c0d4dbd9abd1ddb78cd44..604933d06d6b68cd930398b2d38851f39cc49490 100644
--- a/src/main/java/eu/siacs/conversations/xmpp/manager/AvatarManager.java
+++ b/src/main/java/eu/siacs/conversations/xmpp/manager/AvatarManager.java
@@ -842,7 +842,8 @@ public class AvatarManager extends AbstractManager {
     }
 
     public ListenableFuture<Void> fetchAndStoreVCard(final Jid address, final String expectedHash) {
-        final var future = connection.getManager(VCardManager.class).retrievePhoto(address);
+        final var future =
+                connection.getManager(VCardManager.class).retrievePhotoCacheException(address);
         return Futures.transformAsync(
                 future,
                 photo -> {
diff --git a/src/main/java/eu/siacs/conversations/xmpp/manager/VCardManager.java b/src/main/java/eu/siacs/conversations/xmpp/manager/VCardManager.java
index 6fbbbffe2393083220d7637bb3c4065b85ce8717..2d78d336029259bba51627d60137a153a3d169d4 100644
--- a/src/main/java/eu/siacs/conversations/xmpp/manager/VCardManager.java
+++ b/src/main/java/eu/siacs/conversations/xmpp/manager/VCardManager.java
@@ -2,6 +2,8 @@ package eu.siacs.conversations.xmpp.manager;
 
 import android.content.Context;
 import android.util.Log;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.MoreExecutors;
@@ -14,10 +16,17 @@ import im.conversations.android.xmpp.model.stanza.Iq;
 import im.conversations.android.xmpp.model.vcard.BinaryValue;
 import im.conversations.android.xmpp.model.vcard.Photo;
 import im.conversations.android.xmpp.model.vcard.VCard;
+import java.time.Duration;
 import java.util.Objects;
 
 public class VCardManager extends AbstractManager {
 
+    private final Cache<Jid, Exception> photoExceptionCache =
+            CacheBuilder.newBuilder()
+                    .maximumSize(24_576)
+                    .expireAfterWrite(Duration.ofHours(36))
+                    .build();
+
     public VCardManager(final Context context, final XmppConnection connection) {
         super(context, connection);
     }
@@ -37,10 +46,25 @@ public class VCardManager extends AbstractManager {
                 MoreExecutors.directExecutor());
     }
 
-    public ListenableFuture<byte[]> retrievePhoto(final Jid address) {
-
-        // TODO add a caching variant
+    public ListenableFuture<byte[]> retrievePhotoCacheException(final Jid address) {
+        final var existingException = this.photoExceptionCache.getIfPresent(address);
+        if (existingException != null) {
+            return Futures.immediateFailedFuture(existingException);
+        }
+        final var future = retrievePhoto(address);
+        return Futures.catchingAsync(
+                future,
+                Exception.class,
+                ex -> {
+                    if (ex instanceof IllegalStateException || ex instanceof IqErrorException) {
+                        photoExceptionCache.put(address, ex);
+                    }
+                    return Futures.immediateFailedFuture(ex);
+                },
+                MoreExecutors.directExecutor());
+    }
 
+    private ListenableFuture<byte[]> retrievePhoto(final Jid address) {
         final var vCardFuture = retrieve(address);
         return Futures.transform(
                 vCardFuture,