Experimental: Allow sending custom emotes ("emojis")

Using MSC2545 image packs TODO: - not use pills, or make them look differently here? - edits and drafts lose it note: upstream issue, same for user pills Change-Id: I27daf5835e32b818e512b61b57c09bea8c205e94
SchildiChat · Jun 16, 2022 · fd34eba · fd34eba
1 parent 7f9a3df
commit fd34eba
Show file tree

Hide file tree

Showing 14 changed files with 181 additions and 22 deletions.
diff --git a/...ix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/EventType.kt b/...ix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/EventType.kt
@@ -109,6 +109,9 @@ object EventType {
     val POLL_RESPONSE = listOf("org.matrix.msc3381.poll.response", "m.poll.response")
     val POLL_END = listOf("org.matrix.msc3381.poll.end", "m.poll.end")
 
+    // Emotes
+    const val ROOM_EMOTES = "im.ponies.room_emotes"
+
     // Unwedging
     internal const val DUMMY = "m.dummy"
 

diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/EmoteImage.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/EmoteImage.kt
@@ -0,0 +1,13 @@
+package org.matrix.android.sdk.api.session.room.model
+
+import com.squareup.moshi.Json
+import com.squareup.moshi.JsonClass
+import org.matrix.android.sdk.api.session.room.model.message.ImageInfo
+
+@JsonClass(generateAdapter = true)
+data class EmoteImage(
+        @Json(name = "url") val url: String,
+        @Json(name = "body") val body: String? = null,
+        @Json(name = "info") val info: ImageInfo? = null,
+        @Json(name = "usage") val usage: List<String>? = null,
+)
diff --git a/...k-android/src/main/java/org/matrix/android/sdk/api/session/room/model/RoomEmoteContent.kt b/...k-android/src/main/java/org/matrix/android/sdk/api/session/room/model/RoomEmoteContent.kt
@@ -0,0 +1,19 @@
+package org.matrix.android.sdk.api.session.room.model
+
+import com.squareup.moshi.Json
+import com.squareup.moshi.JsonClass
+import org.matrix.android.sdk.api.session.room.powerlevels.Role
+
+/**
+ * Class representing the EventType.ROOM_EMOTE state event content.
+ */
+@JsonClass(generateAdapter = true)
+data class RoomEmoteContent(
+        @Json(name = "images") val images: Map<String, EmoteImage>? = null,
+        // TODO: "pack" support
+) {
+    companion object {
+        const val USAGE_EMOTICON = "emoticon"
+        const val USAGE_STICKER = "sticker"
+    }
+}
diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/util/MatrixItem.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/util/MatrixItem.kt
@@ -112,6 +112,13 @@ sealed class MatrixItem(
         override fun updateAvatar(newAvatar: String?) = copy(avatarUrl = newAvatar)
     }
 
+    data class EmoteItem(override val id: String,
+                         override val displayName: String? = null,
+                         override val avatarUrl: String? = null) :
+            MatrixItem(id, displayName, avatarUrl) {
+        override fun updateAvatar(newAvatar: String?) = copy(avatarUrl = newAvatar)
+    }
+
     protected fun checkId() {
         if (!id.startsWith(getIdPrefix())) {
             error("Wrong usage of MatrixItem: check the id $id should start with ${getIdPrefix()}")
@@ -131,6 +138,7 @@ sealed class MatrixItem(
         is EveryoneInRoomItem -> '!'
         is RoomAliasItem      -> '#'
         is GroupItem          -> '+'
+        is EmoteItem          -> 'm'
     }
 
     fun firstLetterOfDisplayName(): String {

diff --git a/...android/src/main/java/org/matrix/android/sdk/internal/session/room/send/MarkdownParser.kt b/...android/src/main/java/org/matrix/android/sdk/internal/session/room/send/MarkdownParser.kt
@@ -63,7 +63,8 @@ internal class MarkdownParser @Inject constructor(
             htmlText
         }
 
-        return if (isFormattedTextPertinent(source, cleanHtmlText)) {
+        // SC-note: upstream checks with "source" instead of "text.toString()", but this breaks with stuff like "a :turtle:", where :turtle: is a custom emote
+        return if (isFormattedTextPertinent(text.toString(), cleanHtmlText)) {
             // According to https://matrix.org/docs/spec/client_server/latest#m-room-message-msgtypes:
             // The plain text version of the HTML should be provided in the body.
             // But it caused too many problems so it has been removed in #2002

diff --git a/...d/src/main/java/org/matrix/android/sdk/internal/session/room/send/pills/TextPillsUtils.kt b/...d/src/main/java/org/matrix/android/sdk/internal/session/room/send/pills/TextPillsUtils.kt
@@ -70,7 +70,15 @@ internal class TextPillsUtils @Inject constructor(
                 // append text before pill
                 append(text, currIndex, start)
                 // append the pill
-                append(String.format(template, urlSpan.matrixItem.id, displayNameResolver.getBestName(urlSpan.matrixItem)))
+                // Different handling necessary for custom emotes
+                if (urlSpan.matrixItem is MatrixItem.EmoteItem) {
+                    // Note we use the same template for both HTML and MARKDOWN conversion. We do this since markdown inline images are not mighty enough
+                    // for custom emotes (i.e., that would drop the data-mx-emoticon tag, which we want to keep). But we can use inline html in markdown.
+                    val imgHtml = "<img data-mx-emoticon height=\"18\" src=\"${urlSpan.matrixItem.avatarUrl}\" title=\":${urlSpan.matrixItem.displayName}:\" alt=\":${urlSpan.matrixItem.displayName}:\">"
+                    append(imgHtml)
+                } else {
+                    append(String.format(template, urlSpan.matrixItem.id, displayNameResolver.getBestName(urlSpan.matrixItem)))
+                }
                 currIndex = end
             }
             // append text after the last pill
@@ -111,3 +119,14 @@ internal class TextPillsUtils @Inject constructor(
         }
     }
 }
+
+fun CharSequence.requiresFormattedMessage(): Boolean {
+    val spannableString = SpannableString.valueOf(this)
+    val pills = spannableString
+            ?.getSpans(0, length, MatrixItemSpan::class.java)
+            ?.map { MentionLinkSpec(it, spannableString.getSpanStart(it), spannableString.getSpanEnd(it)) }
+            // We cannot send emotes without markdown/formatted messages
+            ?.filter { it.span.matrixItem is MatrixItem.EmoteItem }
+            ?: return false
+    return pills.isNotEmpty()
+}
diff --git a/...or/src/main/java/im/vector/app/features/autocomplete/emoji/AutocompleteEmojiController.kt b/...or/src/main/java/im/vector/app/features/autocomplete/emoji/AutocompleteEmojiController.kt
@@ -21,11 +21,15 @@ import androidx.recyclerview.widget.RecyclerView
 import com.airbnb.epoxy.TypedEpoxyController
 import im.vector.app.EmojiCompatFontProvider
 import im.vector.app.features.autocomplete.AutocompleteClickListener
+import im.vector.app.features.home.AvatarRenderer
 import im.vector.app.features.reactions.data.EmojiItem
+import org.matrix.android.sdk.api.session.Session
+import org.matrix.android.sdk.api.session.content.ContentUrlResolver
 import javax.inject.Inject
 
 class AutocompleteEmojiController @Inject constructor(
-        private val fontProvider: EmojiCompatFontProvider
+        private val fontProvider: EmojiCompatFontProvider,
+        private val session: Session
 ) : TypedEpoxyController<List<EmojiItem>>() {
 
     var emojiTypeface: Typeface? = fontProvider.typeface
@@ -36,7 +40,7 @@ class AutocompleteEmojiController @Inject constructor(
         }
     }
 
-    var listener: AutocompleteClickListener<String>? = null
+    var listener: AutocompleteClickListener<EmojiItem>? = null
 
     override fun buildModels(data: List<EmojiItem>?) {
         if (data.isNullOrEmpty()) {
@@ -49,8 +53,11 @@ class AutocompleteEmojiController @Inject constructor(
                     autocompleteEmojiItem {
                         id(emojiItem.name)
                         emojiItem(emojiItem)
+                        // For caching reasons, we use the AvatarRenderer's thumbnail size here
+                        emoteUrl(host.session.contentUrlResolver().resolveThumbnail(emojiItem.mxcUrl,
+                                AvatarRenderer.THUMBNAIL_SIZE, AvatarRenderer.THUMBNAIL_SIZE, ContentUrlResolver.ThumbnailMethod.SCALE))
                         emojiTypeFace(host.emojiTypeface)
-                        onClickListener { host.listener?.onItemClick(emojiItem.emoji) }
+                        onClickListener { host.listener?.onItemClick(emojiItem) }
                     }
                 }
 

diff --git a/vector/src/main/java/im/vector/app/features/autocomplete/emoji/AutocompleteEmojiItem.kt b/vector/src/main/java/im/vector/app/features/autocomplete/emoji/AutocompleteEmojiItem.kt
@@ -17,7 +17,9 @@
 package im.vector.app.features.autocomplete.emoji
 
 import android.graphics.Typeface
+import android.widget.ImageView
 import android.widget.TextView
+import androidx.core.view.isVisible
 import com.airbnb.epoxy.EpoxyAttribute
 import com.airbnb.epoxy.EpoxyModelClass
 import im.vector.app.R
@@ -26,14 +28,19 @@ import im.vector.app.core.epoxy.VectorEpoxyHolder
 import im.vector.app.core.epoxy.VectorEpoxyModel
 import im.vector.app.core.epoxy.onClick
 import im.vector.app.core.extensions.setTextOrHide
+import im.vector.app.core.glide.GlideApp
 import im.vector.app.features.reactions.data.EmojiItem
+import org.matrix.android.sdk.api.extensions.orFalse
 
 @EpoxyModelClass(layout = R.layout.item_autocomplete_emoji)
 abstract class AutocompleteEmojiItem : VectorEpoxyModel<AutocompleteEmojiItem.Holder>() {
 
     @EpoxyAttribute
     lateinit var emojiItem: EmojiItem
 
+    @EpoxyAttribute
+    var emoteUrl: String? = null
+
     @EpoxyAttribute
     var emojiTypeFace: Typeface? = null
 
@@ -42,7 +49,18 @@ abstract class AutocompleteEmojiItem : VectorEpoxyModel<AutocompleteEmojiItem.Ho
 
     override fun bind(holder: Holder) {
         super.bind(holder)
-        holder.emojiText.text = emojiItem.emoji
+        if (emoteUrl?.isNotEmpty().orFalse()) {
+            holder.emojiText.isVisible = false
+            holder.emoteImage.isVisible = true
+            GlideApp.with(holder.emoteImage)
+                    .load(emoteUrl)
+                    .centerCrop()
+                    .into(holder.emoteImage)
+        } else {
+            holder.emojiText.text = emojiItem.emoji
+            holder.emojiText.isVisible = true
+            holder.emoteImage.isVisible = false
+        }
         holder.emojiText.typeface = emojiTypeFace ?: Typeface.DEFAULT
         holder.emojiNameText.text = emojiItem.name
         holder.emojiKeywordText.setTextOrHide(emojiItem.keywords.joinToString())
@@ -51,6 +69,7 @@ abstract class AutocompleteEmojiItem : VectorEpoxyModel<AutocompleteEmojiItem.Ho
 
     class Holder : VectorEpoxyHolder() {
         val emojiText by bind<TextView>(R.id.itemAutocompleteEmoji)
+        val emoteImage by bind<ImageView>(R.id.itemAutocompleteEmote)
         val emojiNameText by bind<TextView>(R.id.itemAutocompleteEmojiName)
         val emojiKeywordText by bind<TextView>(R.id.itemAutocompleteEmojiSubname)
     }

diff --git a/vector/src/main/java/im/vector/app/features/autocomplete/emoji/AutocompleteEmojiPresenter.kt b/vector/src/main/java/im/vector/app/features/autocomplete/emoji/AutocompleteEmojiPresenter.kt
@@ -18,20 +18,35 @@ package im.vector.app.features.autocomplete.emoji
 
 import android.content.Context
 import androidx.recyclerview.widget.RecyclerView
+import dagger.assisted.Assisted
+import dagger.assisted.AssistedFactory
+import dagger.assisted.AssistedInject
 import im.vector.app.features.autocomplete.AutocompleteClickListener
 import im.vector.app.features.autocomplete.RecyclerViewPresenter
 import im.vector.app.features.reactions.data.EmojiDataSource
+import im.vector.app.features.reactions.data.EmojiItem
+import im.vector.app.features.settings.VectorPreferences
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.SupervisorJob
 import kotlinx.coroutines.cancelChildren
 import kotlinx.coroutines.launch
-import javax.inject.Inject
+import org.matrix.android.sdk.api.session.Session
+import org.matrix.android.sdk.api.session.events.model.EventType
+import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.api.session.getRoom
+import org.matrix.android.sdk.api.session.room.getStateEvent
+import org.matrix.android.sdk.api.session.room.model.RoomEmoteContent
 
-class AutocompleteEmojiPresenter @Inject constructor(context: Context,
-                                                     private val emojiDataSource: EmojiDataSource,
-                                                     private val controller: AutocompleteEmojiController) :
-        RecyclerViewPresenter<String>(context), AutocompleteClickListener<String> {
+class AutocompleteEmojiPresenter @AssistedInject constructor(context: Context,
+                                                             @Assisted val roomId: String,
+                                                             private val session: Session,
+                                                             private val vectorPreferences: VectorPreferences,
+                                                             private val emojiDataSource: EmojiDataSource,
+                                                             private val controller: AutocompleteEmojiController) :
+        RecyclerViewPresenter<EmojiItem>(context), AutocompleteClickListener<EmojiItem> {
+
+    private val room by lazy { session.getRoom(roomId)!! }
 
     private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
 
@@ -44,23 +59,45 @@ class AutocompleteEmojiPresenter @Inject constructor(context: Context,
         controller.listener = null
     }
 
+    @AssistedFactory
+    interface Factory {
+        fun create(roomId: String): AutocompleteEmojiPresenter
+    }
+
     override fun instantiateAdapter(): RecyclerView.Adapter<*> {
         return controller.adapter
     }
 
-    override fun onItemClick(t: String) {
+    override fun onItemClick(t: EmojiItem) {
         dispatchClick(t)
     }
 
     override fun onQuery(query: CharSequence?) {
         coroutineScope.launch {
+            // Plain emojis
             val data = if (query.isNullOrBlank()) {
                 // Return common emojis
                 emojiDataSource.getQuickReactions()
             } else {
                 emojiDataSource.filterWith(query.toString())
             }
-            controller.setData(data)
+
+            // Custom emotes
+            // TODO may want to add headers (compare @room vs @person completion) for
+            // - Standard emojis
+            // - Room-specific emotes
+            // - Global emotes (exported from other rooms)
+            val images = room.getStateEvent(EventType.ROOM_EMOTES)?.content?.toModel<RoomEmoteContent>()?.images.orEmpty()
+            val emoteData = images.filter {
+                val usages = it.value.usage
+                usages.isNullOrEmpty() || RoomEmoteContent.USAGE_EMOTICON in usages
+            }.filter {
+                query == null || it.key.contains(query, true)
+            }.map {
+                EmojiItem(it.key, "", mxcUrl = it.value.url)
+            }.sortedBy { it.name }.distinct()
+
+            controller.setData(emoteData + data)
         }
     }
 }
diff --git a/vector/src/main/java/im/vector/app/features/home/AvatarRenderer.kt b/vector/src/main/java/im/vector/app/features/home/AvatarRenderer.kt
@@ -60,7 +60,7 @@ class AvatarRenderer @Inject constructor(private val activeSessionHolder: Active
                                          private val dimensionConverter: DimensionConverter) {
 
     companion object {
-        private const val THUMBNAIL_SIZE = 250
+        const val THUMBNAIL_SIZE = 250
     }
 
     @UiThread

diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/AutoCompleter.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/AutoCompleter.kt
@@ -40,6 +40,7 @@ import im.vector.app.features.command.Command
 import im.vector.app.features.displayname.getBestName
 import im.vector.app.features.home.AvatarRenderer
 import im.vector.app.features.html.PillImageSpan
+import im.vector.app.features.reactions.data.EmojiItem
 import im.vector.app.features.themes.ThemeUtils
 import org.matrix.android.sdk.api.session.group.model.GroupSummary
 import org.matrix.android.sdk.api.session.room.model.RoomSummary
@@ -55,12 +56,14 @@ class AutoCompleter @AssistedInject constructor(
         private val commandAutocompletePolicy: CommandAutocompletePolicy,
         autocompleteCommandPresenterFactory: AutocompleteCommandPresenter.Factory,
         private val autocompleteMemberPresenterFactory: AutocompleteMemberPresenter.Factory,
+        private val autocompleteEmojiPresenterFactory: AutocompleteEmojiPresenter.Factory,
         private val autocompleteRoomPresenter: AutocompleteRoomPresenter,
         private val autocompleteGroupPresenter: AutocompleteGroupPresenter,
-        private val autocompleteEmojiPresenter: AutocompleteEmojiPresenter
+        //private val autocompleteEmojiPresenter: AutocompleteEmojiPresenter
 ) {
 
     private lateinit var autocompleteMemberPresenter: AutocompleteMemberPresenter
+    private lateinit var autocompleteEmojiPresenter: AutocompleteEmojiPresenter
 
     @AssistedFactory
     interface Factory {
@@ -189,13 +192,14 @@ class AutoCompleter @AssistedInject constructor(
     }
 
     private fun setupEmojis(backgroundDrawable: Drawable, editText: EditText) {
-        Autocomplete.on<String>(editText)
+        autocompleteEmojiPresenter = autocompleteEmojiPresenterFactory.create(roomId)
+        Autocomplete.on<EmojiItem>(editText)
                 .with(CharPolicy(TRIGGER_AUTO_COMPLETE_EMOJIS, false))
                 .with(autocompleteEmojiPresenter)
                 .with(ELEVATION_DP)
                 .with(backgroundDrawable)
-                .with(object : AutocompleteCallback<String> {
-                    override fun onPopupItemClicked(editable: Editable, item: String): Boolean {
+                .with(object : AutocompleteCallback<EmojiItem> {
+                    override fun onPopupItemClicked(editable: Editable, item: EmojiItem): Boolean {
                         // Infer that the last ":" before the current cursor position is the original popup trigger
                         var startIndex = editable.subSequence(0, editText.selectionStart).lastIndexOf(":")
                         if (startIndex == -1) {
@@ -210,7 +214,25 @@ class AutoCompleter @AssistedInject constructor(
 
                         // Replace the word by its completion
                         editable.delete(startIndex, endIndex)
-                        editable.insert(startIndex, item)
+                        if (item.mxcUrl.isNotEmpty()) {
+                            // Add emote html
+                            val emote = ":${item.name}:"
+                            editable.insert(startIndex, emote)
+
+                            // Add span to make it look nice
+                            val matrixItem = MatrixItem.EmoteItem(item.mxcUrl, item.name, item.mxcUrl)
+                            val span = PillImageSpan(
+                                    glideRequests,
+                                    avatarRenderer,
+                                    editText.context,
+                                    matrixItem
+                            )
+                            span.bind(editText)
+
+                            editable.setSpan(span, startIndex, startIndex + emote.length, Spannable.SPAN_EXCLUSIVE_EXCLUSIVE)
+                        } else {
+                            editable.insert(startIndex, item.emoji)
+                        }
                         return true
                     }
 

diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/TimelineFragment.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/TimelineFragment.kt
@@ -80,7 +80,6 @@ import de.spiritcroc.recyclerview.widget.BetterLinearLayoutManager
 import de.spiritcroc.recyclerview.widget.LinearLayoutManager
 import im.vector.app.R
 import im.vector.app.core.animations.play
-import im.vector.app.core.date.DateFormatKind
 import im.vector.app.core.dialogs.ConfirmationDialogBuilder
 import im.vector.app.core.dialogs.GalleryOrCameraDialogHelper
 import im.vector.app.core.epoxy.LayoutManagerStateRestorer
@@ -264,6 +263,7 @@ import org.matrix.android.sdk.api.session.widgets.model.WidgetType
 import org.matrix.android.sdk.api.util.MatrixItem
 import org.matrix.android.sdk.api.util.MimeTypes
 import org.matrix.android.sdk.api.util.toMatrixItem
+import org.matrix.android.sdk.internal.session.room.send.pills.requiresFormattedMessage
 import reactivecircus.flowbinding.android.view.focusChanges
 import reactivecircus.flowbinding.android.widget.textChanges
 import timber.log.Timber
@@ -1852,7 +1852,8 @@ class TimelineFragment @Inject constructor(
             // We collapse ASAP, if not there will be a slight annoying delay
             views.composerLayout.collapse(true)
             lockSendButton = true
-            messageComposerViewModel.handle(MessageComposerAction.SendMessage(text, vectorPreferences.isMarkdownEnabled()))
+            val forceMarkdown = text.requiresFormattedMessage()
+            messageComposerViewModel.handle(MessageComposerAction.SendMessage(text, forceMarkdown || vectorPreferences.isMarkdownEnabled()))
             emojiPopup.dismiss()
 
             if (vectorPreferences.jumpToBottomOnSend()) {