Custom and remote emojis (#405)

* initial custom emoji schema

* some more progress idk

* emoji process

* better emoji processing

* it just works

* implement mastodon's /api/v1/custom_emojis

* proper url and shortcode in /api/v1/custom_emojis

* add emoji list to statues

* add some tests and stand against clippy

* handle some unsound unwraps

* make emoji a resolvable AP type

* fix tests

* add a missing test fixture

* address PR review

---------

Co-authored-by: aumetra <aumetra@cryptolab.net>

Author: zeerooth

crates/kitsune-core/src/activitypub/fetcher.rs

@@ -14,25 +14,30 @@ use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper};
 use diesel_async::RunQueryDsl;
 use headers::{ContentType, HeaderMapExt};
 use http::HeaderValue;
+use iso8601_timestamp::Timestamp;
 use kitsune_cache::{ArcCache, CacheBackend};
 use kitsune_db::{
     model::{
         account::{Account, AccountConflictChangeset, NewAccount, UpdateAccountMedia},
+        custom_emoji::CustomEmoji,
+        media_attachment::{MediaAttachment, NewMediaAttachment},
         post::Post,
     },
-    schema::{accounts, posts},
+    schema::{accounts, custom_emojis, media_attachments, posts},
     PgPool,
 };
 use kitsune_embed::Client as EmbedClient;
 use kitsune_http_client::Client;
+
 use kitsune_search::SearchBackend;
 use kitsune_type::{
-    ap::{actor::Actor, Object},
+    ap::{actor::Actor, emoji::Emoji, Object},
     jsonld::RdfNode,
 };
 use mime::Mime;
 use scoped_futures::ScopedFutureExt;
 use serde::de::DeserializeOwned;
+use speedy_uuid::Uuid;
 use typed_builder::TypedBuilder;
 use url::Url;
 
@@ -175,7 +180,7 @@ impl Fetcher {
 
         let mut actor: Actor = self.fetch_ap_resource(url.as_str()).await?;
 
-        let mut domain = url.host_str().unwrap();
+        let mut domain = url.host_str().ok_or(ApiError::MissingHost)?;
         let domain_buf;
         let fetch_webfinger = opts
             .acct
@@ -203,7 +208,7 @@ impl Fetcher {
         };
         if !used_webfinger && actor.id != url.as_str() {
             url = Url::parse(&actor.id)?;
-            domain = url.host_str().unwrap();
+            domain = url.host_str().ok_or(ApiError::MissingHost)?;
         }
 
         actor.clean_html();
@@ -300,6 +305,88 @@ impl Fetcher {
         Ok(account)
     }
 
+    pub async fn fetch_emoji(&self, url: &str) -> Result<CustomEmoji> {
+        let existing_emoji = self
+            .db_pool
+            .with_connection(|db_conn| {
+                async move {
+                    custom_emojis::table
+                        .filter(custom_emojis::remote_id.eq(url))
+                        .select(CustomEmoji::as_select())
+                        .first(db_conn)
+                        .await
+                        .optional()
+                }
+                .scoped()
+            })
+            .await?;
+
+        if let Some(emoji) = existing_emoji {
+            return Ok(emoji);
+        }
+
+        let mut url = Url::parse(url)?;
+        if !self.federation_filter.is_url_allowed(&url)? {
+            return Err(ApiError::Unauthorised.into());
+        }
+
+        let emoji: Emoji = self.client.get(url.as_str()).await?.jsonld().await?;
+
+        let mut domain = url.host_str().ok_or(ApiError::MissingHost)?;
+
+        if emoji.id != url.as_str() {
+            url = Url::parse(&emoji.id)?;
+            domain = url.host_str().ok_or(ApiError::MissingHost)?;
+        }
+
+        let content_type = emoji
+            .icon
+            .media_type
+            .as_deref()
+            .or_else(|| mime_guess::from_path(&emoji.icon.url).first_raw())
+            .ok_or(ApiError::UnsupportedMediaType)?;
+
+        let name_pure = emoji.name.replace(':', "");
+
+        let emoji: CustomEmoji = self
+            .db_pool
+            .with_transaction(|tx| {
+                async move {
+                    let media_attachment = diesel::insert_into(media_attachments::table)
+                        .values(NewMediaAttachment {
+                            id: Uuid::now_v7(),
+                            account_id: None,
+                            content_type,
+                            description: None,
+                            blurhash: None,
+                            file_path: None,
+                            remote_url: Some(&emoji.icon.url),
+                        })
+                        .returning(MediaAttachment::as_returning())
+                        .get_result::<MediaAttachment>(tx)
+                        .await?;
+                    let emoji = diesel::insert_into(custom_emojis::table)
+                        .values(CustomEmoji {
+                            id: Uuid::now_v7(),
+                            remote_id: emoji.id,
+                            shortcode: name_pure.to_string(),
+                            domain: Some(domain.to_string()),
+                            media_attachment_id: media_attachment.id,
+                            endorsed: false,
+                            created_at: Timestamp::now_utc(),
+                            updated_at: Timestamp::now_utc(),
+                        })
+                        .returning(CustomEmoji::as_returning())
+                        .get_result::<CustomEmoji>(tx)
+                        .await?;
+                    Ok::<_, Error>(emoji)
+                }
+                .scope_boxed()
+            })
+            .await?;
+        Ok(emoji)
+    }
+
     #[async_recursion]
     pub(super) async fn fetch_object_inner(
         &self,
@@ -382,7 +469,10 @@ mod test {
     use iso8601_timestamp::Timestamp;
     use kitsune_cache::NoopCache;
     use kitsune_config::instance::FederationFilterConfiguration;
-    use kitsune_db::{model::account::Account, schema::accounts};
+    use kitsune_db::{
+        model::{account::Account, media_attachment::MediaAttachment},
+        schema::{accounts, media_attachments},
+    };
     use kitsune_http_client::Client;
     use kitsune_search::NoopSearchService;
     use kitsune_test::{build_ap_response, database_test};
@@ -915,6 +1005,55 @@ mod test {
         .await;
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn fetch_emoji() {
+        database_test(|db_pool| async move {
+            let client = Client::builder().service(service_fn(handle));
+
+            let fetcher = Fetcher::builder()
+                .client(client.clone())
+                .db_pool(db_pool.clone())
+                .embed_client(None)
+                .federation_filter(
+                    FederationFilterService::new(&FederationFilterConfiguration::Deny {
+                        domains: Vec::new(),
+                    })
+                    .unwrap(),
+                )
+                .search_backend(NoopSearchService)
+                .webfinger(Webfinger::with_client(client, Arc::new(NoopCache.into())))
+                .post_cache(Arc::new(NoopCache.into()))
+                .user_cache(Arc::new(NoopCache.into()))
+                .build();
+
+            let emoji = fetcher
+                .fetch_emoji("https://corteximplant.com/emojis/7952")
+                .await
+                .expect("Fetch emoji");
+            assert_eq!(emoji.shortcode, "Blobhaj");
+            assert_eq!(emoji.domain, Some(String::from("corteximplant.com")));
+
+            let media_attachment = db_pool
+                .with_connection(|db_conn| {
+                    media_attachments::table
+                        .find(emoji.media_attachment_id)
+                        .select(MediaAttachment::as_select())
+                        .get_result::<MediaAttachment>(db_conn)
+                        .scoped()
+                })
+                .await
+                .expect("Get media attachment");
+
+            assert_eq!(
+                media_attachment.remote_url,
+                Some(String::from(
+                    "https://corteximplant.com/system/custom_emojis/images/000/007/952/original/33b7f12bd094b815.png"
+                )));
+        })
+        .await;
+    }
+
     async fn handle(req: Request<Body>) -> Result<Response<Body>, Infallible> {
         match req.uri().path_and_query().unwrap().as_str() {
             "/users/0x0" => {
@@ -933,6 +1072,16 @@ mod test {
                 );
                 Ok::<_, Infallible>(build_ap_response(body))
             }
+            "/emojis/7952" => {
+                let body =
+                    include_str!("../../../../test-fixtures/corteximplant.com_emoji_7952.json");
+                Ok::<_, Infallible>(build_ap_response(body))
+            }
+            "/emojis/8933" => {
+                let body =
+                    include_str!("../../../../test-fixtures/corteximplant.com_emoji_8933.json");
+                Ok::<_, Infallible>(build_ap_response(body))
+            }
             "/.well-known/webfinger?resource=acct:0x0@corteximplant.com" => {
                 let body = include_str!("../../../../test-fixtures/0x0_jrd.json");
                 Ok::<_, Infallible>(Response::new(Body::from(body)))

crates/kitsune-core/src/activitypub/mod.rs

@@ -5,17 +5,20 @@ use crate::{
 };
 use diesel::{ExpressionMethods, SelectableHelper};
 use diesel_async::{AsyncPgConnection, RunQueryDsl};
-use futures_util::FutureExt;
+use futures_util::{future::try_join_all, FutureExt, TryFutureExt};
 use http::Uri;
 use iso8601_timestamp::Timestamp;
 use kitsune_db::{
     model::{
         account::Account,
+        custom_emoji::PostCustomEmoji,
         media_attachment::{NewMediaAttachment, NewPostMediaAttachment},
         mention::NewMention,
         post::{FullPostChangeset, NewPost, Post, PostConflictChangeset, Visibility},
     },
-    schema::{media_attachments, posts, posts_media_attachments, posts_mentions},
+    schema::{
+        media_attachments, posts, posts_custom_emojis, posts_media_attachments, posts_mentions,
+    },
     PgPool,
 };
 use kitsune_embed::Client as EmbedClient;
@@ -64,6 +67,43 @@ async fn handle_mentions(
     Ok(())
 }
 
+async fn handle_custom_emojis(
+    db_conn: &mut AsyncPgConnection,
+    post_id: Uuid,
+    fetcher: &Fetcher,
+    tags: &[Tag],
+) -> Result<()> {
+    let emoji_iter = tags.iter().filter(|tag| tag.r#type == TagType::Emoji);
+
+    let emoji_count = emoji_iter.clone().count();
+    if emoji_count == 0 {
+        return Ok(());
+    }
+
+    let futures = emoji_iter.clone().filter_map(|emoji| {
+        let remote_id = emoji.id.as_ref()?;
+        Some(fetcher.fetch_emoji(remote_id).map_ok(move |f| (f, emoji)))
+    });
+
+    let emojis = try_join_all(futures)
+        .await?
+        .iter()
+        .map(|(resolved_emoji, emoji_tag)| PostCustomEmoji {
+            post_id,
+            custom_emoji_id: resolved_emoji.id,
+            emoji_text: emoji_tag.name.to_string(),
+        })
+        .collect::<Vec<PostCustomEmoji>>();
+
+    diesel::insert_into(posts_custom_emojis::table)
+        .values(emojis)
+        .on_conflict_do_nothing()
+        .execute(db_conn)
+        .await?;
+
+    Ok(())
+}
+
 /// Process a bunch of ActivityPub attachments
 ///
 /// # Returns
@@ -92,7 +132,7 @@ pub async fn process_attachments(
 
                     Some(NewMediaAttachment {
                         id: attachment_id,
-                        account_id: author.id,
+                        account_id: Some(author.id),
                         content_type,
                         description: attachment.name.as_deref(),
                         blurhash: attachment.blurhash.as_deref(),
@@ -130,6 +170,7 @@ struct PreprocessedObject<'a> {
     content_lang: Language,
     db_pool: &'a PgPool,
     object: Box<Object>,
+    fetcher: &'a Fetcher,
     search_backend: &'a AnySearchBackend,
 }
 
@@ -201,6 +242,7 @@ async fn preprocess_object(
         content_lang,
         db_pool,
         object,
+        fetcher,
         search_backend,
     })
 }
@@ -215,6 +257,7 @@ pub async fn process_new_object(process_data: ProcessNewObject<'_>) -> Result<Po
         content_lang,
         db_pool,
         object,
+        fetcher,
         search_backend,
     } = preprocess_object(process_data).boxed().await?;
 
@@ -263,6 +306,7 @@ pub async fn process_new_object(process_data: ProcessNewObject<'_>) -> Result<Po
                     .await?;
 
                 handle_mentions(tx, &user, new_post.id, &object.tag).await?;
+                handle_custom_emojis(tx, new_post.id, fetcher, &object.tag).await?;
 
                 Ok::<_, Error>(new_post)
             }
@@ -287,6 +331,7 @@ pub async fn update_object(process_data: ProcessNewObject<'_>) -> Result<Post> {
         content_lang,
         db_pool,
         object,
+        fetcher: _,
         search_backend,
     } = preprocess_object(process_data).await?;
 

crates/kitsune-core/src/consts.rs

@@ -1,6 +1,7 @@
 use const_format::concatcp;
 
 pub const API_MAX_LIMIT: usize = 40;
+pub const MAX_EMOJI_SHORTCODE_LENGTH: usize = 64;
 pub const MAX_MEDIA_DESCRIPTION_LENGTH: usize = 5000;
 pub const USER_AGENT: &str = concatcp!(env!("CARGO_PKG_NAME"), "/", VERSION);
 pub const VERSION: &str = concatcp!(env!("CARGO_PKG_VERSION"), "-", env!("VERGEN_GIT_SHA"));

crates/kitsune-core/src/error.rs

@@ -20,6 +20,9 @@ pub enum ApiError {
     #[error("Invalid captcha")]
     InvalidCaptcha,
 
+    #[error("Missing host")]
+    MissingHost,
+
     #[error("Not found")]
     NotFound,
 

crates/kitsune-core/src/lib.rs

@@ -56,7 +56,9 @@ use kitsune_search::{AnySearchBackend, NoopSearchService, SqlSearchService};
 use kitsune_storage::{fs::Storage as FsStorage, s3::Storage as S3Storage, AnyStorageBackend};
 use rusty_s3::{Bucket as S3Bucket, Credentials as S3Credentials};
 use serde::{de::DeserializeOwned, Serialize};
+use service::custom_emoji::CustomEmojiService;
 use service::search::SearchService;
+
 use std::{
     fmt::Display,
     str::FromStr,
@@ -272,6 +274,12 @@ pub async fn prepare_state(
     let captcha_backend = config.captcha.as_ref().map(prepare_captcha);
     let captcha_service = CaptchaService::builder().backend(captcha_backend).build();
 
+    let custom_emoji_service = CustomEmojiService::builder()
+        .attachment_service(attachment_service.clone())
+        .db_pool(db_pool.clone())
+        .url_service(url_service.clone())
+        .build();
+
     let instance_service = InstanceService::builder()
         .db_pool(db_pool.clone())
         .name(config.instance.name.as_str())
@@ -292,6 +300,7 @@ pub async fn prepare_state(
 
     let post_resolver = PostResolver::builder()
         .account(account_service.clone())
+        .custom_emoji(custom_emoji_service.clone())
         .build();
 
     let post_service = PostService::builder()
@@ -344,6 +353,7 @@ pub async fn prepare_state(
         service: Service {
             account: account_service,
             captcha: captcha_service,
+            custom_emoji: custom_emoji_service,
             federation_filter: federation_filter_service,
             instance: instance_service,
             job: job_service,