Skip to content

Commit 6df04f0

Browse files
aumetraaumetra
authored
Replace redis-rs with fred (#535)
* impl fred types for speedy-uuid * begin port * progress * rm multiplex-pool * progress * progress * progress * progress * up * fix kitsune-test * let-else * upgrade * fix indefinite looping * up * fix * limit fetch depth further
1 parent dd949ab commit 6df04f0

File tree

64 files changed

+742
-872
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+742
-872
lines changed

‎Cargo.lock

+358-356
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+12-8
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@ members = [
6262
"lib/masto-id-convert",
6363
"lib/mrf-manifest",
6464
"lib/mrf-tool",
65-
"lib/multiplex-pool",
6665
"lib/post-process",
6766
"lib/speedy-uuid",
6867
"lib/tick-tock-mock",
@@ -75,6 +74,14 @@ members = [
7574
]
7675
resolver = "2"
7776

77+
[workspace.dependencies]
78+
fred = { version = "9.0.3", features = [
79+
"enable-rustls-ring",
80+
"i-scripts",
81+
"partial-tracing",
82+
"sha-1",
83+
] }
84+
7885
[workspace.lints.clippy]
7986
all = "warn"
8087
pedantic = "warn"
@@ -126,13 +133,10 @@ diesel-async = { git = "https://github.com/weiznich/diesel_async.git", rev = "d0
126133
scraper = { git = "https://github.com/causal-agent/scraper.git", rev = "d67111f5cc0b7da6e6ff10e4549d87cf09ba3e5b" }
127134
tokio-postgres-rustls = { git = "https://github.com/jbg/tokio-postgres-rustls.git", rev = "b16c1bc0f5d4f91324174fd1bd839d743a70f86a" }
128135

129-
# Support XAUTOCLAIM
130-
redis = { git = "https://github.com/redis-rs/redis-rs.git", rev = "0c544b548b52180acda8e8394ab21b2761497e50" }
131-
132136
# Patch to make OpenTelemetry with with hyper 1
133-
opentelemetry = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "b44cb130e4a102b0d676289e91c003f4b1008d08" }
134-
opentelemetry-http = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "b44cb130e4a102b0d676289e91c003f4b1008d08" }
135-
opentelemetry-otlp = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "b44cb130e4a102b0d676289e91c003f4b1008d08" }
136-
opentelemetry_sdk = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "b44cb130e4a102b0d676289e91c003f4b1008d08" }
137+
opentelemetry = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "40fb924215c84d6e3818351b2c0e3d2ba01a7db9" }
138+
opentelemetry-http = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "40fb924215c84d6e3818351b2c0e3d2ba01a7db9" }
139+
opentelemetry-otlp = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "40fb924215c84d6e3818351b2c0e3d2ba01a7db9" }
140+
opentelemetry_sdk = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "40fb924215c84d6e3818351b2c0e3d2ba01a7db9" }
137141
tonic = { git = "https://github.com/hyperium/tonic.git", rev = "9b306af386528f11dbd022bc372d367adc4e96b5" }
138142
tracing-opentelemetry = { git = "https://github.com/aumetra/tracing-opentelemetry.git", branch = "v0.1.x-http1" }

‎crates/kitsune-activitypub/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ kitsune-util = { path = "../kitsune-util" }
3131
kitsune-wasm-mrf = { path = "../kitsune-wasm-mrf" }
3232
mime = "0.3.17"
3333
mime_guess = { version = "2.0.4", default-features = false }
34-
serde = "1.0.201"
34+
serde = "1.0.202"
3535
sha2 = "0.10.8"
3636
simd-json = "0.13.10"
3737
speedy-uuid = { path = "../../lib/speedy-uuid" }

‎crates/kitsune-activitypub/src/fetcher/object.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use kitsune_error::Result;
88

99
// Maximum call depth of fetching new posts. Prevents unbounded recursion.
1010
// Setting this to >=40 would cause the `fetch_infinitely_long_reply_chain` test to run into stack overflow
11-
pub const MAX_FETCH_DEPTH: u32 = 15;
11+
pub const MAX_FETCH_DEPTH: u32 = 10;
1212

1313
impl Fetcher {
1414
#[instrument(skip(self))]

‎crates/kitsune-cache/Cargo.toml

+2-6
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,10 @@ license.workspace = true
77

88
[dependencies]
99
enum_dispatch = "0.3.13"
10+
fred = { workspace = true }
1011
kitsune-error = { path = "../kitsune-error" }
1112
moka = { version = "0.12.7", features = ["sync"] }
12-
multiplex-pool = { path = "../../lib/multiplex-pool" }
13-
redis = { version = "0.25.3", default-features = false, features = [
14-
"connection-manager",
15-
"tokio-comp",
16-
] }
17-
serde = "1.0.201"
13+
serde = "1.0.202"
1814
simd-json = "0.13.10"
1915
tracing = "0.1.40"
2016
triomphe = "0.1.11"

‎crates/kitsune-cache/src/redis.rs

+14-13
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use super::CacheBackend;
2+
use fred::{clients::RedisPool, interfaces::KeysInterface, types::Expiration};
23
use kitsune_error::Result;
3-
use redis::{aio::ConnectionManager, AsyncCommands};
44
use serde::{de::DeserializeOwned, Serialize};
55
use std::{fmt::Display, marker::PhantomData, time::Duration};
66
use typed_builder::TypedBuilder;
@@ -14,7 +14,7 @@ where
1414
namespace: String,
1515
#[builder(setter(into))]
1616
prefix: String,
17-
redis_conn: multiplex_pool::Pool<ConnectionManager>,
17+
redis_conn: RedisPool,
1818
ttl: Duration,
1919

2020
// Type phantom data
@@ -28,11 +28,7 @@ impl<K, V> Redis<K, V>
2828
where
2929
K: ?Sized,
3030
{
31-
pub fn new<P>(
32-
redis_conn: multiplex_pool::Pool<ConnectionManager>,
33-
prefix: P,
34-
ttl: Duration,
35-
) -> Self
31+
pub fn new<P>(redis_conn: RedisPool, prefix: P, ttl: Duration) -> Self
3632
where
3733
P: Into<String>,
3834
{
@@ -55,22 +51,20 @@ where
5551
{
5652
#[instrument(skip_all, fields(%key))]
5753
async fn delete(&self, key: &K) -> Result<()> {
58-
let mut conn = self.redis_conn.get();
5954
let key = self.compute_key(key);
6055

6156
debug!(%key, "Deleting cache entry");
62-
conn.del(key).await?;
57+
self.redis_conn.del(key).await?;
6358

6459
Ok(())
6560
}
6661

6762
#[instrument(skip_all, fields(%key))]
6863
async fn get(&self, key: &K) -> Result<Option<V>> {
69-
let mut conn = self.redis_conn.get();
7064
let key = self.compute_key(key);
7165

7266
debug!(%key, "Fetching cache entry");
73-
if let Some(serialised) = conn.get::<_, Option<String>>(&key).await? {
67+
if let Some(serialised) = self.redis_conn.get::<Option<String>, _>(&key).await? {
7468
let mut serialised_bytes = serialised.into_bytes();
7569
let deserialised = simd_json::from_slice(&mut serialised_bytes)?;
7670
Ok(Some(deserialised))
@@ -81,12 +75,19 @@ where
8175

8276
#[instrument(skip_all, fields(%key))]
8377
async fn set(&self, key: &K, value: &V) -> Result<()> {
84-
let mut conn = self.redis_conn.get();
8578
let key = self.compute_key(key);
8679
let serialised = simd_json::to_string(value)?;
8780

8881
debug!(%key, ttl = ?self.ttl, "Setting cache entry");
89-
conn.set_ex(key, serialised, self.ttl.as_secs()).await?;
82+
self.redis_conn
83+
.set(
84+
key,
85+
serialised,
86+
Some(Expiration::EX(self.ttl.as_secs() as i64)),
87+
None,
88+
false,
89+
)
90+
.await?;
9091

9192
Ok(())
9293
}

‎crates/kitsune-captcha/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ enum_dispatch = "0.3.13"
1010
http = "1.1.0"
1111
kitsune-error = { path = "../kitsune-error" }
1212
kitsune-http-client = { path = "../kitsune-http-client" }
13-
serde = { version = "1.0.201", features = ["derive"] }
13+
serde = { version = "1.0.202", features = ["derive"] }
1414
serde_urlencoded = "0.7.1"
1515
simd-json = "0.13.10"
1616
strum = { version = "0.26.2", features = ["derive"] }

‎crates/kitsune-config/Cargo.toml

+3-3
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@ license.workspace = true
99
eyre = "0.6.12"
1010
human-size = { version = "0.4.3", features = ["serde"] }
1111
isolang = { version = "2.4.0", features = ["serde"] }
12-
serde = { version = "1.0.201", features = ["derive"] }
13-
smol_str = { version = "0.2.1", features = ["serde"] }
12+
serde = { version = "1.0.202", features = ["derive"] }
13+
smol_str = { version = "0.2.2", features = ["serde"] }
1414
tokio = { version = "1.37.0", features = ["fs"] }
15-
toml = { version = "0.8.12", default-features = false, features = ["parse"] }
15+
toml = { version = "0.8.13", default-features = false, features = ["parse"] }
1616

1717
[lints]
1818
workspace = true

‎crates/kitsune-core/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ const_format = "0.2.32"
1212
kitsune-db = { path = "../kitsune-db" }
1313
kitsune-error = { path = "../kitsune-error" }
1414
paste = "1.0.15"
15-
serde = { version = "1.0.201", features = ["derive"] }
15+
serde = { version = "1.0.202", features = ["derive"] }
1616
triomphe = { version = "0.1.11", features = ["unsize"] }
1717
typed-builder = "0.18.2"
1818
unsize = "1.1.0"

‎crates/kitsune-db/Cargo.toml

+2-2
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,14 @@ kitsune-language = { path = "../kitsune-language" }
2727
kitsune-type = { path = "../kitsune-type" }
2828
num-derive = "0.4.2"
2929
num-traits = "0.2.19"
30-
rustls = { version = "0.23.5", default-features = false, features = [
30+
rustls = { version = "0.23.7", default-features = false, features = [
3131
"logging",
3232
"ring",
3333
"std",
3434
"tls12",
3535
] }
3636
rustls-native-certs = "0.7.0"
37-
serde = { version = "1.0.201", features = ["derive"] }
37+
serde = { version = "1.0.202", features = ["derive"] }
3838
simd-json = "0.13.10"
3939
speedy-uuid = { path = "../../lib/speedy-uuid", features = ["diesel"] }
4040
tokio = { version = "1.37.0", features = ["rt"] }

‎crates/kitsune-derive/impl/Cargo.toml

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ license.workspace = true
99
proc-macro = true
1010

1111
[dependencies]
12-
proc-macro2 = "1.0.82"
12+
proc-macro2 = "1.0.83"
1313
quote = "1.0.36"
14-
syn = { version = "2.0.63", features = ["full"] }
14+
syn = { version = "2.0.65", features = ["full"] }
1515

1616
[lints]
1717
workspace = true

‎crates/kitsune-embed/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ kitsune-error = { path = "../kitsune-error" }
1717
kitsune-http-client = { path = "../kitsune-http-client" }
1818
once_cell = "1.19.0"
1919
scraper = { version = "0.19.0", default-features = false }
20-
smol_str = "0.2.1"
20+
smol_str = "0.2.2"
2121

2222
[lints]
2323
workspace = true

‎crates/kitsune-http-client/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ hyper-rustls = { version = "0.27.1", default-features = false, features = [
3131
] }
3232
kitsune-type = { path = "../kitsune-type" }
3333
pin-project = "1.1.5"
34-
serde = "1.0.201"
34+
serde = "1.0.202"
3535
simdutf8 = { version = "0.1.4", features = ["aarch64_neon"] }
3636
simd-json = "0.13.10"
3737
tower = { version = "0.4.13", features = ["util"] }

‎crates/kitsune-jobs/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ kitsune-core = { path = "../kitsune-core" }
1515
kitsune-db = { path = "../kitsune-db" }
1616
kitsune-email = { path = "../kitsune-email" }
1717
kitsune-error = { path = "../kitsune-error" }
18-
serde = { version = "1.0.201", features = ["derive"] }
18+
serde = { version = "1.0.202", features = ["derive"] }
1919
speedy-uuid = { path = "../../lib/speedy-uuid" }
2020
tracing = "0.1.40"
2121
typed-builder = "0.18.2"

‎crates/kitsune-mastodon/Cargo.toml

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ kitsune-type = { path = "../kitsune-type" }
1919
kitsune-url = { path = "../kitsune-url" }
2020
kitsune-util = { path = "../kitsune-util" }
2121
mime = "0.3.17"
22-
serde = "1.0.201"
22+
serde = "1.0.202"
2323
simd-json = "0.13.10"
24-
smol_str = "0.2.1"
24+
smol_str = "0.2.2"
2525
speedy-uuid = { path = "../../lib/speedy-uuid" }
2626
tracing = "0.1.40"
2727
typed-builder = "0.18.2"

‎crates/kitsune-observability/Cargo.toml

+4-4
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,19 @@ http-body-util = "0.1.1"
1212
hyper = { version = "1.3.1", default-features = false }
1313
kitsune-config = { path = "../kitsune-config" }
1414
kitsune-http-client = { path = "../kitsune-http-client" }
15-
opentelemetry = { version = "0.22.0", default-features = false, features = [
15+
opentelemetry = { version = "0.23.0", default-features = false, features = [
1616
"trace",
1717
] }
18-
opentelemetry-http = "0.11.1"
19-
opentelemetry-otlp = { version = "0.15.0", default-features = false, features = [
18+
opentelemetry-http = "0.12.0"
19+
opentelemetry-otlp = { version = "0.16.0", default-features = false, features = [
2020
"grpc-tonic",
2121
"http-json",
2222
"http-proto",
2323
"tls",
2424
"tls-roots",
2525
"trace",
2626
] }
27-
opentelemetry_sdk = { version = "0.22.1", default-features = false, features = [
27+
opentelemetry_sdk = { version = "0.23.0", default-features = false, features = [
2828
"rt-tokio",
2929
] }
3030
tracing = "0.1.40"

‎crates/kitsune-oidc/Cargo.toml

+2-6
Original file line numberDiff line numberDiff line change
@@ -7,26 +7,22 @@ license.workspace = true
77

88
[dependencies]
99
enum_dispatch = "0.3.13"
10+
fred = { workspace = true }
1011
http = "1.1.0"
1112
http-body-util = "0.1.1"
1213
kitsune-config = { path = "../kitsune-config" }
1314
kitsune-derive = { path = "../kitsune-derive" }
1415
kitsune-error = { path = "../kitsune-error" }
1516
kitsune-http-client = { path = "../kitsune-http-client" }
1617
moka = { version = "0.12.7", features = ["sync"] }
17-
multiplex-pool = { path = "../../lib/multiplex-pool" }
1818
oauth2 = { version = "5.0.0-alpha.4", default-features = false }
1919
once_cell = "1.19.0"
2020
openidconnect = { version = "4.0.0-alpha.1", default-features = false, features = [
2121
# Accept these two, per specification invalid, cases to increase compatibility
2222
"accept-rfc3339-timestamps",
2323
"accept-string-booleans",
2424
] }
25-
redis = { version = "0.25.3", default-features = false, features = [
26-
"connection-manager",
27-
"tokio-comp",
28-
] }
29-
serde = { version = "1.0.201", features = ["derive"] }
25+
serde = { version = "1.0.202", features = ["derive"] }
3026
simd-json = "0.13.10"
3127
speedy-uuid = { path = "../../lib/speedy-uuid", features = ["serde"] }
3228
url = "2.5.0"

‎crates/kitsune-oidc/src/lib.rs

+4-8
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ use crate::state::{
22
store::{InMemory as InMemoryStore, Redis as RedisStore},
33
LoginState, OAuth2LoginState, Store,
44
};
5+
use fred::{clients::RedisPool, types::RedisConfig};
56
use kitsune_config::oidc::{Configuration, StoreConfiguration};
67
use kitsune_derive::kitsune_service;
78
use kitsune_error::{bail, kitsune_error, Result};
8-
use multiplex_pool::RoundRobinStrategy;
99
use openidconnect::{
1010
core::{CoreAuthenticationFlow, CoreClient, CoreProviderMetadata},
1111
AccessTokenHash, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, Nonce,
@@ -92,13 +92,9 @@ impl OidcService {
9292
let login_state_store = match config.store {
9393
StoreConfiguration::InMemory => InMemoryStore::new(LOGIN_STATE_STORE_SIZE).into(),
9494
StoreConfiguration::Redis(ref redis_config) => {
95-
let client = redis::Client::open(redis_config.url.as_str())?;
96-
let pool = multiplex_pool::Pool::from_producer(
97-
|| client.get_connection_manager(),
98-
10,
99-
RoundRobinStrategy::default(),
100-
)
101-
.await?;
95+
let config = RedisConfig::from_url(redis_config.url.as_str())?;
96+
// TODO: Make pool size configurable
97+
let pool = RedisPool::new(config, None, None, None, 10)?;
10298

10399
RedisStore::new(pool).into()
104100
}

‎crates/kitsune-oidc/src/state/store/redis.rs

+7-7
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
use super::Store;
22
use crate::state::LoginState;
3+
use fred::{clients::RedisPool, interfaces::KeysInterface};
34
use kitsune_error::Result;
4-
use redis::{aio::ConnectionManager, AsyncCommands};
55

66
const REDIS_PREFIX: &str = "OIDC-LOGIN-STATE";
77

88
#[derive(Clone)]
99
pub struct Redis {
10-
pool: multiplex_pool::Pool<ConnectionManager>,
10+
pool: RedisPool,
1111
}
1212

1313
impl Redis {
14-
pub fn new(pool: multiplex_pool::Pool<ConnectionManager>) -> Self {
14+
pub fn new(pool: RedisPool) -> Self {
1515
Self { pool }
1616
}
1717

@@ -23,17 +23,17 @@ impl Redis {
2323

2424
impl Store for Redis {
2525
async fn get_and_remove(&self, key: &str) -> Result<LoginState> {
26-
let mut conn = self.pool.get();
27-
let raw_value: String = conn.get_del(Self::format_key(key)).await?;
26+
let raw_value: String = self.pool.getdel(Self::format_key(key)).await?;
2827

2928
let mut raw_value = raw_value.into_bytes();
3029
Ok(simd_json::from_slice(&mut raw_value)?)
3130
}
3231

3332
async fn set(&self, key: &str, value: LoginState) -> Result<()> {
3433
let raw_value = simd_json::to_string(&value)?;
35-
let mut conn = self.pool.get();
36-
conn.set(Self::format_key(key), raw_value).await?;
34+
self.pool
35+
.set(Self::format_key(key), raw_value, None, None, false)
36+
.await?;
3737

3838
Ok(())
3939
}

‎crates/kitsune-s3/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ kitsune-error = { path = "../kitsune-error" }
1313
kitsune-http-client = { path = "../kitsune-http-client" }
1414
quick-xml = { version = "0.31.0", features = ["serialize"] }
1515
rusty-s3 = "0.5.0"
16-
serde = { version = "1.0.201", features = ["derive"] }
16+
serde = { version = "1.0.202", features = ["derive"] }
1717
typed-builder = "0.18.2"
1818

1919
[dev-dependencies]

‎crates/kitsune-scss-compiler/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ license.workspace = true
88
[dependencies]
99
eyre = "0.6.12"
1010
glob = "0.3.1"
11-
grass_compiler = "0.13.2"
11+
grass_compiler = "0.13.3"
1212
tracing = { version = "0.1.40", default-features = false }
1313

1414
[lints]

0 commit comments

Comments
 (0)