zelo72
/
flodgatt
mirror of https://github.com/mastodon/flodgatt
1
0
Fork 0
Code Issues 1 Releases Wiki Activity

Add tests for polling for multiple messages 

This commit adds a mock Redis interface and adds tests that poll the
mock interface for multiple messages at a time.  These tests test that
Flodgatt is robust against receiving incomplete messages, including if
the message break results in receiving invalid UTF8.
This commit is contained in:
Daniel Sockwell 2020-05-04 20:14:51 -04:00
parent 4de9a94230
commit c3ec5c8eb7
30 changed files with 1007 additions and 371 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -416,7 +416,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "flodgatt"
version = "0.9.8"
version = "0.9.9"
dependencies = [
"criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"dotenv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",

2
Cargo.toml
View File

@ -1,7 +1,7 @@
[package]
name = "flodgatt"
description = "A blazingly fast drop-in replacement for the Mastodon streaming api server"
version = "0.9.8"
version = "0.9.9"
authors = ["Daniel Long Sockwell <daniel@codesections.com", "Julian Laubstein <contact@julianlaubstein.de>"]
edition = "2018"

111
benches/parse_redis.rs
View File

@ -1,7 +1,12 @@
use criterion::{black_box, criterion_group, criterion_main, Criterion};
use flodgatt::response::{RedisMsg, RedisParseOutput};
use flodgatt::config;
use flodgatt::request::{Content::*, Reach::*, Stream::*, Timeline};
use flodgatt::response::{Event, Manager, RedisMsg, RedisParseOutput};
use flodgatt::Id;
use futures::{Async, Stream};
use lru::LruCache;
use std::convert::TryFrom;
use std::fs;
fn parse_long_redis_input<'a>(input: &'a str) -> RedisMsg<'a> {
if let RedisParseOutput::Msg(msg) = RedisParseOutput::try_from(input).unwrap() {
@ -12,27 +17,34 @@ fn parse_long_redis_input<'a>(input: &'a str) -> RedisMsg<'a> {
}
}
// fn parse_to_timeline(msg: RedisMsg) -> Timeline {
// let trimmed_tl_txt = &msg.timeline_txt["timeline:".len()..];
// let tl = Timeline::from_redis_text(trimmed_tl_txt, &mut LruCache::new(1000)).unwrap();
// assert_eq!(tl, Timeline(User(Id(1)), Federated, All));
// tl
// }
// fn parse_to_checked_event(msg: RedisMsg) -> EventKind {
// EventKind::TypeSafe(serde_json::from_str(msg.event_txt).unwrap())
// }
fn parse_to_timeline(msg: RedisMsg) -> Timeline {
let trimmed_tl_txt = &msg.timeline_txt["timeline:".len()..];
let tl = Timeline::from_redis_text(trimmed_tl_txt, &mut LruCache::new(1000)).unwrap();
assert_eq!(tl, Timeline(User(Id(1)), Federated, All));
tl
}
fn parse_to_checked_event(msg: RedisMsg) -> Event {
Event::TypeSafe(serde_json::from_str(msg.event_txt).unwrap())
}
// fn parse_to_dyn_event(msg: RedisMsg) -> EventKind {
// EventKind::Dynamic(serde_json::from_str(msg.event_txt).unwrap())
// }
fn parse_to_dyn_event(msg: RedisMsg) -> Event {
Event::Dynamic(serde_json::from_str(msg.event_txt).unwrap())
}
// fn redis_msg_to_event_string(msg: RedisMsg) -> String {
// msg.event_txt.to_string()
// }
fn redis_msg_to_event_string(msg: RedisMsg) -> String {
msg.event_txt.to_string()
}
// fn string_to_checked_event(event_txt: &String) -> EventKind {
// EventKind::TypeSafe(serde_json::from_str(event_txt).unwrap())
// }
fn string_to_checked_event(event_txt: &String) -> Event {
Event::TypeSafe(serde_json::from_str(event_txt).unwrap())
}
fn input_msg(i: usize) -> Vec<u8> {
fs::read_to_string(format!("test_data/redis_input_{:03}.resp", i))
.expect("test input not found")
.as_bytes()
.to_vec()
}
fn criterion_benchmark(c: &mut Criterion) {
let input = ONE_MESSAGE_FOR_THE_USER_TIMLINE_FROM_REDIS;
@ -42,25 +54,54 @@ fn criterion_benchmark(c: &mut Criterion) {
b.iter(|| black_box(parse_long_redis_input(input)))
});
// let msg = parse_long_redis_input(input);
// group.bench_function("parse RedisMsg to Timeline", |b| {
// b.iter(|| black_box(parse_to_timeline(msg.clone())))
// });
let msg = parse_long_redis_input(input);
group.bench_function("parse RedisMsg to Timeline", |b| {
b.iter(|| black_box(parse_to_timeline(msg.clone())))
});
// group.bench_function("parse RedisMsg -> DynamicEvent", |b| {
// b.iter(|| black_box(parse_to_dyn_event(msg.clone())))
// });
group.bench_function("parse RedisMsg -> CheckedEvent", |b| {
b.iter(|| black_box(parse_to_checked_event(msg.clone())))
});
// group.bench_function("parse RedisMsg -> CheckedEvent", |b| {
// b.iter(|| black_box(parse_to_checked_event(msg.clone())))
// });
group.bench_function("parse RedisMsg -> DynamicEvent", |b| {
b.iter(|| black_box(parse_to_dyn_event(msg.clone())))
});
// group.bench_function("parse RedisMsg -> String -> CheckedEvent", |b| {
// b.iter(|| {
// let txt = black_box(redis_msg_to_event_string(msg.clone()));
// black_box(string_to_checked_event(&txt));
// })
// });
group.bench_function("parse RedisMsg -> String -> CheckedEvent", |b| {
b.iter(|| {
let txt = black_box(redis_msg_to_event_string(msg.clone()));
black_box(string_to_checked_event(&txt));
})
});
group.bench_function("parse six messages from Redis", |b| {
b.iter_batched(
|| {
let mut manager = Manager::try_from(&config::Redis::default()).expect("bench");
for i in 1..=6 {
manager.redis_conn.add(&input_msg(i));
}
manager
},
|mut m| {
black_box({
let mut i = 1;
while let Ok(Async::Ready(Some(len))) = m.redis_conn.poll_redis(m.unread_idx.1)
{
m.unread_idx = (0, m.unread_idx.1 + len);
while let Ok(Async::Ready(Some((_tl, event)))) = m.poll() {
// println!("Parsing Event #{:03}", i + 1);
// assert_eq!(event, output(i));
i += 1;
}
}
assert_eq!(i, 7)
})
},
criterion::BatchSize::SmallInput,
)
});
}
criterion_group!(benches, criterion_benchmark);

12
src/main.rs
View File

@ -83,12 +83,16 @@ fn main() -> Result<(), Error> {
let manager = shared_manager.clone();
let stream = Interval::new(Instant::now(), poll_freq)
.map_err(|e| log::error!("{}", e))
.for_each(
move |_| match manager.lock().unwrap_or_else(RedisManager::recover).poll() {
.for_each(move |_| {
match manager
.lock()
.unwrap_or_else(RedisManager::recover)
.send_msgs()
{
Err(e) => Ok(log::error!("{}", e)),
Ok(_) => Ok(()),
},
);
}
});
warp::spawn(lazy(move || stream));
warp::serve(ws.or(sse).with(cors).or(status).recover(Handler::err))

5
src/request.rs
View File

@ -9,6 +9,11 @@ mod subscription;
pub use err::{Error, Timeline as TimelineErr};
pub use subscription::{Blocks, Subscription};
pub use timeline::Timeline;
#[cfg(feature = "bench")]
pub use timeline::{Content, Reach, Stream};
#[cfg(not(feature = "bench"))]
use timeline::{Content, Reach, Stream};
pub use self::postgres::PgPool;

10
src/request/timeline.rs
View File

@ -1,6 +1,7 @@
pub(crate) use self::inner::{Content, Reach, Scope, Stream, UserData};
pub use self::inner::{Content, Reach, Scope, Stream};
use super::err::Timeline as Error;
use super::query::Query;
pub(crate) use inner::UserData;
use lru::LruCache;
use warp::reject::Rejection;
@ -11,7 +12,7 @@ mod inner;
type Result<T> = std::result::Result<T, Error>;
#[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)]
pub struct Timeline(pub(crate) Stream, pub(crate) Reach, pub(crate) Content);
pub struct Timeline(pub Stream, pub Reach, pub Content);
impl Timeline {
pub fn empty() -> Self {
@ -61,10 +62,7 @@ impl Timeline {
})
}
pub(crate) fn from_redis_text(
timeline: &str,
cache: &mut LruCache<String, i64>,
) -> Result<Self> {
pub fn from_redis_text(timeline: &str, cache: &mut LruCache<String, i64>) -> Result<Self> {
use {Content::*, Error::*, Reach::*, Stream::*};
let mut tag_id = |t: &str| cache.get(&t.to_string()).map_or(Err(BadTag), |id| Ok(*id));

8
src/request/timeline/inner.rs
View File

@ -5,7 +5,7 @@ use hashbrown::HashSet;
use std::convert::TryFrom;
#[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)]
pub(crate) enum Stream {
pub enum Stream {
User(Id),
List(i64),
Direct(i64),
@ -15,20 +15,20 @@ pub(crate) enum Stream {
}
#[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)]
pub(crate) enum Reach {
pub enum Reach {
Local,
Federated,
}
#[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)]
pub(crate) enum Content {
pub enum Content {
All,
Media,
Notification,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub(crate) enum Scope {
pub enum Scope {
Read,
Statuses,
Notifications,

4
src/response.rs
View File

@ -14,4 +14,6 @@ mod stream;
pub use redis::Error;
#[cfg(feature = "bench")]
pub use redis::{RedisMsg, RedisParseOutput};
pub use event::EventKind;
#[cfg(feature = "bench")]
pub use redis::{Manager, RedisMsg, RedisParseOutput};

7
src/response/event.rs
View File

@ -1,9 +1,12 @@
#[cfg(not(test))]
mod checked_event;
#[cfg(test)]
pub mod checked_event;
mod dynamic_event;
pub mod err;
use self::checked_event::CheckedEvent;
use self::dynamic_event::{DynEvent, EventKind};
pub use self::checked_event::CheckedEvent;
pub use self::dynamic_event::{DynEvent, EventKind};
use crate::Id;
use hashbrown::HashSet;

9
src/response/event/checked_event.rs
View File

@ -1,5 +1,4 @@
mod account;
pub(crate) mod account;
mod announcement;
mod announcement_reaction;
mod conversation;
@ -7,9 +6,9 @@ mod emoji;
mod id;
mod mention;
mod notification;
mod status;
mod tag;
mod visibility;
pub(crate) mod status;
pub(crate) mod tag;
pub(crate) mod visibility;
pub(self) use super::Payload;
pub(super) use announcement_reaction::AnnouncementReaction;

66
src/response/event/checked_event/account.rs
View File

@ -4,47 +4,47 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Account {
pub(crate) struct Account {
pub id: Id,
pub(super) username: String,
pub(crate) username: String,
pub acct: String,
pub(super) url: String,
pub(super) display_name: String,
pub(super) note: String,
pub(super) avatar: String,
pub(super) avatar_static: String,
pub(super) header: String,
pub(super) header_static: String,
pub(super) locked: bool,
pub(super) emojis: Vec<Emoji>,
pub(super) discoverable: Option<bool>, // Shouldn't be option?
pub(super) created_at: String,
pub(super) statuses_count: i64,
pub(super) followers_count: i64,
pub(super) following_count: i64,
pub(super) moved: Option<String>,
pub(super) fields: Option<Vec<Field>>,
pub(super) bot: Option<bool>,
pub(super) source: Option<Source>,
pub(super) group: Option<bool>, // undocumented
pub(super) last_status_at: Option<String>, // undocumented
pub(crate) url: String,
pub(crate) display_name: String,
pub(crate) note: String,
pub(crate) avatar: String,
pub(crate) avatar_static: String,
pub(crate) header: String,
pub(crate) header_static: String,
pub(crate) locked: bool,
pub(crate) emojis: Vec<Emoji>,
pub(crate) discoverable: Option<bool>, // Shouldn't be option?
pub(crate) created_at: String,
pub(crate) statuses_count: i64,
pub(crate) followers_count: i64,
pub(crate) following_count: i64,
pub(crate) moved: Option<String>,
pub(crate) fields: Option<Vec<Field>>,
pub(crate) bot: Option<bool>,
pub(crate) source: Option<Source>,
pub(crate) group: Option<bool>, // undocumented
pub(crate) last_status_at: Option<String>, // undocumented
}
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Field {
pub(super) name: String,
pub(super) value: String,
pub(super) verified_at: Option<String>,
pub(crate) struct Field {
pub(crate) name: String,
pub(crate) value: String,
pub(crate) verified_at: Option<String>,
}
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Source {
pub(super) note: String,
pub(super) fields: Vec<Field>,
pub(super) privacy: Option<Visibility>,
pub(super) sensitive: bool,
pub(super) language: String,
pub(super) follow_requests_count: i64,
pub(crate) struct Source {
pub(crate) note: String,
pub(crate) fields: Vec<Field>,
pub(crate) privacy: Option<Visibility>,
pub(crate) sensitive: bool,
pub(crate) language: String,
pub(crate) follow_requests_count: i64,
}

2
src/response/event/checked_event/emoji.rs
View File

@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Emoji {
pub(crate) struct Emoji {
shortcode: String,
url: String,
static_url: String,

2
src/response/event/checked_event/mention.rs
View File

@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Mention {
pub(crate) struct Mention {
pub id: Id,
username: String,
acct: String,

59
src/response/event/checked_event/status.rs
View File

@ -1,5 +1,5 @@
mod application;
pub(super) mod attachment;
pub(crate) mod attachment;
mod card;
mod poll;
@ -22,37 +22,36 @@ use std::string::String;
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub struct Status {
pub(super) id: Id,
pub(super) uri: String,
pub(super) created_at: String,
pub(super) account: Account,
pub(super) content: String,
pub(super) visibility: Visibility,
pub(super) sensitive: bool,
pub(super) spoiler_text: String,
pub(super) media_attachments: Vec<Attachment>,
pub(super) application: Option<Application>, // Should be non-optional?
pub(super) mentions: Vec<Mention>,
pub(super) tags: Vec<Tag>,
pub(super) emojis: Vec<Emoji>,
pub(super) reblogs_count: i64,
pub(super) favourites_count: i64,
pub(super) replies_count: i64,
pub(super) url: Option<String>,
pub(super) in_reply_to_id: Option<Id>,
pub(super) in_reply_to_account_id: Option<Id>,
pub(super) reblog: Option<Box<Status>>,
pub(super) poll: Option<Poll>,
pub(super) card: Option<Card>,
pub(crate) id: Id,
pub(crate) uri: String,
pub(crate) created_at: String,
pub(crate) account: Account,
pub(crate) content: String,
pub(crate) visibility: Visibility,
pub(crate) sensitive: bool,
pub(crate) spoiler_text: String,
pub(crate) media_attachments: Vec<Attachment>,
pub(crate) application: Option<Application>, // Should be non-optional?
pub(crate) mentions: Vec<Mention>,
pub(crate) tags: Vec<Tag>,
pub(crate) emojis: Vec<Emoji>,
pub(crate) reblogs_count: i64,
pub(crate) favourites_count: i64,
pub(crate) replies_count: i64,
pub(crate) url: Option<String>,
pub(crate) in_reply_to_id: Option<Id>,
pub(crate) in_reply_to_account_id: Option<Id>,
pub(crate) reblog: Option<Box<Status>>,
pub(crate) poll: Option<Poll>,
pub(crate) card: Option<Card>,
pub(crate) language: Option<String>,
pub(super) text: Option<String>,
pub(crate) text: Option<String>,
// ↓↓↓ Only for authorized users
pub(super) favourited: Option<bool>,
pub(super) reblogged: Option<bool>,
pub(super) muted: Option<bool>,
pub(super) bookmarked: Option<bool>,
pub(super) pinned: Option<bool>,
pub(crate) favourited: Option<bool>,
pub(crate) reblogged: Option<bool>,
pub(crate) muted: Option<bool>,
pub(crate) bookmarked: Option<bool>,
pub(crate) pinned: Option<bool>,
}
impl Payload for Status {

12
src/response/event/checked_event/status/application.rs
View File

@ -2,10 +2,10 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(in super::super) struct Application {
pub(super) name: String,
pub(super) website: Option<String>,
pub(super) vapid_key: Option<String>,
pub(super) client_id: Option<String>,
pub(super) client_secret: Option<String>,
pub(crate) struct Application {
pub(crate) name: String,
pub(crate) website: Option<String>,
pub(crate) vapid_key: Option<String>,
pub(crate) client_id: Option<String>,
pub(crate) client_secret: Option<String>,
}

22
src/response/event/checked_event/status/attachment.rs
View File

@ -2,21 +2,21 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(in super::super) struct Attachment {
pub(in super::super) id: String,
pub(in super::super) r#type: AttachmentType,
pub(in super::super) url: String,
pub(in super::super) preview_url: String,
pub(in super::super) remote_url: Option<String>,
pub(in super::super) text_url: Option<String>,
pub(in super::super) meta: Option<serde_json::Value>, // TODO - is this the best type for the API?
pub(in super::super) description: Option<String>,
pub(in super::super) blurhash: Option<String>,
pub(crate) struct Attachment {
pub(crate) id: String,
pub(crate) r#type: AttachmentType,
pub(crate) url: String,
pub(crate) preview_url: String,
pub(crate) remote_url: Option<String>,
pub(crate) text_url: Option<String>,
pub(crate) meta: Option<serde_json::Value>, // TODO - is this the best type for the API?
pub(crate) description: Option<String>,
pub(crate) blurhash: Option<String>,
}
#[serde(rename_all = "lowercase", deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(in super::super) enum AttachmentType {
pub(crate) enum AttachmentType {
Unknown,
Image,
Gifv,

30
src/response/event/checked_event/status/card.rs
View File

@ -2,25 +2,25 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(in super::super) struct Card {
pub(super) url: String,
pub(super) title: String,
pub(super) description: String,
pub(super) r#type: CardType,
pub(super) author_name: Option<String>,
pub(super) author_url: Option<String>,
pub(super) provider_name: Option<String>,
pub(super) provider_url: Option<String>,
pub(super) html: Option<String>,
pub(super) width: Option<i64>,
pub(super) height: Option<i64>,
pub(super) image: Option<String>,
pub(super) embed_url: Option<String>,
pub(crate) struct Card {
pub(crate) url: String,
pub(crate) title: String,
pub(crate) description: String,
pub(crate) r#type: CardType,
pub(crate) author_name: Option<String>,
pub(crate) author_url: Option<String>,
pub(crate) provider_name: Option<String>,
pub(crate) provider_url: Option<String>,
pub(crate) html: Option<String>,
pub(crate) width: Option<i64>,
pub(crate) height: Option<i64>,
pub(crate) image: Option<String>,
pub(crate) embed_url: Option<String>,
}
#[serde(rename_all = "lowercase", deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) enum CardType {
pub(crate) enum CardType {
Link,
Photo,
Video,

28
src/response/event/checked_event/status/poll.rs
View File

@ -3,22 +3,22 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(in super::super) struct Poll {
pub(super) id: String,
pub(super) expires_at: String,
pub(super) expired: bool,
pub(super) multiple: bool,
pub(super) votes_count: i64,
pub(super) voters_count: Option<i64>,
pub(super) voted: Option<bool>,
pub(super) own_votes: Option<Vec<i64>>,
pub(super) options: Vec<PollOptions>,
pub(super) emojis: Vec<Emoji>,
pub(crate) struct Poll {
pub(crate) id: String,
pub(crate) expires_at: String,
pub(crate) expired: bool,
pub(crate) multiple: bool,
pub(crate) votes_count: i64,
pub(crate) voters_count: Option<i64>,
pub(crate) voted: Option<bool>,
pub(crate) own_votes: Option<Vec<i64>>,
pub(crate) options: Vec<PollOptions>,
pub(crate) emojis: Vec<Emoji>,
}
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct PollOptions {
pub(super) title: String,
pub(super) votes_count: Option<i32>,
pub(crate) struct PollOptions {
pub(crate) title: String,
pub(crate) votes_count: Option<i32>,
}

16
src/response/event/checked_event/tag.rs
View File

@ -2,16 +2,16 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Tag {
pub(super) name: String,
pub(super) url: String,
pub(super) history: Option<Vec<History>>,
pub(crate) struct Tag {
pub(crate) name: String,
pub(crate) url: String,
pub(crate) history: Option<Vec<History>>,
}
#[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct History {
pub(super) day: String,
pub(super) uses: String,
pub(super) accounts: String,
pub(crate) struct History {
pub(crate) day: String,
pub(crate) uses: String,
pub(crate) accounts: String,
}

2
src/response/event/checked_event/visibility.rs
View File

@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
#[serde(rename_all = "lowercase", deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) enum Visibility {
pub(crate) enum Visibility {
Public,
Unlisted,
Private,

4
src/response/event/dynamic_event.rs
View File

@ -18,7 +18,7 @@ pub struct DynEvent {
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum EventKind {
pub enum EventKind {
Update(DynStatus),
NonUpdate,
}
@ -30,7 +30,7 @@ impl Default for EventKind {
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct DynStatus {
pub struct DynStatus {
pub(crate) id: Id,
pub(crate) username: String,
pub(crate) language: Option<String>,

2
src/response/redis.rs
View File

@ -13,7 +13,7 @@ pub use msg::{RedisMsg, RedisParseOutput};
use connection::RedisConnErr;
use msg::RedisParseErr;
enum RedisCmd {
pub(crate) enum RedisCmd {
Subscribe,
Unsubscribe,
}

330
src/response/redis/connection.rs
View File

@ -1,149 +1,227 @@
mod err;
pub(crate) use err::RedisConnErr;
pub(super) use connection::*;
pub use err::RedisConnErr;
#[cfg(any(test, feature = "bench"))]
pub(self) use mock_connection as connection;
use super::Error as ManagerErr;
use super::RedisCmd;
use crate::config::Redis;
use crate::request::Timeline;
#[cfg(not(any(test, feature = "bench")))]
mod connection {
use super::super::Error as ManagerErr;
use super::super::RedisCmd;
use super::err::RedisConnErr;
use crate::config::Redis;
use crate::request::Timeline;
use futures::{Async, Poll};
use lru::LruCache;
use std::io::{self, Read, Write};
use std::net::TcpStream;
use std::time::Duration;
use futures::{Async, Poll};
use lru::LruCache;
use std::io::{self, Read, Write};
use std::net::TcpStream;
use std::time::Duration;
type Result<T> = std::result::Result<T, RedisConnErr>;
type Result<T> = std::result::Result<T, RedisConnErr>;
#[derive(Debug)]
pub(super) struct RedisConn {
primary: TcpStream,
secondary: TcpStream,
pub(super) namespace: Option<String>,
// TODO: eventually, it might make sense to have Mastodon publish to timelines with
// the tag number instead of the tag name. This would save us from dealing
// with a cache here and would be consistent with how lists/users are handled.
pub(super) tag_name_cache: LruCache<i64, String>,
pub(super) input: Vec<u8>,
}
impl RedisConn {
pub(super) fn new(redis_cfg: &Redis) -> Result<Self> {
let addr = [&*redis_cfg.host, ":", &*redis_cfg.port.to_string()].concat();
let conn = Self::new_connection(&addr, redis_cfg.password.as_ref())?;
conn.set_nonblocking(true)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
Ok(Self {
primary: conn,
secondary: Self::new_connection(&addr, redis_cfg.password.as_ref())?,
tag_name_cache: LruCache::new(1000),
namespace: redis_cfg.namespace.clone().0,
input: vec![0; 4096 * 4],
})
#[derive(Debug)]
pub struct RedisConn {
primary: TcpStream,
secondary: TcpStream,
pub(in super::super) namespace: Option<String>,
// TODO: eventually, it might make sense to have Mastodon publish to timelines with
// the tag number instead of the tag name. This would save us from dealing
// with a cache here and would be consistent with how lists/users are handled.
pub(in super::super) tag_name_cache: LruCache<i64, String>,
pub(in super::super) input: Vec<u8>,
}
pub(super) fn poll_redis(&mut self, start: usize) -> Poll<usize, ManagerErr> {
const BLOCK: usize = 4096 * 2;
if self.input.len() < start + BLOCK {
self.input.resize(self.input.len() * 2, 0);
log::info!("Resizing input buffer to {} KiB.", self.input.len() / 1024);
// log::info!("Current buffer: {}", String::from_utf8_lossy(&self.input));
impl RedisConn {
pub(in super::super) fn new(redis_cfg: &Redis) -> Result<Self> {
let addr = [&*redis_cfg.host, ":", &*redis_cfg.port.to_string()].concat();
let conn = Self::new_connection(&addr, redis_cfg.password.as_ref())?;
conn.set_nonblocking(true)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
Ok(Self {
primary: conn,
secondary: Self::new_connection(&addr, redis_cfg.password.as_ref())?,
tag_name_cache: LruCache::new(1000),
namespace: redis_cfg.namespace.clone().0,
input: vec![0; 4096 * 4],
})
}
use Async::*;
match self.primary.read(&mut self.input[start..start + BLOCK]) {
Ok(n) => Ok(Ready(n)),
Err(e) if matches!(e.kind(), io::ErrorKind::WouldBlock) => Ok(NotReady),
Err(e) => {
Ready(log::error!("{}", e));
Ok(Ready(0))
pub(in super::super) fn poll_redis(&mut self, i: usize) -> Poll<Option<usize>, ManagerErr> {
const BLOCK: usize = 4096 * 2;
if self.input.len() < i + BLOCK {
self.input.resize(self.input.len() * 2, 0);
log::info!("Resizing input buffer to {} KiB.", self.input.len() / 1024);
// log::info!("Current buffer: {}", String::from_utf8_lossy(&self.input));
}
use Async::*;
match self.primary.read(&mut self.input[i..i + BLOCK]) {
Ok(n) if n == 0 => Ok(Ready(None)),
Ok(n) => Ok(Ready(Some(n))),
Err(e) if matches!(e.kind(), io::ErrorKind::WouldBlock) => Ok(NotReady),
Err(e) => {
Ready(log::error!("{}", e));
Ok(Ready(None))
}
}
}
pub(crate) fn send_cmd(&mut self, cmd: RedisCmd, timelines: &[Timeline]) -> Result<()> {
let namespace = self.namespace.take();
let timelines: Result<Vec<String>> = timelines
.iter()
.map(|tl| {
let hashtag = tl.tag().and_then(|id| self.tag_name_cache.get(&id));
match &namespace {
Some(ns) => Ok(format!("{}:{}", ns, tl.to_redis_raw_timeline(hashtag)?)),
None => Ok(tl.to_redis_raw_timeline(hashtag)?),
}
})
.collect();
let (primary_cmd, secondary_cmd) = cmd.into_sendable(&timelines?[..]);
self.primary.write_all(&primary_cmd)?;
// We also need to set a key to tell the Puma server that we've subscribed or
// unsubscribed to the channel because it stops publishing updates when it thinks
// no one is subscribed.
// (Documented in [PR #3278](https://github.com/tootsuite/mastodon/pull/3278))
// Question: why can't the Puma server just use NUMSUB for this?
self.secondary.write_all(&secondary_cmd)?;
Ok(())
}
fn new_connection(addr: &str, pass: Option<&String>) -> Result<TcpStream> {
let mut conn = TcpStream::connect(&addr)?;
if let Some(password) = pass {
Self::auth_connection(&mut conn, &addr, password)?;
}
Self::validate_connection(&mut conn, &addr)?;
conn.set_read_timeout(Some(Duration::from_millis(10)))
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
Self::set_connection_name(&mut conn, &addr)?;
Ok(conn)
}
fn auth_connection(conn: &mut TcpStream, addr: &str, pass: &str) -> Result<()> {
conn.write_all(
&[
b"*2\r\n$4\r\nauth\r\n$",
pass.len().to_string().as_bytes(),
b"\r\n",
pass.as_bytes(),
b"\r\n",
]
.concat(),
)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 5];
conn.read_exact(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
if String::from_utf8_lossy(&buffer) != "+OK\r\n" {
Err(RedisConnErr::IncorrectPassword(pass.to_string()))?
}
Ok(())
}
fn validate_connection(conn: &mut TcpStream, addr: &str) -> Result<()> {
conn.write_all(b"PING\r\n")
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 100];
conn.read(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let reply = String::from_utf8_lossy(&buffer);
match &*reply {
r if r.starts_with("+PONG\r\n") => Ok(()),
r if r.starts_with("-NOAUTH") => Err(RedisConnErr::MissingPassword),
r if r.starts_with("HTTP/1.") => Err(RedisConnErr::NotRedis(addr.to_string())),
_ => Err(RedisConnErr::InvalidRedisReply(reply.to_string())),
}
}
fn set_connection_name(conn: &mut TcpStream, addr: &str) -> Result<()> {
conn.write_all(b"*3\r\n$6\r\nCLIENT\r\n$7\r\nSETNAME\r\n$8\r\nflodgatt\r\n")
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 100];
conn.read(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let reply = String::from_utf8_lossy(&buffer);
match &*reply {
r if r.starts_with("+OK\r\n") => Ok(()),
_ => Err(RedisConnErr::InvalidRedisReply(reply.to_string())),
}
}
}
}
#[cfg(any(test, feature = "bench"))]
mod mock_connection {
use super::super::Error as ManagerErr;
use super::super::RedisCmd;
use super::err::RedisConnErr;
use crate::config::Redis;
use crate::request::Timeline;
pub(crate) fn send_cmd(&mut self, cmd: RedisCmd, timelines: &[Timeline]) -> Result<()> {
let namespace = self.namespace.take();
let timelines: Result<Vec<String>> = timelines
.iter()
.map(|tl| {
let hashtag = tl.tag().and_then(|id| self.tag_name_cache.get(&id));
match &namespace {
Some(ns) => Ok(format!("{}:{}", ns, tl.to_redis_raw_timeline(hashtag)?)),
None => Ok(tl.to_redis_raw_timeline(hashtag)?),
}
use futures::{Async, Poll};
use lru::LruCache;
use std::collections::VecDeque;
type Result<T> = std::result::Result<T, RedisConnErr>;
#[derive(Debug)]
pub struct RedisConn {
pub(in super::super) namespace: Option<String>,
pub(in super::super) tag_name_cache: LruCache<i64, String>,
pub(in super::super) input: Vec<u8>,
pub(in super::super) test_input: VecDeque<u8>,
}
impl RedisConn {
pub(in super::super) fn new(redis_cfg: &Redis) -> Result<Self> {
Ok(Self {
tag_name_cache: LruCache::new(1000),
namespace: redis_cfg.namespace.clone().0,
input: vec![0; 4096 * 4],
test_input: VecDeque::new(),
})
.collect();
let (primary_cmd, secondary_cmd) = cmd.into_sendable(&timelines?[..]);
self.primary.write_all(&primary_cmd)?;
// We also need to set a key to tell the Puma server that we've subscribed or
// unsubscribed to the channel because it stops publishing updates when it thinks
// no one is subscribed.
// (Documented in [PR #3278](https://github.com/tootsuite/mastodon/pull/3278))
// Question: why can't the Puma server just use NUMSUB for this?
self.secondary.write_all(&secondary_cmd)?;
Ok(())
}
fn new_connection(addr: &str, pass: Option<&String>) -> Result<TcpStream> {
let mut conn = TcpStream::connect(&addr)?;
if let Some(password) = pass {
Self::auth_connection(&mut conn, &addr, password)?;
}
Self::validate_connection(&mut conn, &addr)?;
conn.set_read_timeout(Some(Duration::from_millis(10)))
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
Self::set_connection_name(&mut conn, &addr)?;
Ok(conn)
}
pub fn poll_redis(&mut self, start: usize) -> Poll<Option<usize>, ManagerErr> {
const BLOCK: usize = 4096 * 2;
if self.input.len() < start + BLOCK {
self.input.resize(self.input.len() * 2, 0);
log::info!("Resizing input buffer to {} KiB.", self.input.len() / 1024);
}
fn auth_connection(conn: &mut TcpStream, addr: &str, pass: &str) -> Result<()> {
conn.write_all(
&[
b"*2\r\n$4\r\nauth\r\n$",
pass.len().to_string().as_bytes(),
b"\r\n",
pass.as_bytes(),
b"\r\n",
]
.concat(),
)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 5];
conn.read_exact(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
if String::from_utf8_lossy(&buffer) != "+OK\r\n" {
Err(RedisConnErr::IncorrectPassword(pass.to_string()))?
for i in 0..BLOCK {
if let Some(byte) = self.test_input.pop_front() {
self.input[start + i] = byte;
} else if i > 0 {
return Ok(Async::Ready(Some(i)));
} else {
return Ok(Async::Ready(None));
}
}
Ok(Async::Ready(Some(BLOCK)))
}
Ok(())
}
fn validate_connection(conn: &mut TcpStream, addr: &str) -> Result<()> {
conn.write_all(b"PING\r\n")
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 100];
conn.read(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let reply = String::from_utf8_lossy(&buffer);
match &*reply {
r if r.starts_with("+PONG\r\n") => Ok(()),
r if r.starts_with("-NOAUTH") => Err(RedisConnErr::MissingPassword),
r if r.starts_with("HTTP/1.") => Err(RedisConnErr::NotRedis(addr.to_string())),
_ => Err(RedisConnErr::InvalidRedisReply(reply.to_string())),
pub fn add(&mut self, input: &[u8]) {
for byte in input {
self.test_input.push_back(*byte)
}
}
}
pub(crate) fn send_cmd(&mut self, cmd: RedisCmd, timelines: &[Timeline]) -> Result<()> {
// stub - does nothing; silences some unused-code warnings
let timelines: Result<Vec<String>> = timelines
.iter()
.map(|tl| Ok(tl.to_redis_raw_timeline(None).expect("test")))
.collect();
fn set_connection_name(conn: &mut TcpStream, addr: &str) -> Result<()> {
conn.write_all(b"*3\r\n$6\r\nCLIENT\r\n$7\r\nSETNAME\r\n$8\r\nflodgatt\r\n")
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 100];
conn.read(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let reply = String::from_utf8_lossy(&buffer);
match &*reply {
r if r.starts_with("+OK\r\n") => Ok(()),
_ => Err(RedisConnErr::InvalidRedisReply(reply.to_string())),
let _ = cmd.into_sendable(&timelines?);
Ok(())
}
}
}

150
src/response/redis/connection/connection.rs Normal file
View File

@ -0,0 +1,150 @@
use super::super::Error as ManagerErr;
use super::super::RedisCmd;
use super::RedisConnErr;
use crate::config::Redis;
use crate::request::Timeline;
use futures::{Async, Poll};
use lru::LruCache;
use std::io::{self, Read, Write};
use std::net::TcpStream;
use std::time::Duration;
type Result<T> = std::result::Result<T, RedisConnErr>;
#[derive(Debug)]
pub(in super::super) struct RedisConn {
primary: TcpStream,
secondary: TcpStream,
pub(in super::super) namespace: Option<String>,
// TODO: eventually, it might make sense to have Mastodon publish to timelines with
// the tag number instead of the tag name. This would save us from dealing
// with a cache here and would be consistent with how lists/users are handled.
pub(in super::super) tag_name_cache: LruCache<i64, String>,
}
impl RedisConn {
pub(in super::super) fn new(redis_cfg: &Redis) -> Result<Self> {
let addr = [&*redis_cfg.host, ":", &*redis_cfg.port.to_string()].concat();
let conn = Self::new_connection(&addr, redis_cfg.password.as_ref())?;
conn.set_nonblocking(true)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
Ok(Self {
primary: conn,
secondary: Self::new_connection(&addr, redis_cfg.password.as_ref())?,
tag_name_cache: LruCache::new(1000),
namespace: redis_cfg.namespace.clone().0,
})
}
pub(in super::super) fn poll_redis(
&mut self,
start: usize,
buffer: &mut Vec<u8>,
) -> Poll<usize, ManagerErr> {
const BLOCK: usize = 4096 * 2;
// if self.input.len() < start + BLOCK {
// self.input.resize(self.input.len() * 2, 0);
// log::info!("Resizing input buffer to {} KiB.", self.input.len() / 1024);
// // log::info!("Current buffer: {}", String::from_utf8_lossy(&self.input));
// }
use Async::*;
match self.primary.read(&mut buffer[start..start + BLOCK]) {
Ok(n) => Ok(Ready(n)),
Err(e) if matches!(e.kind(), io::ErrorKind::WouldBlock) => Ok(NotReady),
Err(e) => {
Ready(log::error!("{}", e));
Ok(Ready(0))
}
}
}
pub(crate) fn send_cmd(&mut self, cmd: RedisCmd, timelines: &[Timeline]) -> Result<()> {
let namespace = self.namespace.take();
let timelines: Result<Vec<String>> = timelines
.iter()
.map(|tl| {
let hashtag = tl.tag().and_then(|id| self.tag_name_cache.get(&id));
match &namespace {
Some(ns) => Ok(format!("{}:{}", ns, tl.to_redis_raw_timeline(hashtag)?)),
None => Ok(tl.to_redis_raw_timeline(hashtag)?),
}
})
.collect();
let (primary_cmd, secondary_cmd) = cmd.into_sendable(&timelines?[..]);
self.primary.write_all(&primary_cmd)?;
// We also need to set a key to tell the Puma server that we've subscribed or
// unsubscribed to the channel because it stops publishing updates when it thinks
// no one is subscribed.
// (Documented in [PR #3278](https://github.com/tootsuite/mastodon/pull/3278))
// Question: why can't the Puma server just use NUMSUB for this?
self.secondary.write_all(&secondary_cmd)?;
Ok(())
}
fn new_connection(addr: &str, pass: Option<&String>) -> Result<TcpStream> {
let mut conn = TcpStream::connect(&addr)?;
if let Some(password) = pass {
Self::auth_connection(&mut conn, &addr, password)?;
}
Self::validate_connection(&mut conn, &addr)?;
conn.set_read_timeout(Some(Duration::from_millis(10)))
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
Self::set_connection_name(&mut conn, &addr)?;
Ok(conn)
}
fn auth_connection(conn: &mut TcpStream, addr: &str, pass: &str) -> Result<()> {
conn.write_all(
&[
b"*2\r\n$4\r\nauth\r\n$",
pass.len().to_string().as_bytes(),
b"\r\n",
pass.as_bytes(),
b"\r\n",
]
.concat(),
)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 5];
conn.read_exact(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
if String::from_utf8_lossy(&buffer) != "+OK\r\n" {
Err(RedisConnErr::IncorrectPassword(pass.to_string()))?
}
Ok(())
}
fn validate_connection(conn: &mut TcpStream, addr: &str) -> Result<()> {
conn.write_all(b"PING\r\n")
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 100];
conn.read(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let reply = String::from_utf8_lossy(&buffer);
match &*reply {
r if r.starts_with("+PONG\r\n") => Ok(()),
r if r.starts_with("-NOAUTH") => Err(RedisConnErr::MissingPassword),
r if r.starts_with("HTTP/1.") => Err(RedisConnErr::NotRedis(addr.to_string())),
_ => Err(RedisConnErr::InvalidRedisReply(reply.to_string())),
}
}
fn set_connection_name(conn: &mut TcpStream, addr: &str) -> Result<()> {
conn.write_all(b"*3\r\n$6\r\nCLIENT\r\n$7\r\nSETNAME\r\n$8\r\nflodgatt\r\n")
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 100];
conn.read(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let reply = String::from_utf8_lossy(&buffer);
match &*reply {
r if r.starts_with("+OK\r\n") => Ok(()),
_ => Err(RedisConnErr::InvalidRedisReply(reply.to_string())),
}
}
}

1
src/response/redis/connection/err.rs
View File

@ -13,6 +13,7 @@ pub enum RedisConnErr {
}
impl RedisConnErr {
#[allow(unused)] // Not used during testing due to conditional compilation
pub(super) fn with_addr<T: AsRef<str>>(address: T, inner: std::io::Error) -> Self {
Self::ConnectionErr {
addr: address.as_ref().to_string(),

71
src/response/redis/connection/mock.rs Normal file
View File

@ -0,0 +1,71 @@
pub(crate) use super::RedisConnErr;
use super::super::Error as ManagerErr;
use super::super::RedisCmd;
use crate::config::Redis;
use crate::request::Timeline;
use futures::{Async, Poll};
use lru::LruCache;
type Result<T> = std::result::Result<T, RedisConnErr>;
#[derive(Debug)]
pub(in super::super) struct RedisConn {
pub(in super::super) namespace: Option<String>,
// TODO: eventually, it might make sense to have Mastodon publish to timelines with
// the tag number instead of the tag name. This would save us from dealing
// with a cache here and would be consistent with how lists/users are handled.
pub(in super::super) tag_name_cache: LruCache<i64, String>,
pub(in super::super) input: Vec<u8>,
}
impl RedisConn {
pub(in super::super) fn new(redis_cfg: &Redis) -> Result<Self> {
Ok(Self {
tag_name_cache: LruCache::new(1000),
namespace: redis_cfg.namespace.clone().0,
input: vec![0; 4096 * 4],
})
}
pub(in super::super) fn poll_redis(&mut self, start: usize) -> Poll<usize, ManagerErr> {
const BLOCK: usize = 4096 * 2;
if self.input.len() < start + BLOCK {
self.input.resize(self.input.len() * 2, 0);
log::info!("Resizing input buffer to {} KiB.", self.input.len() / 1024);
// log::info!("Current buffer: {}", String::from_utf8_lossy(&self.input));
}
use Async::*;
// self.input[start..start + BLOCK] = &"foo".as_bytes();
let mut n = 0;
for i in 0..BLOCK {
if let Some(byte) = TEST_INPUT.get(start + i) {
self.input[start + 1] = *byte;
n += 1;
}
}
Ok(Ready(n))
// match self.primary.read(&mut self.input[start..start + BLOCK]) {
// Ok(n) => Ok(Ready(n)),
// Err(e) if matches!(e.kind(), io::ErrorKind::WouldBlock) => Ok(NotReady),
// Err(e) => {
// Ready(log::error!("{}", e));
// Ok(Ready(0))
// }
// }
}
pub(crate) fn send_cmd(&mut self, _cmd: RedisCmd, _timelines: &[Timeline]) -> Result<()> {
Ok(())
}
}
const TEST_INPUT: &[u8] = r##"*3
$7
message
$15
timeline:public
$3790
{"event":"update","payload":{"id":"102775370117886890","created_at":"2019-09-11T18:42:19.000Z","in_reply_to_id":null,"in_reply_to_account_id":null,"sensitive":false,"spoiler_text":"","visibility":"unlisted","language":"en","uri":"https://mastodon.host/users/federationbot/statuses/102775346916917099","url":"https://mastodon.host/@federationbot/102775346916917099","replies_count":0,"reblogs_count":0,"favourites_count":0,"favourited":false,"reblogged":false,"muted":false,"content":"<p>Trending tags:<br><a href=\"https://mastodon.host/tags/neverforget\" class=\"mention hashtag\" rel=\"nofollow noopener\" target=\"_blank\">#<span>neverforget</span></a><br><a href=\"https://mastodon.host/tags/4styles\" class=\"mention hashtag\" rel=\"nofollow noopener\" target=\"_blank\">#<span>4styles</span></a><br><a href=\"https://mastodon.host/tags/newpipe\" class=\"mention hashtag\" rel=\"nofollow noopener\" target=\"_blank\">#<span>newpipe</span></a><br><a href=\"https://mastodon.host/tags/uber\" class=\"mention hashtag\" rel=\"nofollow noopener\" target=\"_blank\">#<span>uber</span></a><br><a href=\"https://mastodon.host/tags/mercredifiction\" class=\"mention hashtag\" rel=\"nofollow noopener\" target=\"_blank\">#<span>mercredifiction</span></a></p>","reblog":null,"account":{"id":"78","username":"federationbot","acct":"federationbot@mastodon.host","display_name":"Federation Bot","locked":false,"bot":false,"created_at":"2019-09-10T15:04:25.559Z","note":"<p>Hello, I am mastodon.host official semi bot.</p><p>Follow me if you want to have some updates on the view of the fediverse from here ( I only post unlisted ). </p><p>I also randomly boost one of my followers toot every hour !</p><p>If you don't feel confortable with me following you, tell me: unfollow and I'll do it :)</p><p>If you want me to follow you, just tell me follow ! </p><p>If you want automatic follow for new users on your instance and you are an instance admin, contact me !</p><p>Other commands are private :)</p>","url":"https://mastodon.host/@federationbot","avatar":"https://instance.codesections.com/system/accounts/avatars/000/000/078/original/d9e2be5398629cf8.jpeg?1568127863","avatar_static":"https://instance.codesections.com/system/accounts/avatars/000/000/078/original/d9e2be5398629cf8.jpeg?1568127863","header":"https://instance.codesections.com/headers/original/missing.png","header_static":"https://instance.codesections.com/headers/original/missing.png","followers_count":16636,"following_count":179532,"statuses_count":50554,"emojis":[],"fields":[{"name":"More stats","value":"<a href=\"https://mastodon.host/stats.html\" rel=\"nofollow noopener\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"\">mastodon.host/stats.html</span><span class=\"invisible\"></span></a>","verified_at":null},{"name":"More infos","value":"<a href=\"https://mastodon.host/about/more\" rel=\"nofollow noopener\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"\">mastodon.host/about/more</span><span class=\"invisible\"></span></a>","verified_at":null},{"name":"Owner/Friend","value":"<span class=\"h-card\"><a href=\"https://mastodon.host/@gled\" class=\"u-url mention\" rel=\"nofollow noopener\" target=\"_blank\">@<span>gled</span></a></span>","verified_at":null}]},"media_attachments":[],"mentions":[],"tags":[{"name":"4styles","url":"https://instance.codesections.com/tags/4styles"},{"name":"neverforget","url":"https://instance.codesections.com/tags/neverforget"},{"name":"mercredifiction","url":"https://instance.codesections.com/tags/mercredifiction"},{"name":"uber","url":"https://instance.codesections.com/tags/uber"},{"name":"newpipe","url":"https://instance.codesections.com/tags/newpipe"}],"emojis":[],"card":null,"poll":null},"queued_at":1568227693541}"##.as_bytes();

141
src/response/redis/manager.rs
View File

@ -25,73 +25,111 @@ type EventChannel = Sender<Arc<Event>>;
/// The item that streams from Redis and is polled by the `ClientAgent`
pub struct Manager {
redis_conn: RedisConn,
pub redis_conn: RedisConn,
timelines: HashMap<Timeline, HashMap<u32, EventChannel>>,
ping_time: Instant,
channel_id: u32,
unread_idx: (usize, usize),
pub unread_idx: (usize, usize),
tag_id_cache: LruCache<String, i64>,
}
impl Stream for Manager {
type Item = ();
type Item = (Timeline, Arc<Event>);
type Error = Error;
fn poll(&mut self) -> Poll<Option<()>, Error> {
if self.ping_time.elapsed() > Duration::from_secs(30) {
self.send_pings()?
}
while let Async::Ready(msg_len) = self.redis_conn.poll_redis(self.unread_idx.1)? {
self.unread_idx = (0, self.unread_idx.1 + msg_len);
let input = &self.redis_conn.input[..self.unread_idx.1];
let mut unread = str::from_utf8(input).unwrap_or_else(|e| {
str::from_utf8(input.split_at(e.valid_up_to()).0).expect("guaranteed by `split_at`")
fn poll(&mut self) -> Poll<Option<Self::Item>, Error> {
let input = &self.redis_conn.input[self.unread_idx.0..self.unread_idx.1];
let (valid, invalid) = str::from_utf8(input)
.map(|v| (v, &b""[..]))
.unwrap_or_else(|e| {
// NOTE - this bounds check occurs more often than necessary; it could occur only when
// polling Redis. However, benchmarking with Criterion shows it to be *very*
// inexpensive (<1 us) and thus not worth removing (doing so would require `unsafe`).
let (valid, invalid) = input.split_at(e.valid_up_to());
(str::from_utf8(valid).expect("split_at"), invalid)
});
while !unread.is_empty() {
use RedisParseOutput::*;
match RedisParseOutput::try_from(unread) {
Ok(Msg(msg)) => {
// If we get a message and it matches the redis_namespace, get the msg's
// Event and send it to all channels matching the msg's Timeline
if let Some(tl) = msg.timeline_matching_ns(&self.redis_conn.namespace) {
let tl = Timeline::from_redis_text(tl, &mut self.tag_id_cache)?;
let event: Arc<Event> = Arc::new(msg.event_txt.try_into()?);
if let Some(channels) = self.timelines.get_mut(&tl) {
for channel in channels.values_mut() {
if let Ok(Async::NotReady) = channel.poll_ready() {
log::warn!("{:?} channel full\ncan't send:{:?}", tl, event);
return Ok(Async::NotReady);
}
let _ = channel.try_send(event.clone()); // err just means channel will be closed
}
}
}
unread = msg.leftover_input;
self.unread_idx.0 = self.unread_idx.1 - unread.len();
if !valid.is_empty() {
use RedisParseOutput::*;
match RedisParseOutput::try_from(valid) {
Ok(Msg(msg)) => {
// If we get a message and it matches the redis_namespace, get the msg's
// Event and send it to all channels matching the msg's Timeline
if let Some(tl) = msg.timeline_matching_ns(&self.redis_conn.namespace) {
self.unread_idx.0 =
self.unread_idx.1 - msg.leftover_input.len() - invalid.len();
let tl = Timeline::from_redis_text(tl, &mut self.tag_id_cache)?;
let event: Arc<Event> = Arc::new(msg.event_txt.try_into()?);
Ok(Async::Ready(Some((tl, event))))
} else {
Ok(Async::Ready(None))
}
Ok(NonMsg(leftover_input)) => {
unread = leftover_input;
self.unread_idx.0 = self.unread_idx.1 - unread.len();
}
Err(RedisParseErr::Incomplete) => {
self.copy_partial_msg();
break;
}
Err(e) => Err(Error::RedisParseErr(e, unread.to_string()))?,
};
}
if self.unread_idx.0 == self.unread_idx.1 {
self.unread_idx = (0, 0)
}
Ok(NonMsg(leftover_input)) => {
self.unread_idx.0 = self.unread_idx.1 - leftover_input.len();
Ok(Async::Ready(None))
}
Err(RedisParseErr::Incomplete) => {
self.copy_partial_msg();
Ok(Async::NotReady)
}
Err(e) => Err(Error::RedisParseErr(e, valid.to_string()))?,
}
} else {
self.unread_idx = (0, 0);
Ok(Async::NotReady)
}
Ok(Async::Ready(Some(())))
}
}
impl Manager {
// untested
pub fn send_msgs(&mut self) -> Poll<(), Error> {
if self.ping_time.elapsed() > Duration::from_secs(30) {
self.send_pings()?
}
while let Ok(Async::Ready(Some(msg_len))) = self.redis_conn.poll_redis(self.unread_idx.1) {
self.unread_idx.1 += msg_len;
while let Ok(Async::Ready(msg)) = self.poll() {
if let Some((tl, event)) = msg {
for channel in self.timelines.entry(tl).or_default().values_mut() {
if let Ok(Async::NotReady) = channel.poll_ready() {
log::warn!("{:?} channel full\ncan't send:{:?}", tl, event);
self.rewind_to_prev_msg();
return Ok(Async::NotReady);
}
let _ = channel.try_send(event.clone()); // err just means channel will be closed
}
}
}
}
Ok(Async::Ready(()))
}
fn rewind_to_prev_msg(&mut self) {
self.unread_idx.0 = loop {
let input = &self.redis_conn.input[..self.unread_idx.0];
let input = str::from_utf8(input).unwrap_or_else(|e| {
str::from_utf8(input.split_at(e.valid_up_to()).0).expect("guaranteed by `split_at`")
});
let index = if let Some(i) = input.rfind("\r\n*") {
i + "\r\n".len()
} else {
0
};
self.unread_idx.0 = index;
if let Ok(Async::Ready(Some(_))) = self.poll() {
break index;
}
}
}
fn copy_partial_msg(&mut self) {
if self.unread_idx.0 == 0 {
// msg already first; no copying needed
@ -107,7 +145,7 @@ impl Manager {
self.redis_conn.input = self.redis_conn.input[self.unread_idx.0..].into();
}
self.unread_idx = (0, self.unread_idx.1 - self.unread_idx.0);
dbg!(&self.unread_idx);
&self.unread_idx;
}
/// Create a new `Manager`, with its own Redis connections (but no active subscriptions).
pub fn try_from(redis_cfg: &config::Redis) -> Result<Self> {
@ -208,3 +246,6 @@ impl Manager {
.collect()
}
}
#[cfg(test)]
mod test;

245
src/response/redis/manager/test.rs Normal file
View File

@ -0,0 +1,245 @@
use super::*;
use crate::config;
use crate::response::event::checked_event::{
account::{Account, Field},
status::attachment::{Attachment, AttachmentType::*},
status::Status,
tag::Tag,
visibility::Visibility::*,
CheckedEvent::*,
};
use crate::Id;
use serde_json::json;
use std::fs;
type TestResult = std::result::Result<(), Box<dyn std::error::Error>>;
fn input(i: usize) -> Vec<u8> {
fs::read_to_string(format!("test_data/redis_input_{:03}.resp", i))
.expect("test input not found")
.as_bytes()
.to_vec()
}
fn output(i: usize) -> Arc<Event> {
vec![
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_001.rs"
))),
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_002.rs"
))),
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_003.rs"
))),
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_004.rs"
))),
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_005.rs"
))),
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_006.rs"
))),
][i]
.clone()
}
#[test]
fn manager_poll_matches_six_events() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
for i in 1..=6 {
manager.redis_conn.add(&input(i));
}
let mut i = 0;
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx = (0, manager.unread_idx.1 + len);
while let Ok(Async::Ready(Some((_tl, event)))) = manager.poll() {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
Ok(assert_eq!(i, 6))
}
#[test]
fn manager_poll_handles_non_utf8() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
let mut input_txt = Vec::new();
for i in 1..=6 {
input_txt.extend_from_slice(&input(i))
}
let invalid_idx = str::from_utf8(&input_txt)?
.chars()
.take_while(|char| char.len_utf8() == 1)
.collect::<Vec<_>>()
.len()
+ 1;
manager.redis_conn.add(&input_txt[..invalid_idx]);
let mut i = 0;
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(Some((_tl, event)))) = manager.poll() {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
manager.redis_conn.add(&input_txt[invalid_idx..]);
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(Some((_tl, event)))) = manager.poll() {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
Ok(assert_eq!(i, 6))
}
#[test]
fn manager_poll_matches_six_events_in_batches() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
for i in 1..=3 {
manager.redis_conn.add(&input(i))
}
let mut i = 0;
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(Some((_tl, event)))) = manager.poll() {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
for i in 4..=6 {
manager.redis_conn.add(&input(i));
}
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(Some((_tl, event)))) = manager.poll() {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
Ok(assert_eq!(i, 6))
}
#[test]
fn manager_poll_handles_non_events() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
for i in 1..=6 {
manager.redis_conn.add(&input(i));
manager
.redis_conn
.add(b"*3\r\n$9\r\nsubscribe\r\n$12\r\ntimeline:308\r\n:1\r\n");
}
let mut i = 0;
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(msg)) = manager.poll() {
if let Some((_tl, event)) = msg {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
}
Ok(assert_eq!(i, 6))
}
#[test]
fn manager_poll_handles_partial_events() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
for i in 1..=3 {
manager.redis_conn.add(&input(i));
}
manager.redis_conn.add(&input(4)[..50]);
let mut i = 0;
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(msg)) = manager.poll() {
if let Some((_tl, event)) = msg {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
}
assert_eq!(i, 3);
manager.redis_conn.add(&input(4)[50..]);
manager.redis_conn.add(&input(5));
manager.redis_conn.add(&input(6));
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(msg)) = manager.poll() {
if let Some((_tl, event)) = msg {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
}
Ok(assert_eq!(i, 6))
}
#[test]
fn manager_poll_handles_full_channel() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
for i in 1..=6 {
manager.redis_conn.add(&input(i));
}
let (mut i, channel_full) = (0, 3);
'outer: loop {
while let Ok(Async::Ready(Some(n))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += n;
while let Ok(Async::Ready(msg)) = manager.poll() {
if let Some((_tl, event)) = msg {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
// Simulates a `ChannelFull` error after sending `channel_full` msgs
if i == channel_full {
break 'outer;
}
}
}
}
let _rewind = (|| {
manager.rewind_to_prev_msg();
i -= 1;
})();
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(msg)) = manager.poll() {
if let Some((_tl, event)) = msg {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
}
Ok(assert_eq!(i, 6))
}

8
src/response/redis/msg.rs
View File

@ -27,15 +27,15 @@ use std::str;
mod err;
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum RedisParseOutput<'a> {
pub enum RedisParseOutput<'a> {
Msg(RedisMsg<'a>),
NonMsg(&'a str),
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct RedisMsg<'a> {
pub(crate) timeline_txt: &'a str,
pub(crate) event_txt: &'a str,
pub struct RedisMsg<'a> {
pub timeline_txt: &'a str,
pub event_txt: &'a str,
pub(crate) leftover_input: &'a str,
}

17
src/response/redis/msg/err.rs
View File

@ -15,8 +15,8 @@ impl fmt::Display for RedisParseErr {
use RedisParseErr::*;
let msg = match self {
Incomplete => "The input from Redis does not form a complete message, likely because \
the input buffer filled partway through a message. Save this input \
and try again with additional input from Redis."
the input buffer filled partway through a message. Save this input and try again \
with additional input from Redis."
.to_string(),
InvalidNumber(parse_int_err) => format!(
"Redis indicated that an item would be a number, but it could not be parsed: {}",
@ -25,20 +25,19 @@ impl fmt::Display for RedisParseErr {
InvalidLineStart(line_start_char) => format!(
"A line from Redis started with `{}`, which is not a valid character to indicate \
the type of the Redis line.",
the type of the Redis line.",
line_start_char
),
InvalidLineEnd(len, line) => format!( // TODO - FIXME
"A Redis line did not have the promised length of {}. \
The line is: {}",
InvalidLineEnd(len, line) => format!(
"A Redis line did not have the promised length of {}. The line is: {}",
len, line
),
IncorrectRedisType => "Received a Redis type that is not supported in this context. \
Flodgatt expects each message from Redis to be a Redis array \
consisting of bulk strings or integers."
Flodgatt expects each message from Redis to be a Redis array consisting of bulk \
strings or integers."
.to_string(),
MissingField => "Redis input was missing a field Flodgatt expected (e.g., a `message` \
without a payload line)"
without a payload line)"
.to_string(),
};
write!(f, "{}", msg)