zelo72
/
flodgatt
mirror of https://github.com/mastodon/flodgatt
1
0
Fork 0
Code Issues Releases Wiki Activity

Add tests for polling for multiple messages (#149) 

* Add tests for polling for multiple messages

This commit adds a mock Redis interface and adds tests that poll the
mock interface for multiple messages at a time.  These tests test that
Flodgatt is robust against receiving incomplete messages, including if
the message break results in receiving invalid UTF8.

* Remove temporary files
This commit is contained in:
Daniel Sockwell 2020-05-07 10:56:11 -04:00 committed by GitHub
parent 4de9a94230
commit daf7d1ae7f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 786 additions and 371 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -416,7 +416,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "flodgatt" name = "flodgatt"
version = "0.9.8" version = "0.9.9"
dependencies = [ dependencies = [
"criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"dotenv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "dotenv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",

2
Cargo.toml
View File

@ -1,7 +1,7 @@
[package] [package]
name = "flodgatt" name = "flodgatt"
description = "A blazingly fast drop-in replacement for the Mastodon streaming api server" description = "A blazingly fast drop-in replacement for the Mastodon streaming api server"
version = "0.9.8" version = "0.9.9"
authors = ["Daniel Long Sockwell <daniel@codesections.com", "Julian Laubstein <contact@julianlaubstein.de>"] authors = ["Daniel Long Sockwell <daniel@codesections.com", "Julian Laubstein <contact@julianlaubstein.de>"]
edition = "2018" edition = "2018"

111
benches/parse_redis.rs
View File

@ -1,7 +1,12 @@
use criterion::{black_box, criterion_group, criterion_main, Criterion}; use criterion::{black_box, criterion_group, criterion_main, Criterion};
use flodgatt::response::{RedisMsg, RedisParseOutput}; use flodgatt::config;
use flodgatt::request::{Content::*, Reach::*, Stream::*, Timeline};
use flodgatt::response::{Event, Manager, RedisMsg, RedisParseOutput};
use flodgatt::Id;
use futures::{Async, Stream};
use lru::LruCache; use lru::LruCache;
use std::convert::TryFrom; use std::convert::TryFrom;
use std::fs;
fn parse_long_redis_input<'a>(input: &'a str) -> RedisMsg<'a> { fn parse_long_redis_input<'a>(input: &'a str) -> RedisMsg<'a> {
if let RedisParseOutput::Msg(msg) = RedisParseOutput::try_from(input).unwrap() { if let RedisParseOutput::Msg(msg) = RedisParseOutput::try_from(input).unwrap() {
@ -12,27 +17,34 @@ fn parse_long_redis_input<'a>(input: &'a str) -> RedisMsg<'a> {
} }
} }
// fn parse_to_timeline(msg: RedisMsg) -> Timeline { fn parse_to_timeline(msg: RedisMsg) -> Timeline {
// let trimmed_tl_txt = &msg.timeline_txt["timeline:".len()..]; let trimmed_tl_txt = &msg.timeline_txt["timeline:".len()..];
// let tl = Timeline::from_redis_text(trimmed_tl_txt, &mut LruCache::new(1000)).unwrap(); let tl = Timeline::from_redis_text(trimmed_tl_txt, &mut LruCache::new(1000)).unwrap();
// assert_eq!(tl, Timeline(User(Id(1)), Federated, All)); assert_eq!(tl, Timeline(User(Id(1)), Federated, All));
// tl tl
// } }
// fn parse_to_checked_event(msg: RedisMsg) -> EventKind { fn parse_to_checked_event(msg: RedisMsg) -> Event {
// EventKind::TypeSafe(serde_json::from_str(msg.event_txt).unwrap()) Event::TypeSafe(serde_json::from_str(msg.event_txt).unwrap())
// } }
// fn parse_to_dyn_event(msg: RedisMsg) -> EventKind { fn parse_to_dyn_event(msg: RedisMsg) -> Event {
// EventKind::Dynamic(serde_json::from_str(msg.event_txt).unwrap()) Event::Dynamic(serde_json::from_str(msg.event_txt).unwrap())
// } }
// fn redis_msg_to_event_string(msg: RedisMsg) -> String { fn redis_msg_to_event_string(msg: RedisMsg) -> String {
// msg.event_txt.to_string() msg.event_txt.to_string()
// } }
// fn string_to_checked_event(event_txt: &String) -> EventKind { fn string_to_checked_event(event_txt: &String) -> Event {
// EventKind::TypeSafe(serde_json::from_str(event_txt).unwrap()) Event::TypeSafe(serde_json::from_str(event_txt).unwrap())
// } }
fn input_msg(i: usize) -> Vec<u8> {
fs::read_to_string(format!("test_data/redis_input_{:03}.resp", i))
.expect("test input not found")
.as_bytes()
.to_vec()
}
fn criterion_benchmark(c: &mut Criterion) { fn criterion_benchmark(c: &mut Criterion) {
let input = ONE_MESSAGE_FOR_THE_USER_TIMLINE_FROM_REDIS; let input = ONE_MESSAGE_FOR_THE_USER_TIMLINE_FROM_REDIS;
@ -42,25 +54,54 @@ fn criterion_benchmark(c: &mut Criterion) {
b.iter(|| black_box(parse_long_redis_input(input))) b.iter(|| black_box(parse_long_redis_input(input)))
}); });
// let msg = parse_long_redis_input(input); let msg = parse_long_redis_input(input);
// group.bench_function("parse RedisMsg to Timeline", |b| { group.bench_function("parse RedisMsg to Timeline", |b| {
// b.iter(|| black_box(parse_to_timeline(msg.clone()))) b.iter(|| black_box(parse_to_timeline(msg.clone())))
// }); });
// group.bench_function("parse RedisMsg -> DynamicEvent", |b| { group.bench_function("parse RedisMsg -> CheckedEvent", |b| {
// b.iter(|| black_box(parse_to_dyn_event(msg.clone()))) b.iter(|| black_box(parse_to_checked_event(msg.clone())))
// }); });
// group.bench_function("parse RedisMsg -> CheckedEvent", |b| { group.bench_function("parse RedisMsg -> DynamicEvent", |b| {
// b.iter(|| black_box(parse_to_checked_event(msg.clone()))) b.iter(|| black_box(parse_to_dyn_event(msg.clone())))
// }); });
// group.bench_function("parse RedisMsg -> String -> CheckedEvent", |b| { group.bench_function("parse RedisMsg -> String -> CheckedEvent", |b| {
// b.iter(|| { b.iter(|| {
// let txt = black_box(redis_msg_to_event_string(msg.clone())); let txt = black_box(redis_msg_to_event_string(msg.clone()));
// black_box(string_to_checked_event(&txt)); black_box(string_to_checked_event(&txt));
// }) })
// }); });
group.bench_function("parse six messages from Redis", |b| {
b.iter_batched(
|| {
let mut manager = Manager::try_from(&config::Redis::default()).expect("bench");
for i in 1..=6 {
manager.redis_conn.add(&input_msg(i));
}
manager
},
|mut m| {
black_box({
let mut i = 1;
while let Ok(Async::Ready(Some(len))) = m.redis_conn.poll_redis(m.unread_idx.1)
{
m.unread_idx = (0, m.unread_idx.1 + len);
while let Ok(Async::Ready(Some((_tl, event)))) = m.poll() {
// println!("Parsing Event #{:03}", i + 1);
// assert_eq!(event, output(i));
i += 1;
}
}
assert_eq!(i, 7)
})
},
criterion::BatchSize::SmallInput,
)
});
} }
criterion_group!(benches, criterion_benchmark); criterion_group!(benches, criterion_benchmark);

12
src/main.rs
View File

@ -83,12 +83,16 @@ fn main() -> Result<(), Error> {
let manager = shared_manager.clone(); let manager = shared_manager.clone();
let stream = Interval::new(Instant::now(), poll_freq) let stream = Interval::new(Instant::now(), poll_freq)
.map_err(|e| log::error!("{}", e)) .map_err(|e| log::error!("{}", e))
.for_each( .for_each(move |_| {
move |_| match manager.lock().unwrap_or_else(RedisManager::recover).poll() { match manager
.lock()
.unwrap_or_else(RedisManager::recover)
.send_msgs()
{
Err(e) => Ok(log::error!("{}", e)), Err(e) => Ok(log::error!("{}", e)),
Ok(_) => Ok(()), Ok(_) => Ok(()),
}, }
); });
warp::spawn(lazy(move || stream)); warp::spawn(lazy(move || stream));
warp::serve(ws.or(sse).with(cors).or(status).recover(Handler::err)) warp::serve(ws.or(sse).with(cors).or(status).recover(Handler::err))

5
src/request.rs
View File

@ -9,6 +9,11 @@ mod subscription;
pub use err::{Error, Timeline as TimelineErr}; pub use err::{Error, Timeline as TimelineErr};
pub use subscription::{Blocks, Subscription}; pub use subscription::{Blocks, Subscription};
pub use timeline::Timeline; pub use timeline::Timeline;
#[cfg(feature = "bench")]
pub use timeline::{Content, Reach, Stream};
#[cfg(not(feature = "bench"))]
use timeline::{Content, Reach, Stream}; use timeline::{Content, Reach, Stream};
pub use self::postgres::PgPool; pub use self::postgres::PgPool;

10
src/request/timeline.rs
View File

@ -1,6 +1,7 @@
pub(crate) use self::inner::{Content, Reach, Scope, Stream, UserData}; pub use self::inner::{Content, Reach, Scope, Stream};
use super::err::Timeline as Error; use super::err::Timeline as Error;
use super::query::Query; use super::query::Query;
pub(crate) use inner::UserData;
use lru::LruCache; use lru::LruCache;
use warp::reject::Rejection; use warp::reject::Rejection;
@ -11,7 +12,7 @@ mod inner;
type Result<T> = std::result::Result<T, Error>; type Result<T> = std::result::Result<T, Error>;
#[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)] #[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)]
pub struct Timeline(pub(crate) Stream, pub(crate) Reach, pub(crate) Content); pub struct Timeline(pub Stream, pub Reach, pub Content);
impl Timeline { impl Timeline {
pub fn empty() -> Self { pub fn empty() -> Self {
@ -61,10 +62,7 @@ impl Timeline {
}) })
} }
pub(crate) fn from_redis_text( pub fn from_redis_text(timeline: &str, cache: &mut LruCache<String, i64>) -> Result<Self> {
timeline: &str,
cache: &mut LruCache<String, i64>,
) -> Result<Self> {
use {Content::*, Error::*, Reach::*, Stream::*}; use {Content::*, Error::*, Reach::*, Stream::*};
let mut tag_id = |t: &str| cache.get(&t.to_string()).map_or(Err(BadTag), |id| Ok(*id)); let mut tag_id = |t: &str| cache.get(&t.to_string()).map_or(Err(BadTag), |id| Ok(*id));

8
src/request/timeline/inner.rs
View File

@ -5,7 +5,7 @@ use hashbrown::HashSet;
use std::convert::TryFrom; use std::convert::TryFrom;
#[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)] #[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)]
pub(crate) enum Stream { pub enum Stream {
User(Id), User(Id),
List(i64), List(i64),
Direct(i64), Direct(i64),
@ -15,20 +15,20 @@ pub(crate) enum Stream {
} }
#[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)] #[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)]
pub(crate) enum Reach { pub enum Reach {
Local, Local,
Federated, Federated,
} }
#[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)] #[derive(Clone, Debug, Copy, Eq, Hash, PartialEq)]
pub(crate) enum Content { pub enum Content {
All, All,
Media, Media,
Notification, Notification,
} }
#[derive(Clone, Debug, PartialEq, Eq, Hash)] #[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub(crate) enum Scope { pub enum Scope {
Read, Read,
Statuses, Statuses,
Notifications, Notifications,

4
src/response.rs
View File

@ -14,4 +14,6 @@ mod stream;
pub use redis::Error; pub use redis::Error;
#[cfg(feature = "bench")] #[cfg(feature = "bench")]
pub use redis::{RedisMsg, RedisParseOutput}; pub use event::EventKind;
#[cfg(feature = "bench")]
pub use redis::{Manager, RedisMsg, RedisParseOutput};

7
src/response/event.rs
View File

@ -1,9 +1,12 @@
#[cfg(not(test))]
mod checked_event; mod checked_event;
#[cfg(test)]
pub mod checked_event;
mod dynamic_event; mod dynamic_event;
pub mod err; pub mod err;
use self::checked_event::CheckedEvent; pub use self::checked_event::CheckedEvent;
use self::dynamic_event::{DynEvent, EventKind}; pub use self::dynamic_event::{DynEvent, EventKind};
use crate::Id; use crate::Id;
use hashbrown::HashSet; use hashbrown::HashSet;

9
src/response/event/checked_event.rs
View File

@ -1,5 +1,4 @@
mod account; pub(crate) mod account;
mod announcement; mod announcement;
mod announcement_reaction; mod announcement_reaction;
mod conversation; mod conversation;
@ -7,9 +6,9 @@ mod emoji;
mod id; mod id;
mod mention; mod mention;
mod notification; mod notification;
mod status; pub(crate) mod status;
mod tag; pub(crate) mod tag;
mod visibility; pub(crate) mod visibility;
pub(self) use super::Payload; pub(self) use super::Payload;
pub(super) use announcement_reaction::AnnouncementReaction; pub(super) use announcement_reaction::AnnouncementReaction;

66
src/response/event/checked_event/account.rs
View File

@ -4,47 +4,47 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Account { pub(crate) struct Account {
pub id: Id, pub id: Id,
pub(super) username: String, pub(crate) username: String,
pub acct: String, pub acct: String,
pub(super) url: String, pub(crate) url: String,
pub(super) display_name: String, pub(crate) display_name: String,
pub(super) note: String, pub(crate) note: String,
pub(super) avatar: String, pub(crate) avatar: String,
pub(super) avatar_static: String, pub(crate) avatar_static: String,
pub(super) header: String, pub(crate) header: String,
pub(super) header_static: String, pub(crate) header_static: String,
pub(super) locked: bool, pub(crate) locked: bool,
pub(super) emojis: Vec<Emoji>, pub(crate) emojis: Vec<Emoji>,
pub(super) discoverable: Option<bool>, // Shouldn't be option? pub(crate) discoverable: Option<bool>, // Shouldn't be option?
pub(super) created_at: String, pub(crate) created_at: String,
pub(super) statuses_count: i64, pub(crate) statuses_count: i64,
pub(super) followers_count: i64, pub(crate) followers_count: i64,
pub(super) following_count: i64, pub(crate) following_count: i64,
pub(super) moved: Option<String>, pub(crate) moved: Option<String>,
pub(super) fields: Option<Vec<Field>>, pub(crate) fields: Option<Vec<Field>>,
pub(super) bot: Option<bool>, pub(crate) bot: Option<bool>,
pub(super) source: Option<Source>, pub(crate) source: Option<Source>,
pub(super) group: Option<bool>, // undocumented pub(crate) group: Option<bool>, // undocumented
pub(super) last_status_at: Option<String>, // undocumented pub(crate) last_status_at: Option<String>, // undocumented
} }
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Field { pub(crate) struct Field {
pub(super) name: String, pub(crate) name: String,
pub(super) value: String, pub(crate) value: String,
pub(super) verified_at: Option<String>, pub(crate) verified_at: Option<String>,
} }
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Source { pub(crate) struct Source {
pub(super) note: String, pub(crate) note: String,
pub(super) fields: Vec<Field>, pub(crate) fields: Vec<Field>,
pub(super) privacy: Option<Visibility>, pub(crate) privacy: Option<Visibility>,
pub(super) sensitive: bool, pub(crate) sensitive: bool,
pub(super) language: String, pub(crate) language: String,
pub(super) follow_requests_count: i64, pub(crate) follow_requests_count: i64,
} }

2
src/response/event/checked_event/emoji.rs
View File

@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Emoji { pub(crate) struct Emoji {
shortcode: String, shortcode: String,
url: String, url: String,
static_url: String, static_url: String,

2
src/response/event/checked_event/mention.rs
View File

@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Mention { pub(crate) struct Mention {
pub id: Id, pub id: Id,
username: String, username: String,
acct: String, acct: String,

59
src/response/event/checked_event/status.rs
View File

@ -1,5 +1,5 @@
mod application; mod application;
pub(super) mod attachment; pub(crate) mod attachment;
mod card; mod card;
mod poll; mod poll;
@ -22,37 +22,36 @@ use std::string::String;
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub struct Status { pub struct Status {
pub(super) id: Id, pub(crate) id: Id,
pub(super) uri: String, pub(crate) uri: String,
pub(super) created_at: String, pub(crate) created_at: String,
pub(super) account: Account, pub(crate) account: Account,
pub(super) content: String, pub(crate) content: String,
pub(super) visibility: Visibility, pub(crate) visibility: Visibility,
pub(super) sensitive: bool, pub(crate) sensitive: bool,
pub(super) spoiler_text: String, pub(crate) spoiler_text: String,
pub(super) media_attachments: Vec<Attachment>, pub(crate) media_attachments: Vec<Attachment>,
pub(super) application: Option<Application>, // Should be non-optional? pub(crate) application: Option<Application>, // Should be non-optional?
pub(super) mentions: Vec<Mention>, pub(crate) mentions: Vec<Mention>,
pub(super) tags: Vec<Tag>, pub(crate) tags: Vec<Tag>,
pub(super) emojis: Vec<Emoji>, pub(crate) emojis: Vec<Emoji>,
pub(super) reblogs_count: i64, pub(crate) reblogs_count: i64,
pub(super) favourites_count: i64, pub(crate) favourites_count: i64,
pub(super) replies_count: i64, pub(crate) replies_count: i64,
pub(super) url: Option<String>, pub(crate) url: Option<String>,
pub(super) in_reply_to_id: Option<Id>, pub(crate) in_reply_to_id: Option<Id>,
pub(super) in_reply_to_account_id: Option<Id>, pub(crate) in_reply_to_account_id: Option<Id>,
pub(super) reblog: Option<Box<Status>>, pub(crate) reblog: Option<Box<Status>>,
pub(super) poll: Option<Poll>, pub(crate) poll: Option<Poll>,
pub(super) card: Option<Card>, pub(crate) card: Option<Card>,
pub(crate) language: Option<String>, pub(crate) language: Option<String>,
pub(crate) text: Option<String>,
pub(super) text: Option<String>,
// ↓↓↓ Only for authorized users // ↓↓↓ Only for authorized users
pub(super) favourited: Option<bool>, pub(crate) favourited: Option<bool>,
pub(super) reblogged: Option<bool>, pub(crate) reblogged: Option<bool>,
pub(super) muted: Option<bool>, pub(crate) muted: Option<bool>,
pub(super) bookmarked: Option<bool>, pub(crate) bookmarked: Option<bool>,
pub(super) pinned: Option<bool>, pub(crate) pinned: Option<bool>,
} }
impl Payload for Status { impl Payload for Status {

12
src/response/event/checked_event/status/application.rs
View File

@ -2,10 +2,10 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(in super::super) struct Application { pub(crate) struct Application {
pub(super) name: String, pub(crate) name: String,
pub(super) website: Option<String>, pub(crate) website: Option<String>,
pub(super) vapid_key: Option<String>, pub(crate) vapid_key: Option<String>,
pub(super) client_id: Option<String>, pub(crate) client_id: Option<String>,
pub(super) client_secret: Option<String>, pub(crate) client_secret: Option<String>,
} }

22
src/response/event/checked_event/status/attachment.rs
View File

@ -2,21 +2,21 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(in super::super) struct Attachment { pub(crate) struct Attachment {
pub(in super::super) id: String, pub(crate) id: String,
pub(in super::super) r#type: AttachmentType, pub(crate) r#type: AttachmentType,
pub(in super::super) url: String, pub(crate) url: String,
pub(in super::super) preview_url: String, pub(crate) preview_url: String,
pub(in super::super) remote_url: Option<String>, pub(crate) remote_url: Option<String>,
pub(in super::super) text_url: Option<String>, pub(crate) text_url: Option<String>,
pub(in super::super) meta: Option<serde_json::Value>, // TODO - is this the best type for the API? pub(crate) meta: Option<serde_json::Value>, // TODO - is this the best type for the API?
pub(in super::super) description: Option<String>, pub(crate) description: Option<String>,
pub(in super::super) blurhash: Option<String>, pub(crate) blurhash: Option<String>,
} }
#[serde(rename_all = "lowercase", deny_unknown_fields)] #[serde(rename_all = "lowercase", deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(in super::super) enum AttachmentType { pub(crate) enum AttachmentType {
Unknown, Unknown,
Image, Image,
Gifv, Gifv,

30
src/response/event/checked_event/status/card.rs
View File

@ -2,25 +2,25 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(in super::super) struct Card { pub(crate) struct Card {
pub(super) url: String, pub(crate) url: String,
pub(super) title: String, pub(crate) title: String,
pub(super) description: String, pub(crate) description: String,
pub(super) r#type: CardType, pub(crate) r#type: CardType,
pub(super) author_name: Option<String>, pub(crate) author_name: Option<String>,
pub(super) author_url: Option<String>, pub(crate) author_url: Option<String>,
pub(super) provider_name: Option<String>, pub(crate) provider_name: Option<String>,
pub(super) provider_url: Option<String>, pub(crate) provider_url: Option<String>,
pub(super) html: Option<String>, pub(crate) html: Option<String>,
pub(super) width: Option<i64>, pub(crate) width: Option<i64>,
pub(super) height: Option<i64>, pub(crate) height: Option<i64>,
pub(super) image: Option<String>, pub(crate) image: Option<String>,
pub(super) embed_url: Option<String>, pub(crate) embed_url: Option<String>,
} }
#[serde(rename_all = "lowercase", deny_unknown_fields)] #[serde(rename_all = "lowercase", deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) enum CardType { pub(crate) enum CardType {
Link, Link,
Photo, Photo,
Video, Video,

28
src/response/event/checked_event/status/poll.rs
View File

@ -3,22 +3,22 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(in super::super) struct Poll { pub(crate) struct Poll {
pub(super) id: String, pub(crate) id: String,
pub(super) expires_at: String, pub(crate) expires_at: String,
pub(super) expired: bool, pub(crate) expired: bool,
pub(super) multiple: bool, pub(crate) multiple: bool,
pub(super) votes_count: i64, pub(crate) votes_count: i64,
pub(super) voters_count: Option<i64>, pub(crate) voters_count: Option<i64>,
pub(super) voted: Option<bool>, pub(crate) voted: Option<bool>,
pub(super) own_votes: Option<Vec<i64>>, pub(crate) own_votes: Option<Vec<i64>>,
pub(super) options: Vec<PollOptions>, pub(crate) options: Vec<PollOptions>,
pub(super) emojis: Vec<Emoji>, pub(crate) emojis: Vec<Emoji>,
} }
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct PollOptions { pub(crate) struct PollOptions {
pub(super) title: String, pub(crate) title: String,
pub(super) votes_count: Option<i32>, pub(crate) votes_count: Option<i32>,
} }

16
src/response/event/checked_event/tag.rs
View File

@ -2,16 +2,16 @@ use serde::{Deserialize, Serialize};
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct Tag { pub(crate) struct Tag {
pub(super) name: String, pub(crate) name: String,
pub(super) url: String, pub(crate) url: String,
pub(super) history: Option<Vec<History>>, pub(crate) history: Option<Vec<History>>,
} }
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) struct History { pub(crate) struct History {
pub(super) day: String, pub(crate) day: String,
pub(super) uses: String, pub(crate) uses: String,
pub(super) accounts: String, pub(crate) accounts: String,
} }

2
src/response/event/checked_event/visibility.rs
View File

@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
#[serde(rename_all = "lowercase", deny_unknown_fields)] #[serde(rename_all = "lowercase", deny_unknown_fields)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub(super) enum Visibility { pub(crate) enum Visibility {
Public, Public,
Unlisted, Unlisted,
Private, Private,

4
src/response/event/dynamic_event.rs
View File

@ -18,7 +18,7 @@ pub struct DynEvent {
} }
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum EventKind { pub enum EventKind {
Update(DynStatus), Update(DynStatus),
NonUpdate, NonUpdate,
} }
@ -30,7 +30,7 @@ impl Default for EventKind {
} }
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct DynStatus { pub struct DynStatus {
pub(crate) id: Id, pub(crate) id: Id,
pub(crate) username: String, pub(crate) username: String,
pub(crate) language: Option<String>, pub(crate) language: Option<String>,

2
src/response/redis.rs
View File

@ -13,7 +13,7 @@ pub use msg::{RedisMsg, RedisParseOutput};
use connection::RedisConnErr; use connection::RedisConnErr;
use msg::RedisParseErr; use msg::RedisParseErr;
enum RedisCmd { pub(crate) enum RedisCmd {
Subscribe, Subscribe,
Unsubscribe, Unsubscribe,
} }

330
src/response/redis/connection.rs
View File

@ -1,149 +1,227 @@
mod err; mod err;
pub(crate) use err::RedisConnErr; pub(super) use connection::*;
pub use err::RedisConnErr;
#[cfg(any(test, feature = "bench"))]
pub(self) use mock_connection as connection;
use super::Error as ManagerErr; #[cfg(not(any(test, feature = "bench")))]
use super::RedisCmd; mod connection {
use crate::config::Redis; use super::super::Error as ManagerErr;
use crate::request::Timeline; use super::super::RedisCmd;
use super::err::RedisConnErr;
use crate::config::Redis;
use crate::request::Timeline;
use futures::{Async, Poll}; use futures::{Async, Poll};
use lru::LruCache; use lru::LruCache;
use std::io::{self, Read, Write}; use std::io::{self, Read, Write};
use std::net::TcpStream; use std::net::TcpStream;
use std::time::Duration; use std::time::Duration;
type Result<T> = std::result::Result<T, RedisConnErr>; type Result<T> = std::result::Result<T, RedisConnErr>;
#[derive(Debug)] #[derive(Debug)]
pub(super) struct RedisConn { pub struct RedisConn {
primary: TcpStream, primary: TcpStream,
secondary: TcpStream, secondary: TcpStream,
pub(super) namespace: Option<String>, pub(in super::super) namespace: Option<String>,
// TODO: eventually, it might make sense to have Mastodon publish to timelines with // TODO: eventually, it might make sense to have Mastodon publish to timelines with
// the tag number instead of the tag name. This would save us from dealing // the tag number instead of the tag name. This would save us from dealing
// with a cache here and would be consistent with how lists/users are handled. // with a cache here and would be consistent with how lists/users are handled.
pub(super) tag_name_cache: LruCache<i64, String>, pub(in super::super) tag_name_cache: LruCache<i64, String>,
pub(super) input: Vec<u8>, pub(in super::super) input: Vec<u8>,
}
impl RedisConn {
pub(super) fn new(redis_cfg: &Redis) -> Result<Self> {
let addr = [&*redis_cfg.host, ":", &*redis_cfg.port.to_string()].concat();
let conn = Self::new_connection(&addr, redis_cfg.password.as_ref())?;
conn.set_nonblocking(true)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
Ok(Self {
primary: conn,
secondary: Self::new_connection(&addr, redis_cfg.password.as_ref())?,
tag_name_cache: LruCache::new(1000),
namespace: redis_cfg.namespace.clone().0,
input: vec![0; 4096 * 4],
})
} }
pub(super) fn poll_redis(&mut self, start: usize) -> Poll<usize, ManagerErr> {
const BLOCK: usize = 4096 * 2; impl RedisConn {
if self.input.len() < start + BLOCK { pub(in super::super) fn new(redis_cfg: &Redis) -> Result<Self> {
self.input.resize(self.input.len() * 2, 0); let addr = [&*redis_cfg.host, ":", &*redis_cfg.port.to_string()].concat();
log::info!("Resizing input buffer to {} KiB.", self.input.len() / 1024);
// log::info!("Current buffer: {}", String::from_utf8_lossy(&self.input)); let conn = Self::new_connection(&addr, redis_cfg.password.as_ref())?;
conn.set_nonblocking(true)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
Ok(Self {
primary: conn,
secondary: Self::new_connection(&addr, redis_cfg.password.as_ref())?,
tag_name_cache: LruCache::new(1000),
namespace: redis_cfg.namespace.clone().0,
input: vec![0; 4096 * 4],
})
} }
use Async::*; pub(in super::super) fn poll_redis(&mut self, i: usize) -> Poll<Option<usize>, ManagerErr> {
match self.primary.read(&mut self.input[start..start + BLOCK]) { const BLOCK: usize = 4096 * 2;
Ok(n) => Ok(Ready(n)), if self.input.len() < i + BLOCK {
Err(e) if matches!(e.kind(), io::ErrorKind::WouldBlock) => Ok(NotReady), self.input.resize(self.input.len() * 2, 0);
Err(e) => { log::info!("Resizing input buffer to {} KiB.", self.input.len() / 1024);
Ready(log::error!("{}", e)); // log::info!("Current buffer: {}", String::from_utf8_lossy(&self.input));
Ok(Ready(0)) }
use Async::*;
match self.primary.read(&mut self.input[i..i + BLOCK]) {
Ok(n) if n == 0 => Ok(Ready(None)),
Ok(n) => Ok(Ready(Some(n))),
Err(e) if matches!(e.kind(), io::ErrorKind::WouldBlock) => Ok(NotReady),
Err(e) => {
Ready(log::error!("{}", e));
Ok(Ready(None))
}
}
}
pub(crate) fn send_cmd(&mut self, cmd: RedisCmd, timelines: &[Timeline]) -> Result<()> {
let namespace = self.namespace.take();
let timelines: Result<Vec<String>> = timelines
.iter()
.map(|tl| {
let hashtag = tl.tag().and_then(|id| self.tag_name_cache.get(&id));
match &namespace {
Some(ns) => Ok(format!("{}:{}", ns, tl.to_redis_raw_timeline(hashtag)?)),
None => Ok(tl.to_redis_raw_timeline(hashtag)?),
}
})
.collect();
let (primary_cmd, secondary_cmd) = cmd.into_sendable(&timelines?[..]);
self.primary.write_all(&primary_cmd)?;
// We also need to set a key to tell the Puma server that we've subscribed or
// unsubscribed to the channel because it stops publishing updates when it thinks
// no one is subscribed.
// (Documented in [PR #3278](https://github.com/tootsuite/mastodon/pull/3278))
// Question: why can't the Puma server just use NUMSUB for this?
self.secondary.write_all(&secondary_cmd)?;
Ok(())
}
fn new_connection(addr: &str, pass: Option<&String>) -> Result<TcpStream> {
let mut conn = TcpStream::connect(&addr)?;
if let Some(password) = pass {
Self::auth_connection(&mut conn, &addr, password)?;
}
Self::validate_connection(&mut conn, &addr)?;
conn.set_read_timeout(Some(Duration::from_millis(10)))
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
Self::set_connection_name(&mut conn, &addr)?;
Ok(conn)
}
fn auth_connection(conn: &mut TcpStream, addr: &str, pass: &str) -> Result<()> {
conn.write_all(
&[
b"*2\r\n$4\r\nauth\r\n$",
pass.len().to_string().as_bytes(),
b"\r\n",
pass.as_bytes(),
b"\r\n",
]
.concat(),
)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 5];
conn.read_exact(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
if String::from_utf8_lossy(&buffer) != "+OK\r\n" {
Err(RedisConnErr::IncorrectPassword(pass.to_string()))?
}
Ok(())
}
fn validate_connection(conn: &mut TcpStream, addr: &str) -> Result<()> {
conn.write_all(b"PING\r\n")
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 100];
conn.read(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let reply = String::from_utf8_lossy(&buffer);
match &*reply {
r if r.starts_with("+PONG\r\n") => Ok(()),
r if r.starts_with("-NOAUTH") => Err(RedisConnErr::MissingPassword),
r if r.starts_with("HTTP/1.") => Err(RedisConnErr::NotRedis(addr.to_string())),
_ => Err(RedisConnErr::InvalidRedisReply(reply.to_string())),
}
}
fn set_connection_name(conn: &mut TcpStream, addr: &str) -> Result<()> {
conn.write_all(b"*3\r\n$6\r\nCLIENT\r\n$7\r\nSETNAME\r\n$8\r\nflodgatt\r\n")
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 100];
conn.read(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let reply = String::from_utf8_lossy(&buffer);
match &*reply {
r if r.starts_with("+OK\r\n") => Ok(()),
_ => Err(RedisConnErr::InvalidRedisReply(reply.to_string())),
} }
} }
} }
}
#[cfg(any(test, feature = "bench"))]
mod mock_connection {
use super::super::Error as ManagerErr;
use super::super::RedisCmd;
use super::err::RedisConnErr;
use crate::config::Redis;
use crate::request::Timeline;
pub(crate) fn send_cmd(&mut self, cmd: RedisCmd, timelines: &[Timeline]) -> Result<()> { use futures::{Async, Poll};
let namespace = self.namespace.take(); use lru::LruCache;
let timelines: Result<Vec<String>> = timelines use std::collections::VecDeque;
.iter()
.map(|tl| { type Result<T> = std::result::Result<T, RedisConnErr>;
let hashtag = tl.tag().and_then(|id| self.tag_name_cache.get(&id));
match &namespace { #[derive(Debug)]
Some(ns) => Ok(format!("{}:{}", ns, tl.to_redis_raw_timeline(hashtag)?)), pub struct RedisConn {
None => Ok(tl.to_redis_raw_timeline(hashtag)?), pub(in super::super) namespace: Option<String>,
} pub(in super::super) tag_name_cache: LruCache<i64, String>,
pub(in super::super) input: Vec<u8>,
pub(in super::super) test_input: VecDeque<u8>,
}
impl RedisConn {
pub(in super::super) fn new(redis_cfg: &Redis) -> Result<Self> {
Ok(Self {
tag_name_cache: LruCache::new(1000),
namespace: redis_cfg.namespace.clone().0,
input: vec![0; 4096 * 4],
test_input: VecDeque::new(),
}) })
.collect();
let (primary_cmd, secondary_cmd) = cmd.into_sendable(&timelines?[..]);
self.primary.write_all(&primary_cmd)?;
// We also need to set a key to tell the Puma server that we've subscribed or
// unsubscribed to the channel because it stops publishing updates when it thinks
// no one is subscribed.
// (Documented in [PR #3278](https://github.com/tootsuite/mastodon/pull/3278))
// Question: why can't the Puma server just use NUMSUB for this?
self.secondary.write_all(&secondary_cmd)?;
Ok(())
}
fn new_connection(addr: &str, pass: Option<&String>) -> Result<TcpStream> {
let mut conn = TcpStream::connect(&addr)?;
if let Some(password) = pass {
Self::auth_connection(&mut conn, &addr, password)?;
} }
Self::validate_connection(&mut conn, &addr)?; pub fn poll_redis(&mut self, start: usize) -> Poll<Option<usize>, ManagerErr> {
conn.set_read_timeout(Some(Duration::from_millis(10))) const BLOCK: usize = 4096 * 2;
.map_err(|e| RedisConnErr::with_addr(&addr, e))?; if self.input.len() < start + BLOCK {
Self::set_connection_name(&mut conn, &addr)?; self.input.resize(self.input.len() * 2, 0);
Ok(conn) log::info!("Resizing input buffer to {} KiB.", self.input.len() / 1024);
} }
fn auth_connection(conn: &mut TcpStream, addr: &str, pass: &str) -> Result<()> { for i in 0..BLOCK {
conn.write_all( if let Some(byte) = self.test_input.pop_front() {
&[ self.input[start + i] = byte;
b"*2\r\n$4\r\nauth\r\n$", } else if i > 0 {
pass.len().to_string().as_bytes(), return Ok(Async::Ready(Some(i)));
b"\r\n", } else {
pass.as_bytes(), return Ok(Async::Ready(None));
b"\r\n", }
] }
.concat(), Ok(Async::Ready(Some(BLOCK)))
)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let mut buffer = vec![0_u8; 5];
conn.read_exact(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
if String::from_utf8_lossy(&buffer) != "+OK\r\n" {
Err(RedisConnErr::IncorrectPassword(pass.to_string()))?
} }
Ok(())
}
fn validate_connection(conn: &mut TcpStream, addr: &str) -> Result<()> { pub fn add(&mut self, input: &[u8]) {
conn.write_all(b"PING\r\n") for byte in input {
.map_err(|e| RedisConnErr::with_addr(&addr, e))?; self.test_input.push_back(*byte)
let mut buffer = vec![0_u8; 100]; }
conn.read(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let reply = String::from_utf8_lossy(&buffer);
match &*reply {
r if r.starts_with("+PONG\r\n") => Ok(()),
r if r.starts_with("-NOAUTH") => Err(RedisConnErr::MissingPassword),
r if r.starts_with("HTTP/1.") => Err(RedisConnErr::NotRedis(addr.to_string())),
_ => Err(RedisConnErr::InvalidRedisReply(reply.to_string())),
} }
} pub(crate) fn send_cmd(&mut self, cmd: RedisCmd, timelines: &[Timeline]) -> Result<()> {
// stub - does nothing; silences some unused-code warnings
let timelines: Result<Vec<String>> = timelines
.iter()
.map(|tl| Ok(tl.to_redis_raw_timeline(None).expect("test")))
.collect();
fn set_connection_name(conn: &mut TcpStream, addr: &str) -> Result<()> { let _ = cmd.into_sendable(&timelines?);
conn.write_all(b"*3\r\n$6\r\nCLIENT\r\n$7\r\nSETNAME\r\n$8\r\nflodgatt\r\n")
.map_err(|e| RedisConnErr::with_addr(&addr, e))?; Ok(())
let mut buffer = vec![0_u8; 100];
conn.read(&mut buffer)
.map_err(|e| RedisConnErr::with_addr(&addr, e))?;
let reply = String::from_utf8_lossy(&buffer);
match &*reply {
r if r.starts_with("+OK\r\n") => Ok(()),
_ => Err(RedisConnErr::InvalidRedisReply(reply.to_string())),
} }
} }
} }

1
src/response/redis/connection/err.rs
View File

@ -13,6 +13,7 @@ pub enum RedisConnErr {
} }
impl RedisConnErr { impl RedisConnErr {
#[allow(unused)] // Not used during testing due to conditional compilation
pub(super) fn with_addr<T: AsRef<str>>(address: T, inner: std::io::Error) -> Self { pub(super) fn with_addr<T: AsRef<str>>(address: T, inner: std::io::Error) -> Self {
Self::ConnectionErr { Self::ConnectionErr {
addr: address.as_ref().to_string(), addr: address.as_ref().to_string(),

141
src/response/redis/manager.rs
View File

@ -25,73 +25,111 @@ type EventChannel = Sender<Arc<Event>>;
/// The item that streams from Redis and is polled by the `ClientAgent` /// The item that streams from Redis and is polled by the `ClientAgent`
pub struct Manager { pub struct Manager {
redis_conn: RedisConn, pub redis_conn: RedisConn,
timelines: HashMap<Timeline, HashMap<u32, EventChannel>>, timelines: HashMap<Timeline, HashMap<u32, EventChannel>>,
ping_time: Instant, ping_time: Instant,
channel_id: u32, channel_id: u32,
unread_idx: (usize, usize), pub unread_idx: (usize, usize),
tag_id_cache: LruCache<String, i64>, tag_id_cache: LruCache<String, i64>,
} }
impl Stream for Manager { impl Stream for Manager {
type Item = (); type Item = (Timeline, Arc<Event>);
type Error = Error; type Error = Error;
fn poll(&mut self) -> Poll<Option<()>, Error> { fn poll(&mut self) -> Poll<Option<Self::Item>, Error> {
if self.ping_time.elapsed() > Duration::from_secs(30) { let input = &self.redis_conn.input[self.unread_idx.0..self.unread_idx.1];
self.send_pings()? let (valid, invalid) = str::from_utf8(input)
} .map(|v| (v, &b""[..]))
.unwrap_or_else(|e| {
while let Async::Ready(msg_len) = self.redis_conn.poll_redis(self.unread_idx.1)? { // NOTE - this bounds check occurs more often than necessary; it could occur only when
self.unread_idx = (0, self.unread_idx.1 + msg_len); // polling Redis. However, benchmarking with Criterion shows it to be *very*
// inexpensive (<1 us) and thus not worth removing (doing so would require `unsafe`).
let input = &self.redis_conn.input[..self.unread_idx.1]; let (valid, invalid) = input.split_at(e.valid_up_to());
let mut unread = str::from_utf8(input).unwrap_or_else(|e| { (str::from_utf8(valid).expect("split_at"), invalid)
str::from_utf8(input.split_at(e.valid_up_to()).0).expect("guaranteed by `split_at`")
}); });
while !unread.is_empty() { if !valid.is_empty() {
use RedisParseOutput::*; use RedisParseOutput::*;
match RedisParseOutput::try_from(unread) { match RedisParseOutput::try_from(valid) {
Ok(Msg(msg)) => { Ok(Msg(msg)) => {
// If we get a message and it matches the redis_namespace, get the msg's // If we get a message and it matches the redis_namespace, get the msg's
// Event and send it to all channels matching the msg's Timeline // Event and send it to all channels matching the msg's Timeline
if let Some(tl) = msg.timeline_matching_ns(&self.redis_conn.namespace) { if let Some(tl) = msg.timeline_matching_ns(&self.redis_conn.namespace) {
let tl = Timeline::from_redis_text(tl, &mut self.tag_id_cache)?; self.unread_idx.0 =
let event: Arc<Event> = Arc::new(msg.event_txt.try_into()?); self.unread_idx.1 - msg.leftover_input.len() - invalid.len();
if let Some(channels) = self.timelines.get_mut(&tl) {
for channel in channels.values_mut() { let tl = Timeline::from_redis_text(tl, &mut self.tag_id_cache)?;
if let Ok(Async::NotReady) = channel.poll_ready() { let event: Arc<Event> = Arc::new(msg.event_txt.try_into()?);
log::warn!("{:?} channel full\ncan't send:{:?}", tl, event); Ok(Async::Ready(Some((tl, event))))
return Ok(Async::NotReady); } else {
} Ok(Async::Ready(None))
let _ = channel.try_send(event.clone()); // err just means channel will be closed
}
}
}
unread = msg.leftover_input;
self.unread_idx.0 = self.unread_idx.1 - unread.len();
} }
Ok(NonMsg(leftover_input)) => { }
unread = leftover_input; Ok(NonMsg(leftover_input)) => {
self.unread_idx.0 = self.unread_idx.1 - unread.len(); self.unread_idx.0 = self.unread_idx.1 - leftover_input.len();
} Ok(Async::Ready(None))
Err(RedisParseErr::Incomplete) => { }
self.copy_partial_msg(); Err(RedisParseErr::Incomplete) => {
break; self.copy_partial_msg();
} Ok(Async::NotReady)
Err(e) => Err(Error::RedisParseErr(e, unread.to_string()))?, }
}; Err(e) => Err(Error::RedisParseErr(e, valid.to_string()))?,
}
if self.unread_idx.0 == self.unread_idx.1 {
self.unread_idx = (0, 0)
} }
} else {
self.unread_idx = (0, 0);
Ok(Async::NotReady)
} }
Ok(Async::Ready(Some(())))
} }
} }
impl Manager { impl Manager {
// untested
pub fn send_msgs(&mut self) -> Poll<(), Error> {
if self.ping_time.elapsed() > Duration::from_secs(30) {
self.send_pings()?
}
while let Ok(Async::Ready(Some(msg_len))) = self.redis_conn.poll_redis(self.unread_idx.1) {
self.unread_idx.1 += msg_len;
while let Ok(Async::Ready(msg)) = self.poll() {
if let Some((tl, event)) = msg {
for channel in self.timelines.entry(tl).or_default().values_mut() {
if let Ok(Async::NotReady) = channel.poll_ready() {
log::warn!("{:?} channel full\ncan't send:{:?}", tl, event);
self.rewind_to_prev_msg();
return Ok(Async::NotReady);
}
let _ = channel.try_send(event.clone()); // err just means channel will be closed
}
}
}
}
Ok(Async::Ready(()))
}
fn rewind_to_prev_msg(&mut self) {
self.unread_idx.0 = loop {
let input = &self.redis_conn.input[..self.unread_idx.0];
let input = str::from_utf8(input).unwrap_or_else(|e| {
str::from_utf8(input.split_at(e.valid_up_to()).0).expect("guaranteed by `split_at`")
});
let index = if let Some(i) = input.rfind("\r\n*") {
i + "\r\n".len()
} else {
0
};
self.unread_idx.0 = index;
if let Ok(Async::Ready(Some(_))) = self.poll() {
break index;
}
}
}
fn copy_partial_msg(&mut self) { fn copy_partial_msg(&mut self) {
if self.unread_idx.0 == 0 { if self.unread_idx.0 == 0 {
// msg already first; no copying needed // msg already first; no copying needed
@ -107,7 +145,7 @@ impl Manager {
self.redis_conn.input = self.redis_conn.input[self.unread_idx.0..].into(); self.redis_conn.input = self.redis_conn.input[self.unread_idx.0..].into();
} }
self.unread_idx = (0, self.unread_idx.1 - self.unread_idx.0); self.unread_idx = (0, self.unread_idx.1 - self.unread_idx.0);
dbg!(&self.unread_idx); &self.unread_idx;
} }
/// Create a new `Manager`, with its own Redis connections (but no active subscriptions). /// Create a new `Manager`, with its own Redis connections (but no active subscriptions).
pub fn try_from(redis_cfg: &config::Redis) -> Result<Self> { pub fn try_from(redis_cfg: &config::Redis) -> Result<Self> {
@ -208,3 +246,6 @@ impl Manager {
.collect() .collect()
} }
} }
#[cfg(test)]
mod test;

245
src/response/redis/manager/test.rs Normal file
View File

@ -0,0 +1,245 @@
use super::*;
use crate::config;
use crate::response::event::checked_event::{
account::{Account, Field},
status::attachment::{Attachment, AttachmentType::*},
status::Status,
tag::Tag,
visibility::Visibility::*,
CheckedEvent::*,
};
use crate::Id;
use serde_json::json;
use std::fs;
type TestResult = std::result::Result<(), Box<dyn std::error::Error>>;
fn input(i: usize) -> Vec<u8> {
fs::read_to_string(format!("test_data/redis_input_{:03}.resp", i))
.expect("test input not found")
.as_bytes()
.to_vec()
}
fn output(i: usize) -> Arc<Event> {
vec![
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_001.rs"
))),
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_002.rs"
))),
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_003.rs"
))),
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_004.rs"
))),
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_005.rs"
))),
Arc::new(include!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/test_data/event_006.rs"
))),
][i]
.clone()
}
#[test]
fn manager_poll_matches_six_events() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
for i in 1..=6 {
manager.redis_conn.add(&input(i));
}
let mut i = 0;
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx = (0, manager.unread_idx.1 + len);
while let Ok(Async::Ready(Some((_tl, event)))) = manager.poll() {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
Ok(assert_eq!(i, 6))
}
#[test]
fn manager_poll_handles_non_utf8() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
let mut input_txt = Vec::new();
for i in 1..=6 {
input_txt.extend_from_slice(&input(i))
}
let invalid_idx = str::from_utf8(&input_txt)?
.chars()
.take_while(|char| char.len_utf8() == 1)
.collect::<Vec<_>>()
.len()
+ 1;
manager.redis_conn.add(&input_txt[..invalid_idx]);
let mut i = 0;
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(Some((_tl, event)))) = manager.poll() {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
manager.redis_conn.add(&input_txt[invalid_idx..]);
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(Some((_tl, event)))) = manager.poll() {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
Ok(assert_eq!(i, 6))
}
#[test]
fn manager_poll_matches_six_events_in_batches() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
for i in 1..=3 {
manager.redis_conn.add(&input(i))
}
let mut i = 0;
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(Some((_tl, event)))) = manager.poll() {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
for i in 4..=6 {
manager.redis_conn.add(&input(i));
}
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(Some((_tl, event)))) = manager.poll() {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
Ok(assert_eq!(i, 6))
}
#[test]
fn manager_poll_handles_non_events() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
for i in 1..=6 {
manager.redis_conn.add(&input(i));
manager
.redis_conn
.add(b"*3\r\n$9\r\nsubscribe\r\n$12\r\ntimeline:308\r\n:1\r\n");
}
let mut i = 0;
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(msg)) = manager.poll() {
if let Some((_tl, event)) = msg {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
}
Ok(assert_eq!(i, 6))
}
#[test]
fn manager_poll_handles_partial_events() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
for i in 1..=3 {
manager.redis_conn.add(&input(i));
}
manager.redis_conn.add(&input(4)[..50]);
let mut i = 0;
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(msg)) = manager.poll() {
if let Some((_tl, event)) = msg {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
}
assert_eq!(i, 3);
manager.redis_conn.add(&input(4)[50..]);
manager.redis_conn.add(&input(5));
manager.redis_conn.add(&input(6));
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(msg)) = manager.poll() {
if let Some((_tl, event)) = msg {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
}
Ok(assert_eq!(i, 6))
}
#[test]
fn manager_poll_handles_full_channel() -> TestResult {
let mut manager = Manager::try_from(&config::Redis::default())?;
for i in 1..=6 {
manager.redis_conn.add(&input(i));
}
let (mut i, channel_full) = (0, 3);
'outer: loop {
while let Ok(Async::Ready(Some(n))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += n;
while let Ok(Async::Ready(msg)) = manager.poll() {
if let Some((_tl, event)) = msg {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
// Simulates a `ChannelFull` error after sending `channel_full` msgs
if i == channel_full {
break 'outer;
}
}
}
}
let _rewind = (|| {
manager.rewind_to_prev_msg();
i -= 1;
})();
while let Ok(Async::Ready(Some(len))) = manager.redis_conn.poll_redis(manager.unread_idx.1) {
manager.unread_idx.1 += len;
while let Ok(Async::Ready(msg)) = manager.poll() {
if let Some((_tl, event)) = msg {
println!("Parsing Event #{:03}", i + 1);
assert_eq!(event, output(i));
i += 1;
}
}
}
Ok(assert_eq!(i, 6))
}

8
src/response/redis/msg.rs
View File

@ -27,15 +27,15 @@ use std::str;
mod err; mod err;
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub(crate) enum RedisParseOutput<'a> { pub enum RedisParseOutput<'a> {
Msg(RedisMsg<'a>), Msg(RedisMsg<'a>),
NonMsg(&'a str), NonMsg(&'a str),
} }
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub(crate) struct RedisMsg<'a> { pub struct RedisMsg<'a> {
pub(crate) timeline_txt: &'a str, pub timeline_txt: &'a str,
pub(crate) event_txt: &'a str, pub event_txt: &'a str,
pub(crate) leftover_input: &'a str, pub(crate) leftover_input: &'a str,
} }

17
src/response/redis/msg/err.rs
View File

@ -15,8 +15,8 @@ impl fmt::Display for RedisParseErr {
use RedisParseErr::*; use RedisParseErr::*;
let msg = match self { let msg = match self {
Incomplete => "The input from Redis does not form a complete message, likely because \ Incomplete => "The input from Redis does not form a complete message, likely because \
the input buffer filled partway through a message. Save this input \ the input buffer filled partway through a message. Save this input and try again \
and try again with additional input from Redis." with additional input from Redis."
.to_string(), .to_string(),
InvalidNumber(parse_int_err) => format!( InvalidNumber(parse_int_err) => format!(
"Redis indicated that an item would be a number, but it could not be parsed: {}", "Redis indicated that an item would be a number, but it could not be parsed: {}",
@ -25,20 +25,19 @@ impl fmt::Display for RedisParseErr {
InvalidLineStart(line_start_char) => format!( InvalidLineStart(line_start_char) => format!(
"A line from Redis started with `{}`, which is not a valid character to indicate \ "A line from Redis started with `{}`, which is not a valid character to indicate \
the type of the Redis line.", the type of the Redis line.",
line_start_char line_start_char
), ),
InvalidLineEnd(len, line) => format!( // TODO - FIXME InvalidLineEnd(len, line) => format!(
"A Redis line did not have the promised length of {}. \ "A Redis line did not have the promised length of {}. The line is: {}",
The line is: {}",
len, line len, line
), ),
IncorrectRedisType => "Received a Redis type that is not supported in this context. \ IncorrectRedisType => "Received a Redis type that is not supported in this context. \
Flodgatt expects each message from Redis to be a Redis array \ Flodgatt expects each message from Redis to be a Redis array consisting of bulk \
consisting of bulk strings or integers." strings or integers."
.to_string(), .to_string(),
MissingField => "Redis input was missing a field Flodgatt expected (e.g., a `message` \ MissingField => "Redis input was missing a field Flodgatt expected (e.g., a `message` \
without a payload line)" without a payload line)"
.to_string(), .to_string(),
}; };
write!(f, "{}", msg) write!(f, "{}", msg)