MirrorHub/conduit
1
0
Fork 0
mirror of https://gitlab.com/famedly/conduit.git synced 2024-11-04 17:29:14 +01:00
Code Issues Releases Activity

improvement: batched inserts for tokenids

Browse source
This commit is contained in:
Timo Kösters 2021-08-04 12:54:26 +02:00
parent fce22362d4
commit 9bb4c3cd01
No known key found for this signature in database
GPG key ID: 24DA7517711A2BA4
3 changed files with 24 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/database.rs
View file

@ -271,8 +271,8 @@ impl Database {
eventid_outlierpdu: builder.open_tree("eventid_outlierpdu")?, eventid_outlierpdu: builder.open_tree("eventid_outlierpdu")?,
referencedevents: builder.open_tree("referencedevents")?, referencedevents: builder.open_tree("referencedevents")?,
pdu_cache: Mutex::new(LruCache::new(100_000)), pdu_cache: Mutex::new(LruCache::new(1_000_000)),
auth_chain_cache: Mutex::new(LruCache::new(100_000)), auth_chain_cache: Mutex::new(LruCache::new(1_000_000)),
}, },
account_data: account_data::AccountData { account_data: account_data::AccountData {
roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?,

20
src/database/rooms.rs
View file

@ -905,18 +905,20 @@ impl Rooms {
} }
EventType::RoomMessage => { EventType::RoomMessage => {
if let Some(body) = pdu.content.get("body").and_then(|b| b.as_str()) { if let Some(body) = pdu.content.get("body").and_then(|b| b.as_str()) {
for word in body let mut batch = body
.split_terminator(|c: char| !c.is_alphanumeric()) .split_terminator(|c: char| !c.is_alphanumeric())
.filter(|word| word.len() <= 50) .filter(|word| word.len() <= 50)
.map(str::to_lowercase) .map(str::to_lowercase)
{ .map(|word| {
let mut key = pdu.room_id.as_bytes().to_vec(); let mut key = pdu.room_id.as_bytes().to_vec();
key.push(0xff); key.push(0xff);
key.extend_from_slice(word.as_bytes()); key.extend_from_slice(word.as_bytes());
key.push(0xff); key.push(0xff);
key.extend_from_slice(&pdu_id); key.extend_from_slice(&pdu_id);
self.tokenids.insert(&key, &[])?; (key, Vec::new())
} });
self.tokenids.insert_batch(&mut batch)?;
if body.starts_with(&format!("@conduit:{}: ", db.globals.server_name())) if body.starts_with(&format!("@conduit:{}: ", db.globals.server_name()))
&& self && self

13
src/server_server.rs
View file

@ -1,6 +1,6 @@
use crate::{ use crate::{
client_server::{self, claim_keys_helper, get_keys_helper}, client_server::{self, claim_keys_helper, get_keys_helper},
database::DatabaseGuard, database::{abstraction::sqlite::MILLI, DatabaseGuard},
utils, ConduitResult, Database, Error, PduEvent, Result, Ruma, utils, ConduitResult, Database, Error, PduEvent, Result, Ruma,
}; };
use get_profile_information::v1::ProfileField; use get_profile_information::v1::ProfileField;
@ -1732,7 +1732,12 @@ fn get_auth_chain(starting_events: Vec<EventId>, db: &Database) -> Result<HashSe
cached.clone() cached.clone()
} else { } else {
drop(cache); drop(cache);
let start = Instant::now();
let auth_chain = get_auth_chain_recursive(&event_id, HashSet::new(), db)?; let auth_chain = get_auth_chain_recursive(&event_id, HashSet::new(), db)?;
let elapsed = start.elapsed();
if elapsed > MILLI {
println!("auth chain for {} took {:?}", &event_id, elapsed)
}
cache = db.rooms.auth_chain_cache(); cache = db.rooms.auth_chain_cache();
@ -1747,7 +1752,11 @@ fn get_auth_chain(starting_events: Vec<EventId>, db: &Database) -> Result<HashSe
Ok(full_auth_chain) Ok(full_auth_chain)
} }
fn get_auth_chain_recursive(event_id: &EventId, mut found: HashSet<EventId>, db: &Database) -> Result<HashSet<EventId>> { fn get_auth_chain_recursive(
event_id: &EventId,
mut found: HashSet<EventId>,
db: &Database,
) -> Result<HashSet<EventId>> {
if let Some(pdu) = db.rooms.get_pdu(&event_id)? { if let Some(pdu) = db.rooms.get_pdu(&event_id)? {
for auth_event in &pdu.auth_events { for auth_event in &pdu.auth_events {
if !found.contains(auth_event) { if !found.contains(auth_event) {