MirrorHub/conduit
1
0
Fork 0
mirror of https://gitlab.com/famedly/conduit.git synced 2024-11-04 17:48:52 +01:00
Code Issues Releases Activity

improvement: use sqlite properly

Browse source
This commit is contained in:
Timo Kösters 2021-08-01 16:59:52 +02:00
parent 8174b16c38
commit 2c4f966d60
No known key found for this signature in database
GPG key ID: 24DA7517711A2BA4
2 changed files with 24 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
src/database.rs
View file

@ -47,12 +47,8 @@ pub struct Config {
db_cache_capacity_mb: f64, db_cache_capacity_mb: f64,
#[serde(default = "default_sqlite_read_pool_size")] #[serde(default = "default_sqlite_read_pool_size")]
sqlite_read_pool_size: usize, sqlite_read_pool_size: usize,
#[serde(default = "true_fn")]
sqlite_wal_clean_timer: bool,
#[serde(default = "default_sqlite_wal_clean_second_interval")] #[serde(default = "default_sqlite_wal_clean_second_interval")]
sqlite_wal_clean_second_interval: u32, sqlite_wal_clean_second_interval: u32,
#[serde(default = "default_sqlite_wal_clean_second_timeout")]
sqlite_wal_clean_second_timeout: u32,
#[serde(default = "default_sqlite_spillover_reap_fraction")] #[serde(default = "default_sqlite_spillover_reap_fraction")]
sqlite_spillover_reap_fraction: f64, sqlite_spillover_reap_fraction: f64,
#[serde(default = "default_sqlite_spillover_reap_interval_secs")] #[serde(default = "default_sqlite_spillover_reap_interval_secs")]
@ -120,11 +116,7 @@ fn default_sqlite_read_pool_size() -> usize {
} }
fn default_sqlite_wal_clean_second_interval() -> u32 { fn default_sqlite_wal_clean_second_interval() -> u32 {
60 * 60 15 * 60 // every 15 minutes
}
fn default_sqlite_wal_clean_second_timeout() -> u32 {
2
} }
fn default_sqlite_spillover_reap_fraction() -> f64 { fn default_sqlite_spillover_reap_fraction() -> f64 {
@ -465,7 +457,7 @@ impl Database {
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
{ {
Self::start_wal_clean_task(&db, &config).await; Self::start_wal_clean_task(Arc::clone(&db), &config).await;
Self::start_spillover_reap_task(builder, &config).await; Self::start_spillover_reap_task(builder, &config).await;
} }
@ -620,24 +612,17 @@ impl Database {
} }
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
#[tracing::instrument(skip(lock, config))] #[tracing::instrument(skip(db, config))]
pub async fn start_wal_clean_task(lock: &Arc<TokioRwLock<Self>>, config: &Config) { pub async fn start_wal_clean_task(db: Arc<TokioRwLock<Self>>, config: &Config) {
use tokio::time::{interval, timeout}; use tokio::time::interval;
#[cfg(unix)] #[cfg(unix)]
use tokio::signal::unix::{signal, SignalKind}; use tokio::signal::unix::{signal, SignalKind};
use tracing::info; use tracing::info;
use std::{ use std::time::{Duration, Instant};
sync::Weak,
time::{Duration, Instant},
};
let weak: Weak<TokioRwLock<Database>> = Arc::downgrade(&lock);
let lock_timeout = Duration::from_secs(config.sqlite_wal_clean_second_timeout as u64);
let timer_interval = Duration::from_secs(config.sqlite_wal_clean_second_interval as u64); let timer_interval = Duration::from_secs(config.sqlite_wal_clean_second_interval as u64);
let do_timer = config.sqlite_wal_clean_timer;
tokio::spawn(async move { tokio::spawn(async move {
let mut i = interval(timer_interval); let mut i = interval(timer_interval);
@ -647,45 +632,24 @@ impl Database {
loop { loop {
#[cfg(unix)] #[cfg(unix)]
tokio::select! { tokio::select! {
_ = i.tick(), if do_timer => { _ = i.tick() => {
info!(target: "wal-trunc", "Timer ticked") info!("wal-trunc: Timer ticked");
} }
_ = s.recv() => { _ = s.recv() => {
info!(target: "wal-trunc", "Received SIGHUP") info!("wal-trunc: Received SIGHUP");
} }
}; };
#[cfg(not(unix))] #[cfg(not(unix))]
if do_timer { {
i.tick().await; i.tick().await;
info!(target: "wal-trunc", "Timer ticked") info!("wal-trunc: Timer ticked")
} else {
// timer disabled, and there's no concept of signals on windows, bailing...
return;
} }
if let Some(arc) = Weak::upgrade(&weak) {
info!(target: "wal-trunc", "Rotating sync helpers...");
// This actually creates a very small race condition between firing this and trying to acquire the subsequent write lock.
// Though it is not a huge deal if the write lock doesn't "catch", as it'll harmlessly time out.
arc.read().await.globals.rotate.fire();
info!(target: "wal-trunc", "Locking..."); let start = Instant::now();
let guard = { if let Err(e) = db.read().await.flush_wal() {
if let Ok(guard) = timeout(lock_timeout, arc.write()).await { error!("wal-trunc: Errored: {}", e);
guard
} else {
info!(target: "wal-trunc", "Lock failed in timeout, canceled.");
continue;
}
};
info!(target: "wal-trunc", "Locked, flushing...");
let start = Instant::now();
if let Err(e) = guard.flush_wal() {
error!(target: "wal-trunc", "Errored: {}", e);
} else {
info!(target: "wal-trunc", "Flushed in {:?}", start.elapsed());
}
} else { } else {
break; info!("wal-trunc: Flushed in {:?}", start.elapsed());
} }
} }
}); });

48
src/database/abstraction/sqlite.rs
View file

@ -4,7 +4,7 @@ use crossbeam::channel::{
bounded, unbounded, Receiver as ChannelReceiver, Sender as ChannelSender, TryRecvError, bounded, unbounded, Receiver as ChannelReceiver, Sender as ChannelSender, TryRecvError,
}; };
use parking_lot::{Mutex, MutexGuard, RwLock}; use parking_lot::{Mutex, MutexGuard, RwLock};
use rusqlite::{params, Connection, DatabaseName::Main, OptionalExtension, Params}; use rusqlite::{Connection, DatabaseName::Main, OptionalExtension, Params};
use std::{ use std::{
collections::HashMap, collections::HashMap,
future::Future, future::Future,
@ -122,16 +122,11 @@ impl Pool {
fn prepare_conn<P: AsRef<Path>>(path: P, cache_size: Option<u32>) -> Result<Connection> { fn prepare_conn<P: AsRef<Path>>(path: P, cache_size: Option<u32>) -> Result<Connection> {
let conn = Connection::open(path)?; let conn = Connection::open(path)?;
conn.pragma_update(Some(Main), "journal_mode", &"WAL".to_owned())?; conn.pragma_update(Some(Main), "journal_mode", &"WAL")?;
conn.pragma_update(Some(Main), "synchronous", &"NORMAL")?;
// conn.pragma_update(Some(Main), "wal_autocheckpoint", &250)?;
// conn.pragma_update(Some(Main), "wal_checkpoint", &"FULL".to_owned())?;
conn.pragma_update(Some(Main), "synchronous", &"OFF".to_owned())?;
if let Some(cache_kib) = cache_size { if let Some(cache_kib) = cache_size {
conn.pragma_update(Some(Main), "cache_size", &(-Into::<i64>::into(cache_kib)))?; conn.pragma_update(Some(Main), "cache_size", &(-i64::from(cache_kib)))?;
} }
Ok(conn) Ok(conn)
@ -193,9 +188,6 @@ impl DatabaseEngine for Engine {
config.db_cache_capacity_mb, config.db_cache_capacity_mb,
)?; )?;
pool.write_lock()
.execute("CREATE TABLE IF NOT EXISTS _noop (\"key\" INT)", params![])?;
let arc = Arc::new(Engine { let arc = Arc::new(Engine {
pool, pool,
iter_pool: Mutex::new(ThreadPool::new(10)), iter_pool: Mutex::new(ThreadPool::new(10)),
@ -205,7 +197,7 @@ impl DatabaseEngine for Engine {
} }
fn open_tree(self: &Arc<Self>, name: &str) -> Result<Arc<dyn Tree>> { fn open_tree(self: &Arc<Self>, name: &str) -> Result<Arc<dyn Tree>> {
self.pool.write_lock().execute(format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", name).as_str(), [])?; self.pool.write_lock().execute(&format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", name), [])?;
Ok(Arc::new(SqliteTable { Ok(Arc::new(SqliteTable {
engine: Arc::clone(self), engine: Arc::clone(self),
@ -215,37 +207,15 @@ impl DatabaseEngine for Engine {
} }
fn flush(self: &Arc<Self>) -> Result<()> { fn flush(self: &Arc<Self>) -> Result<()> {
self.pool // we enabled PRAGMA synchronous=normal, so this should not be necessary
.write_lock() Ok(())
.execute_batch(
"
PRAGMA synchronous=FULL;
BEGIN;
DELETE FROM _noop;
INSERT INTO _noop VALUES (1);
COMMIT;
PRAGMA synchronous=OFF;
",
)
.map_err(Into::into)
} }
} }
impl Engine { impl Engine {
pub fn flush_wal(self: &Arc<Self>) -> Result<()> { pub fn flush_wal(self: &Arc<Self>) -> Result<()> {
self.pool self.pool.write_lock().pragma_update(Some(Main), "wal_checkpoint", &"RESTART")?;
.write_lock() Ok(())
.execute_batch(
"
PRAGMA synchronous=FULL; PRAGMA wal_checkpoint=TRUNCATE;
BEGIN;
DELETE FROM _noop;
INSERT INTO _noop VALUES (1);
COMMIT;
PRAGMA wal_checkpoint=PASSIVE; PRAGMA synchronous=OFF;
",
)
.map_err(Into::into)
} }
// Reaps (at most) (.len() * `fraction`) (rounded down, min 1) connections. // Reaps (at most) (.len() * `fraction`) (rounded down, min 1) connections.