✨♻️📝 vanth, varo: Refactor storage backend, fix distribution sampling, and enhance type handling
- Refactored `Store` to use a trait-based backend with type-specific tables, replacing the previous enum approach. - Implemented `Sqlite` and `Memory` backends with proper table management and type-aware operations. - Added serialization/deserialization handling in `Store` using `serde_json`. - Implemented `ToString` for `Ty` and improved equality comparisons using string representation. - Fixed `Distribution::sample` in `varo` to correctly handle 0, 1, and 2+ moments cases. - Updated store integration tests to verify type-specific storage operations including write, read, and deletion. - Commented out unfinished `EntityContents` and related structs pending future implementation.
This commit is contained in:
parent
db531c8c73
commit
87957bfbf8
GPG key ID:
93CA5814B698101C
4 changed files with 275 additions and 105 deletions
|
|
@ -49,10 +49,10 @@ impl Node {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
pub fn load(entity_id: impl Into<EntityId>) -> Result<Option<EntityContents>> {
|
||||
// TODO
|
||||
Ok(None)
|
||||
}
|
||||
// pub fn load(entity_id: impl Into<EntityId>) -> Result<Option<EntityContents>> {
|
||||
// // TODO
|
||||
// Ok(None)
|
||||
// }
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
|
|
@ -76,11 +76,17 @@ pub struct Value {
|
|||
data: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, Eq, Hash)]
|
||||
pub struct Ty {
|
||||
pub path: Vec<String>,
|
||||
}
|
||||
|
||||
impl ToString for Ty {
|
||||
fn to_string(&self) -> String {
|
||||
self.path.join("::")
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for Ty {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.path == other.path
|
||||
|
|
@ -89,7 +95,7 @@ impl PartialEq for Ty {
|
|||
|
||||
impl <T: AsRef<str>> PartialEq<T> for Ty {
|
||||
fn eq(&self, other: &T) -> bool {
|
||||
self.path.join("::") == *other.as_ref()
|
||||
self.to_string() == *other.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -102,16 +108,16 @@ pub trait VanthTuple {
|
|||
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
pub struct EntityContents {
|
||||
components: Vec<ComponentContents>
|
||||
}
|
||||
// #[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
// pub struct EntityContents {
|
||||
// components: Vec<ComponentContents>
|
||||
// }
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
pub struct ComponentContents {
|
||||
id: String,
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ComponentContents<T: Vanth> {
|
||||
content_hash: ContentHash,
|
||||
data: Vec<u8>,
|
||||
_marker: PhantomData<T>,
|
||||
}
|
||||
|
||||
pub trait Component: Serialize {
|
||||
|
|
@ -120,7 +126,7 @@ pub trait Component: Serialize {
|
|||
|
||||
// use a macro to implement VanthTuiple here.
|
||||
|
||||
#[derive(Copy, Clone, Debug, Deserialize, Component, Serialize)]
|
||||
#[derive(Copy, Clone, Debug, Deserialize, Component, Serialize, PartialEq, Eq, Hash)]
|
||||
pub struct ContentHash {
|
||||
pub hash: [u8; 32],
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,115 +1,256 @@
|
|||
use std::{collections::HashMap, path::PathBuf};
|
||||
use std::{collections::HashMap, marker::PhantomData, path::PathBuf};
|
||||
|
||||
use rusqlite::{Connection, params};
|
||||
use rusqlite::{Connection, params, named_params};
|
||||
|
||||
use bevy_ecs::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Deserialize, Component, Serialize)]
|
||||
use crate::{hash, ComponentContents, ContentHash, Ty, Vanth};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Store {
|
||||
backend: Backend,
|
||||
backend: Box<dyn Backend>,
|
||||
}
|
||||
|
||||
type Result<T> = std::result::Result<T, String>;
|
||||
type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub enum Error {
|
||||
Serializiation(String),
|
||||
Database(String),
|
||||
}
|
||||
|
||||
impl From<serde_json::Error> for Error {
|
||||
fn from(err: serde_json::Error) -> Self {
|
||||
Error::Serializiation(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<rusqlite::Error> for Error {
|
||||
fn from(err: rusqlite::Error) -> Self {
|
||||
Error::Database(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Use an SQLite backend with a database file at the provided path.
|
||||
pub fn from_path(path: PathBuf) -> Result<Self> {
|
||||
let conn = Connection::open(&path).map_err(|e| e.to_string())?;
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS kv (key BLOB PRIMARY KEY, value BLOB)",
|
||||
params![],
|
||||
).map_err(|e| e.to_string())?;
|
||||
Ok(Self {
|
||||
backend: Backend::Sqlite(Sqlite { path }),
|
||||
backend: Box::new(Sqlite::new(path)?),
|
||||
})
|
||||
}
|
||||
|
||||
/// Use an in-memory backend.
|
||||
pub fn in_memory() -> Result<Self> {
|
||||
Ok(Self {
|
||||
backend: Backend::Memory(Memory::new()),
|
||||
backend: Box::new(Memory::new()),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn read(&mut self, key: impl AsRef<[u8]>) -> Result<Option<Vec<u8>>> {
|
||||
match &mut self.backend {
|
||||
Backend::Memory(mem) => Ok(mem.values.get(key.as_ref()).cloned()),
|
||||
Backend::Sqlite(sql) => {
|
||||
let conn = Connection::open(&sql.path).map_err(|e| e.to_string())?;
|
||||
let mut stmt = conn.prepare("SELECT value FROM kv WHERE key = ?1").map_err(|e| e.to_string())?;
|
||||
let mut rows = stmt.query(params![key.as_ref()]).map_err(|e| e.to_string())?;
|
||||
if let Some(row) = rows.next().map_err(|e| e.to_string())? {
|
||||
let value: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
|
||||
Ok(Some(value))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
pub fn get_from_hash<T: Vanth + DeserializeOwned>(&mut self, content_hash: ContentHash) -> Result<Option<T>> {
|
||||
let Some(raw) = self.get_raw_from_hash::<T>(content_hash)? else { return Ok(None) };
|
||||
|
||||
let deserialized: T = serde_json::from_slice(&raw)?;
|
||||
Ok(Some(deserialized))
|
||||
}
|
||||
|
||||
pub fn write(&mut self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Result<()> {
|
||||
match &mut self.backend {
|
||||
Backend::Memory(mem) => {
|
||||
mem.values.insert(key.as_ref().to_vec(), value.as_ref().to_vec());
|
||||
Ok(())
|
||||
}
|
||||
Backend::Sqlite(sql) => {
|
||||
let conn = Connection::open(&sql.path).map_err(|e| e.to_string())?;
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO kv (key, value) VALUES (?1, ?2)",
|
||||
params![key.as_ref(), value.as_ref()],
|
||||
).map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
pub fn get_raw_from_hash<T: Vanth>(&mut self, content_hash: ContentHash) -> Result<Option<Vec<u8>>> {
|
||||
self.backend.get_from_hash(T::ty(), content_hash)
|
||||
}
|
||||
|
||||
pub fn delete(&mut self, key: impl AsRef<[u8]>) -> Result<()> {
|
||||
match &mut self.backend {
|
||||
Backend::Memory(mem) => {
|
||||
mem.values.remove(key.as_ref());
|
||||
Ok(())
|
||||
pub fn get_all_of_type<T: Vanth>(&mut self) -> Result<Vec<ComponentContents<T>>> {
|
||||
let raw_items = self.backend.get_all_of_ty(T::ty())?;
|
||||
let mut results = Vec::new();
|
||||
for (content_hash, data) in raw_items {
|
||||
results.push(ComponentContents {
|
||||
_marker: PhantomData,
|
||||
content_hash,
|
||||
data,
|
||||
});
|
||||
}
|
||||
Backend::Sqlite(sql) => {
|
||||
let conn = Connection::open(&sql.path).map_err(|e| e.to_string())?;
|
||||
conn.execute(
|
||||
"DELETE FROM kv WHERE key = ?1",
|
||||
params![key.as_ref()],
|
||||
).map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
pub fn write<T: Vanth + Serialize>(&mut self, value: &T) -> Result<()> {
|
||||
let content_hash = hash(&value);
|
||||
let data = serde_json::to_vec(&value)?;
|
||||
self.backend.write(T::ty(), content_hash, data)
|
||||
}
|
||||
|
||||
pub fn write_raw<T: Vanth>(&mut self, content_hash: ContentHash, content: Vec<u8>) -> Result<()> {
|
||||
self.backend.write(T::ty(), content_hash, content)
|
||||
}
|
||||
|
||||
pub fn delete<T: Vanth>(&mut self, content_hash: ContentHash) -> Result<()> {
|
||||
self.backend.delete_by_hash(T::ty(), content_hash)
|
||||
}
|
||||
|
||||
pub fn delete_all<T: Vanth>(&mut self) -> Result<()> {
|
||||
self.backend.delete_all_of_ty(T::ty())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Component, Serialize)]
|
||||
pub struct Cache {
|
||||
size_limit_bytes: u64,
|
||||
backend: Backend,
|
||||
// backend: Backend,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub enum Backend {
|
||||
Memory(Memory),
|
||||
Sqlite(Sqlite)
|
||||
pub trait Backend: std::fmt::Debug {
|
||||
fn get_from_hash(&mut self, ty: Ty, content_hash: ContentHash) -> Result<Option<Vec<u8>>>;
|
||||
|
||||
fn get_all_of_ty(&mut self, ty: Ty) -> Result<Vec<(ContentHash, Vec<u8>)>>;
|
||||
|
||||
fn write(&mut self, ty: Ty, content_hash: ContentHash, content: Vec<u8>) -> Result<()>;
|
||||
|
||||
fn delete_by_hash(&mut self, ty: Ty, content_hash: ContentHash) -> Result<()>;
|
||||
|
||||
fn delete_all_of_ty(&mut self, ty: Ty) -> Result<()>;
|
||||
}
|
||||
|
||||
/// One table, key-value store. Keys and values are both blobs.
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
/// One table per type. Keys and values are both blobs.
|
||||
#[derive(Debug)]
|
||||
pub struct Sqlite {
|
||||
path: PathBuf,
|
||||
conn: Connection,
|
||||
}
|
||||
|
||||
/// One table, key-value store. Keys and values are both blobs.
|
||||
impl Sqlite {
|
||||
pub fn new(path: PathBuf) -> Result<Self> {
|
||||
let conn = Connection::open(path)?;
|
||||
Ok(Self { conn })
|
||||
}
|
||||
|
||||
fn ensure_table_exists(&self, ty: &Ty) -> Result<()> {
|
||||
let table_name = Self::table_name(ty);
|
||||
let query = format!(
|
||||
"CREATE TABLE IF NOT EXISTS \"{}\" (
|
||||
content_hash BLOB PRIMARY KEY,
|
||||
content BLOB NOT NULL
|
||||
)",
|
||||
table_name
|
||||
);
|
||||
self.conn.execute(&query, [])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn table_name(ty: &Ty) -> String {
|
||||
format!("ty_{}", ty.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl Backend for Sqlite {
|
||||
fn get_from_hash(&mut self, ty: Ty, content_hash: ContentHash) -> Result<Option<Vec<u8>>> {
|
||||
self.ensure_table_exists(&ty)?;
|
||||
let table_name = Self::table_name(&ty);
|
||||
let query = format!("SELECT content FROM \"{}\" WHERE content_hash = :hash", table_name);
|
||||
|
||||
match self.conn.query_row(&query, named_params! {":hash": content_hash.hash.as_slice()}, |row| {
|
||||
row.get::<_, Vec<u8>>(0)
|
||||
}) {
|
||||
Ok(content) => Ok(Some(content)),
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_all_of_ty(&mut self, ty: Ty) -> Result<Vec<(ContentHash, Vec<u8>)>> {
|
||||
self.ensure_table_exists(&ty)?;
|
||||
let table_name = Self::table_name(&ty);
|
||||
let query = format!("SELECT content_hash, content FROM \"{}\"", table_name);
|
||||
|
||||
let mut stmt = self.conn.prepare(&query)?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
let hash_bytes: Vec<u8> = row.get(0)?;
|
||||
let content: Vec<u8> = row.get(1)?;
|
||||
let mut hash_array = [0u8; 32];
|
||||
hash_array.copy_from_slice(&hash_bytes);
|
||||
Ok((ContentHash { hash: hash_array }, content))
|
||||
})?;
|
||||
|
||||
let mut results = Vec::new();
|
||||
for row in rows {
|
||||
results.push(row?);
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
fn write(&mut self, ty: Ty, content_hash: ContentHash, content: Vec<u8>) -> Result<()> {
|
||||
self.ensure_table_exists(&ty)?;
|
||||
let table_name = Self::table_name(&ty);
|
||||
let query = format!(
|
||||
"INSERT OR REPLACE INTO \"{}\" (content_hash, content) VALUES (:hash, :content)",
|
||||
table_name
|
||||
);
|
||||
self.conn.execute(&query, named_params! {":hash": content_hash.hash.as_slice(), ":content": content})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn delete_by_hash(&mut self, ty: Ty, content_hash: ContentHash) -> Result<()> {
|
||||
self.ensure_table_exists(&ty)?;
|
||||
let table_name = Self::table_name(&ty);
|
||||
let query = format!("DELETE FROM \"{}\" WHERE content_hash = :hash", table_name);
|
||||
self.conn.execute(&query, named_params! {":hash": content_hash.hash.as_slice()})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn delete_all_of_ty(&mut self, ty: Ty) -> Result<()> {
|
||||
let table_name = Self::table_name(&ty);
|
||||
let query = format!("DROP TABLE IF EXISTS \"{}\"", table_name);
|
||||
self.conn.execute(&query, [])?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// In-memory storage with one table per type.
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub struct Memory {
|
||||
values: HashMap<Vec<u8>, Vec<u8>>,
|
||||
tables: HashMap<Ty, HashMap<ContentHash, Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl Memory {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
values: HashMap::new(),
|
||||
tables: HashMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Backend for Memory {
|
||||
fn get_from_hash(&mut self, ty: Ty, content_hash: ContentHash) -> Result<Option<Vec<u8>>> {
|
||||
Ok(self.tables
|
||||
.get(&ty)
|
||||
.and_then(|table| table.get(&content_hash))
|
||||
.cloned())
|
||||
}
|
||||
|
||||
fn get_all_of_ty(&mut self, ty: Ty) -> Result<Vec<(ContentHash, Vec<u8>)>> {
|
||||
Ok(self.tables
|
||||
.get(&ty)
|
||||
.map(|table| {
|
||||
table.iter()
|
||||
.map(|(k, v)| (*k, v.clone()))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_else(Vec::new))
|
||||
}
|
||||
|
||||
fn write(&mut self, ty: Ty, content_hash: ContentHash, content: Vec<u8>) -> Result<()> {
|
||||
self.tables
|
||||
.entry(ty)
|
||||
.or_insert_with(HashMap::new)
|
||||
.insert(content_hash, content);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn delete_by_hash(&mut self, ty: Ty, content_hash: ContentHash) -> Result<()> {
|
||||
if let Some(table) = self.tables.get_mut(&ty) {
|
||||
table.remove(&content_hash);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn delete_all_of_ty(&mut self, ty: Ty) -> Result<()> {
|
||||
self.tables.remove(&ty);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,26 +1,47 @@
|
|||
use vanth::store::Store;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vanth::{hash, store::Store, Vanth};
|
||||
use std::path::PathBuf;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Vanth)]
|
||||
struct Foo {
|
||||
inner: i32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Vanth)]
|
||||
struct Bar {
|
||||
inner: String,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sqlite_store() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("test.db");
|
||||
let mut store = Store::from_path(path.clone()).unwrap();
|
||||
|
||||
assert_eq!(store.read(b"key_1"), Ok(None));
|
||||
assert_eq!(store.write(b"key_1", b"value_1"), Ok(()));
|
||||
let foo_1 = Foo { inner: 1 };
|
||||
let foo_2 = Foo { inner: 2 };
|
||||
let bar_1 = Bar { inner: "hello".into() };
|
||||
|
||||
let value = store.read(b"key_1").unwrap();
|
||||
assert_eq!(value.as_deref(), Some(b"value_1" as &[u8]));
|
||||
assert_eq!(store.get_all_of_type::<Foo>().unwrap().len(), 0);
|
||||
assert_eq!(store.get_all_of_type::<Bar>().unwrap().len(), 0);
|
||||
|
||||
drop(store);
|
||||
store.write(&foo_1).unwrap();
|
||||
store.write(&foo_2).unwrap();
|
||||
store.write(&bar_1).unwrap();
|
||||
assert_eq!(store.get_all_of_type::<Foo>().unwrap().len(), 2);
|
||||
assert_eq!(store.get_all_of_type::<Bar>().unwrap().len(), 1);
|
||||
|
||||
let mut store = Store::from_path(path.clone()).unwrap();
|
||||
let foo_2_hash = hash(&foo_2);
|
||||
let foo_2_fetched = store.get_from_hash(foo_2_hash).unwrap().unwrap();
|
||||
assert_ne!(foo_1, foo_2_fetched);
|
||||
assert_eq!(foo_2, foo_2_fetched);
|
||||
|
||||
let value = store.read(b"key_1").unwrap();
|
||||
assert_eq!(value.as_deref(), Some(b"value_1" as &[u8]));
|
||||
store.delete::<Foo>(foo_2_hash).unwrap();
|
||||
assert_eq!(store.get_all_of_type::<Foo>().unwrap().len(), 1);
|
||||
|
||||
store.delete(b"key_1").unwrap();
|
||||
assert_eq!(store.read(b"key_1"), Ok(None));
|
||||
store.delete_all::<Foo>().unwrap();
|
||||
store.delete_all::<Bar>().unwrap();
|
||||
assert_eq!(store.get_all_of_type::<Foo>().unwrap().len(), 0);
|
||||
assert_eq!(store.get_all_of_type::<Bar>().unwrap().len(), 0);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,11 +38,11 @@ pub fn rng_gen_f32(rng: &mut Rng) -> f32 {
|
|||
}
|
||||
|
||||
pub fn rng_gen_gaussian(rng: &mut Rng, mean: f32, std_dev: f32) -> f32 {
|
||||
let u = rng_gen_f32(rng);
|
||||
let v = rng_gen_f32(rng);
|
||||
let s = (-2.0 * (1.0 - u).ln()).sqrt();
|
||||
let angle = 2.0 * PI * v;
|
||||
mean + std_dev * s * angle.cos()
|
||||
let uniform_for_radius_calc = rng_gen_f32(rng);
|
||||
let uniform_for_angle = rng_gen_f32(rng);
|
||||
let radius = (-2.0 * (1.0 - uniform_for_radius_calc).ln()).sqrt();
|
||||
let theta = 2.0 * PI * uniform_for_angle;
|
||||
mean + std_dev * radius * theta.cos()
|
||||
}
|
||||
|
||||
pub trait Varo {
|
||||
|
|
@ -58,10 +58,12 @@ pub struct Distribution {
|
|||
|
||||
impl Distribution {
|
||||
pub fn sample(&self, digest: &mut Rng) -> f32 {
|
||||
if self.moments.len() >= 2 {
|
||||
rng_gen_gaussian(digest, self.moments[0], self.moments[1].sqrt())
|
||||
} else {
|
||||
if self.moments.is_empty() {
|
||||
rng_gen_f32(digest)
|
||||
} else if self.moments.len() == 1 {
|
||||
rng_gen_gaussian(digest, self.moments[0], 1.0)
|
||||
} else {
|
||||
rng_gen_gaussian(digest, self.moments[0], self.moments[1].sqrt())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue