From 44dd67cf1f9d395c82e96c800f435d851dd138be Mon Sep 17 00:00:00 2001 From: Ben Burdette <> Date: Tue, 21 Jul 2026 21:44:04 -0600 Subject: [PATCH 01/40] first whack at websockets private message endpoint --- Cargo.lock | 271 +++++++++++++++++++++++------------ server-lib/Cargo.toml | 1 + server-lib/src/error.rs | 3 + server-lib/src/interfaces.rs | 9 +- server-lib/src/lib.rs | 89 +++++++++++- server-lib/src/sqldata.rs | 7 +- 6 files changed, 274 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 586b3aa2..9adfad63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,9 +59,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.11.1" +version = "3.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44cceded2fb55f3c4b67068fa64962e2ca59614edc5b03167de9ff82ae803da0" +checksum = "48e2faa3e7418ed780cca54829d32782a4008a077230f67457caa063415e99c2" dependencies = [ "actix-codec", "actix-rt", @@ -75,7 +75,7 @@ dependencies = [ "derive_more 2.0.1", "encoding_rs", "flate2", - "foldhash", + "foldhash 0.2.0", "futures-core", "h2", "http 0.2.12", @@ -87,8 +87,8 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rand 0.9.2", - "sha1", + "rand 0.10.2", + "sha1 0.11.0", "smallvec", "tokio", "tokio-util", @@ -160,9 +160,9 @@ dependencies = [ [[package]] name = "actix-router" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" +checksum = "14f8c75c51892f18d9c46150c5ac7beb81c95f78c8b83a634d49f4ca32551fe7" dependencies = [ "bytestring", "cfg-if", @@ -240,7 +240,7 @@ dependencies = [ "futures-core", "http 0.2.12", "http 1.3.1", - "impl-more", + "impl-more 0.1.9", "pin-project-lite", "tokio", "tokio-rustls 0.23.4", @@ -261,9 +261,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.11.0" +version = "4.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a597b77b5c6d6a1e1097fddde329a83665e25c5437c696a3a9a4aa514a614dea" +checksum = "df09e2d9239703dd64056359c920c7f3fba6535ec61a0059e0f44e095ffe02b4" dependencies = [ "actix-codec", "actix-http", @@ -280,10 +280,10 @@ dependencies = [ "cookie 0.16.2", "derive_more 2.0.1", "encoding_rs", - "foldhash", + "foldhash 0.2.0", "futures-core", "futures-util", - "impl-more", + "impl-more 0.3.2", "itoa", "language-tags", "log", @@ -296,7 +296,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2 0.5.10", + "socket2 0.6.0", "time", "tracing", "url", @@ -315,12 +315,19 @@ dependencies = [ ] [[package]] -name = "addr2line" -version = "0.24.2" +name = "actix-ws" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +checksum = "decf53c3cdd63dd6f289980b430238f9a2f6d19f8bce8e418272e08d3da43f0f" dependencies = [ - "gimli", + "actix-codec", + "actix-http", + "actix-web", + "bytestring", + "futures-core", + "futures-sink", + "tokio", + "tokio-util", ] [[package]] @@ -335,7 +342,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "generic-array", ] @@ -347,7 +354,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -778,21 +785,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "backtrace" -version = "0.3.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", -] - [[package]] name = "barrel" version = "0.7.0" @@ -850,6 +842,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-padding" version = "0.3.3" @@ -947,6 +948,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.41" @@ -967,7 +979,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "inout", ] @@ -1040,7 +1052,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der", "spki", "x509-cert", @@ -1083,6 +1095,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "convert_case" version = "0.4.0" @@ -1166,6 +1184,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1192,6 +1219,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ctr" version = "0.9.2" @@ -1248,7 +1284,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der_derive", "flagset", "pem-rfc7468", @@ -1338,11 +1374,22 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.6", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1577,6 +1624,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1747,11 +1800,23 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasi 0.14.4+wasi-0.2.4", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + [[package]] name = "ghash" version = "0.5.1" @@ -1762,12 +1827,6 @@ dependencies = [ "polyval", ] -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - [[package]] name = "girlboss" version = "1.0.0-alpha.4" @@ -1812,7 +1871,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -1872,7 +1931,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1949,6 +2008,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -2202,6 +2270,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" +[[package]] +name = "impl-more" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134d2c4324d61664107020b79019cf6a6aec153f0b79bc9619ee9e794a5fb021" + [[package]] name = "indexmap" version = "2.11.0" @@ -2242,17 +2316,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "io-uring" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" -dependencies = [ - "bitflags 2.9.4", - "cfg-if", - "libc", -] - [[package]] name = "ipnet" version = "2.11.0" @@ -2567,15 +2630,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - [[package]] name = "oid-registry" version = "0.8.1" @@ -2656,7 +2710,7 @@ dependencies = [ "pkcs5", "rand 0.9.2", "rc2", - "sha1", + "sha1 0.10.6", "sha2", "thiserror 2.0.16", "x509-parser", @@ -2703,7 +2757,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", + "digest 0.10.7", "hmac", ] @@ -2784,9 +2838,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2" dependencies = [ "cms", - "const-oid", + "const-oid 0.9.6", "der", - "digest", + "digest 0.10.7", "spki", "x509-cert", "zeroize", @@ -2850,7 +2904,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -2995,6 +3049,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.5.6" @@ -3029,6 +3089,17 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3082,6 +3153,12 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rc2" version = "0.8.1" @@ -3277,12 +3354,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "rustc-demangle" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -3604,8 +3675,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -3615,8 +3697,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -3948,29 +4030,26 @@ checksum = "b130bd8a58c163224b44e217b4239ca7b927d82bf6cc2fea1fc561d15056e3f7" [[package]] name = "tokio" -version = "1.47.1" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", "socket2 0.6.0", "tokio-macros", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", @@ -4028,6 +4107,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] @@ -4139,9 +4219,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.18.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicase" @@ -4188,7 +4268,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "subtle", ] @@ -4780,7 +4860,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der", "spki", ] @@ -4945,6 +5025,7 @@ dependencies = [ "actix-rt", "actix-session", "actix-web", + "actix-ws", "async-stream", "awc", "barrel", diff --git a/server-lib/Cargo.toml b/server-lib/Cargo.toml index e2e60356..153d6ac9 100644 --- a/server-lib/Cargo.toml +++ b/server-lib/Cargo.toml @@ -50,3 +50,4 @@ girlboss = {version = "1.0.0-alpha.4", features = ["actix-rt"]} nom = "8.0.0" # lapin = { version = "3.7.2", default-features = false, features = [ "rustls--ring", "default-runtime" ] } lapin = { version = "2.5.5", default-features = true } +actix-ws = "0.4.0" diff --git a/server-lib/src/error.rs b/server-lib/src/error.rs index 3d459e27..da158bdf 100644 --- a/server-lib/src/error.rs +++ b/server-lib/src/error.rs @@ -1,5 +1,6 @@ use actix_session; use actix_web::error as awe; +use actix_web::ResponseError; use cookie; use girlboss; use regex; @@ -49,6 +50,8 @@ pub fn annotate_string(s: String, source: Error) -> Error { annotate(Error::String(s), source) } +impl ResponseError for Error {} + impl fmt::Display for AnnotatedE { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{} \n source: {}", self.error, self.source) diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index 250e0620..e41970e1 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -81,7 +81,7 @@ pub fn login_data_for_token( pub async fn connect_and_make_lapin_info( state: &State, - token: Option, + token: &Option, ) -> Option { // TODO: maybe attempt reconnect only every X seconds, so as not to // slow processing during rmq outage. @@ -236,7 +236,7 @@ pub async fn zk_interface_loggedin_streaming( pub async fn zk_interface_loggedin( state: &State, conn: &Connection, - token: Option, + token: &Option, uid: UserId, msg: &PrivateRequest, ) -> Result { @@ -351,8 +351,9 @@ pub async fn zk_interface_loggedin( let jid = new_jobid(state, uid); let lgb = state.girlboss.clone(); let server = state.server.clone(); - let li = connect_and_make_lapin_info(state, token.clone()).await; + let li = connect_and_make_lapin_info(state, token).await; let lapin_channelx = li.map(|li| li.channel).clone(); + let token = token.clone(); std::thread::spawn(move || { let rt = actix_rt::System::new(); @@ -410,7 +411,7 @@ pub async fn zk_interface_loggedin( jid, server, lapin_channelx, - token, + token.clone(), )); rt.run() .map_err(|e| { diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index b4fe23a6..657b61e8 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -19,11 +19,13 @@ use actix_session::{ use actix_web::{ cookie::{self, Key}, dev::Server, - web, App, HttpRequest, HttpResponse, HttpServer, Result, + rt, web, App, HttpRequest, HttpResponse, HttpServer, Result, }; +use actix_ws::AggregatedMessage; use chrono; use clap::Arg; use config::Config; +use futures_util::StreamExt as _; use futures_util::TryStreamExt as _; use girlboss::Girlboss; use interfaces::connect_and_make_lapin_info; @@ -323,7 +325,7 @@ async fn receive_files( mut payload: Multipart, req: HttpRequest, ) -> HttpResponse { - let li = connect_and_make_lapin_info(config.get_ref(), get_cookie_id(&req)).await; + let li = connect_and_make_lapin_info(config.get_ref(), &get_cookie_id(&req)).await; match make_file_notes(session, &config, &li, &mut payload).await { Ok(r) => HttpResponse::Ok().json(r), Err(e) => return HttpResponse::InternalServerError().body(format!("{:?}", e)), @@ -428,6 +430,82 @@ fn get_cookie_id(req: &HttpRequest) -> Option { req.cookie("id").map(|c| c.value().to_string()) } +async fn private_ws( + session: Session, + data: web::Data, + req: HttpRequest, + stream: web::Payload, +) -> Result { + let mut state = data.clone(); + + let token = get_cookie_id(&req); + + let (res, mut wssession, stream) = actix_ws::handle(&req, stream)?; + + let mut stream = stream + .aggregate_continuations() + // aggregate continuation frames up to 1MiB + .max_continuation_size(2_usize.pow(20)); + + rt::spawn(async move { + while let Some(msg) = stream.next().await { + match async { + match msg { + Ok(AggregatedMessage::Text(t)) => { + let prq: PrivateRequest = serde_json::from_str(&t)?; + + match zk_interface_check(&session, &mut state, &token, prq).await { + Ok(sr) => wssession.text(serde_json::to_string(&sr)?).await?, + Err(e) => { + error!("'private' err: {:?}", e); + wssession + .text(serde_json::to_string(&PrivateReply::PvyServerError( + PrivateError::PveString(e.to_string()), + ))?) + .await?; + } + } + } + Ok(AggregatedMessage::Binary(b)) => { + wssession.binary(b).await?; + } + Ok(AggregatedMessage::Ping(msg)) => { + wssession.pong(&msg).await?; + } + Ok(AggregatedMessage::Pong(msg)) => { + wssession.ping(&msg).await?; // ping? + } + Ok(AggregatedMessage::Close(_msg)) => { + // wssession.close(msg); + return Ok::<(), Box>(()); + } + + Err(e) => { + error!("error {e}"); + } + }; + Ok(()) + } + .await + { + Ok(()) => (), + Err(e) => { + error!("'user' err: {:?}", e); + let se = UserResponse::UrpServerError(e.to_string()); + match serde_json::to_string(&se) { + Ok(s) => { + wssession.text(s).await; + } + Err(_) => (), + } + } + } + } + }); + + Ok(res) +} + async fn private( session: Session, data: web::Data, @@ -438,7 +516,7 @@ async fn private( let token = get_cookie_id(&req); - match zk_interface_check(&session, &mut state, token, item.into_inner()).await { + match zk_interface_check(&session, &mut state, &token, item.into_inner()).await { Ok(sr) => HttpResponse::Ok().json(sr), Err(e) => { error!("'private' err: {:?}", e); @@ -468,7 +546,7 @@ async fn private_streaming( async fn zk_interface_check( session: &Session, state: &State, - token: Option, + token: &Option, msg: PrivateRequest, ) -> Result { match session.get::("token")? { @@ -536,7 +614,7 @@ async fn private_upstreaming( body: web::Payload, req: HttpRequest, ) -> HttpResponse { - let li = connect_and_make_lapin_info(data.get_ref(), get_cookie_id(&req)).await; + let li = connect_and_make_lapin_info(data.get_ref(), &get_cookie_id(&req)).await; match zk_interface_check_upstreaming(&session, &data.config, &li, body).await { Ok(hr) => hr, Err(e) => { @@ -959,6 +1037,7 @@ pub async fn init_server(mut config: Config) -> Result> { .service(web::resource("/upload").route(web::post().to(receive_files))) .service(web::resource("/public").route(web::post().to(public))) .service(web::resource("/private").route(web::post().to(private))) + .service(web::resource("/privatews").route(web::post().to(private_ws))) .service(web::resource("/stream").route(web::post().to(private_streaming))) .service(web::resource("/upstream").route(web::post().to(private_upstreaming))) .service(web::resource("/user").route(web::post().to(user))) diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index eb23e8d7..064939ed 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -928,11 +928,14 @@ pub struct LapinInfo { pub async fn make_lapin_info( conn: Option<&lapin::Connection>, - token: Option, + token: &Option, ) -> Option { match (conn, token) { (Some(conn), Some(token)) => match make_lapin_channels(conn).await { - Ok(lc) => Some(LapinInfo { channel: lc, token }), + Ok(lc) => Some(LapinInfo { + channel: lc, + token: token.clone(), + }), Err(e) => { error!("{e}"); None From 22d7db933230eea27f9816825b4b528e2f1272ef Mon Sep 17 00:00:00 2001 From: Ben Burdette <> Date: Wed, 22 Jul 2026 11:48:25 -0600 Subject: [PATCH 02/40] more sockets --- elm/elm.json | 1 + elm/src/DataUtil.elm | 1 + elm/src/Main.elm | 40 +++++++ server-lib/src/lib.rs | 5 +- server/static/index.html | 247 ++++++++++++++++++++++++--------------- 5 files changed, 199 insertions(+), 95 deletions(-) diff --git a/elm/elm.json b/elm/elm.json index fc3035e7..33f5c4af 100644 --- a/elm/elm.json +++ b/elm/elm.json @@ -15,6 +15,7 @@ "bburdette/schelme": "3.0.0", "bburdette/toop": "1.2.0", "bburdette/typed-collections": "1.0.2", + "bburdette/websocket": "1.0.3", "bburdette/windowkeys": "1.0.1", "ceddlyburge/dnd-list": "1.0.0", "dillonkearns/elm-markdown": "7.0.0", diff --git a/elm/src/DataUtil.elm b/elm/src/DataUtil.elm index 0627e006..4d149fe0 100644 --- a/elm/src/DataUtil.elm +++ b/elm/src/DataUtil.elm @@ -18,6 +18,7 @@ type alias FileUrlInfo = { location : String , filelocation : String , tauri : Bool + , websockets : Bool } diff --git a/elm/src/Main.elm b/elm/src/Main.elm index 5954c2f7..1283bbb6 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -65,6 +65,7 @@ import Url exposing (Url) import UserSettings import Util exposing (andMap) import View +import WebSocket import WindowKeys import ZkCommon exposing (StylePalette) @@ -93,6 +94,7 @@ type Msg | TauriAdminReplyData JD.Value | TauriPublicReplyData JD.Value | TauriTauriReplyData JD.Value + | ReceiveSocketMsg JD.Value | LoadUrl String | InternalUrl Url | TASelection JD.Value @@ -174,6 +176,7 @@ decodeFlags = |> andMap (JD.field "login" (JD.maybe DataUtil.decodeLoginData)) |> andMap (JD.field "adminsettings" OD.adminSettingsDecoder) |> andMap (JD.field "tauri" JD.bool) + |> andMap (JD.field "websockets" JD.bool) |> andMap (JD.field "mobile" JD.bool) @@ -189,6 +192,7 @@ type alias Flags = , login : Maybe DataUtil.LoginData , adminsettings : OD.AdminSettings , tauri : Bool + , websockets : Bool , mobile : Bool } @@ -869,6 +873,12 @@ showMessage msg = TauriTauriReplyData _ -> "TauriTauriReplyData" + ReceiveSocketMsg _ -> + "ReceiveSocketMsg" + + SelectDialogMsg _ -> + "SelectDialogMsg" + SelectDialogMsg _ -> "SelectDialogMsg" @@ -1381,6 +1391,9 @@ sendZIMsg fui msg = if fui.tauri then sendZIMsgTauri <| PrivateClosureRequest Nothing msg + else if fui.websockets then + sendSocketCommand (WebSocket.encodeCmd <| WebSocket.Send { name = "private", content = JE.encode 2 (Data.privateRequestEncoder msg) }) + else HE.postJsonTask { url = fui.location ++ "/private" @@ -5223,6 +5236,7 @@ init flags url key zone fontsize = { location = flags.location , filelocation = flags.filelocation , tauri = flags.tauri + , websockets = flags.websockets } , navkey = key , seed = seed @@ -5270,6 +5284,24 @@ init flags url key zone fontsize = , { key = "l", ctrl = True, alt = True, shift = False, preventDefault = True } ] + opensock = + if flags.websockets then + sendSocketCommand + (WebSocket.encodeCmd <| + WebSocket.Connect + { name = "private" + , address = + flags.location + -- |> String.replace "https" "ws" + |> String.replace "http" "ws" + |> (\s -> s ++ "/privatews") + , protocol = "" + } + ) + + else + Cmd.none + ( m, c ) = initToRoute imodel imodel.initialRoute in @@ -5278,6 +5310,7 @@ init flags url key zone fontsize = [ c , geterrornote , setkeys + , opensock ] ) @@ -5349,6 +5382,7 @@ main = , receiveUITauriResponse TauriUserReplyData , receivePITauriResponse TauriPublicReplyData , receiveTITauriResponse TauriTauriReplyData + , receiveSocketMsg ReceiveSocketMsg ] ++ rdysubs , onUrlRequest = urlRequest @@ -5412,3 +5446,9 @@ port sendKeyCommand : JE.Value -> Cmd msg skcommand : WindowKeys.WindowKeyCmd -> Cmd Msg skcommand = WindowKeys.send sendKeyCommand + + +port receiveSocketMsg : (JD.Value -> msg) -> Sub msg + + +port sendSocketCommand : JE.Value -> Cmd msg diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index 657b61e8..ad6699a4 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -19,7 +19,7 @@ use actix_session::{ use actix_web::{ cookie::{self, Key}, dev::Server, - rt, web, App, HttpRequest, HttpResponse, HttpServer, Result, + rt, web, App, HttpRequest, HttpResponse, HttpServer, ResponseError, Result, }; use actix_ws::AggregatedMessage; use chrono; @@ -436,6 +436,7 @@ async fn private_ws( req: HttpRequest, stream: web::Payload, ) -> Result { + info!("private_ws"); let mut state = data.clone(); let token = get_cookie_id(&req); @@ -1037,7 +1038,7 @@ pub async fn init_server(mut config: Config) -> Result> { .service(web::resource("/upload").route(web::post().to(receive_files))) .service(web::resource("/public").route(web::post().to(public))) .service(web::resource("/private").route(web::post().to(private))) - .service(web::resource("/privatews").route(web::post().to(private_ws))) + .service(web::resource("/privatews").route(web::get().to(private_ws))) .service(web::resource("/stream").route(web::post().to(private_streaming))) .service(web::resource("/upstream").route(web::post().to(private_upstreaming))) .service(web::resource("/user").route(web::post().to(user))) diff --git a/server/static/index.html b/server/static/index.html index 124cf129..7170d0a3 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -1,96 +1,157 @@ - - - - - - - - - -
- - + + + + + + + + + + +
+ + From fcb4b8d3972fd300126e55a974c65a2304459a86 Mon Sep 17 00:00:00 2001 From: Ben Burdette <> Date: Wed, 22 Jul 2026 15:48:23 -0600 Subject: [PATCH 03/40] private msgs working through websockets! --- elm/src/Main.elm | 59 +++++++++++++++++++-- server-lib/src/lib.rs | 116 ++++++++++++++++++++++++++++++------------ 2 files changed, 138 insertions(+), 37 deletions(-) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index 1283bbb6..8c4649c3 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -879,9 +879,6 @@ showMessage msg = SelectDialogMsg _ -> "SelectDialogMsg" - SelectDialogMsg _ -> - "SelectDialogMsg" - ChangePasswordDialogMsg _ -> "ChangePasswordDialogMsg" @@ -1392,7 +1389,15 @@ sendZIMsg fui msg = sendZIMsgTauri <| PrivateClosureRequest Nothing msg else if fui.websockets then - sendSocketCommand (WebSocket.encodeCmd <| WebSocket.Send { name = "private", content = JE.encode 2 (Data.privateRequestEncoder msg) }) + sendSocketCommand + (WebSocket.encodeCmd <| + WebSocket.Send + { name = "private" + , content = + JE.encode 0 + (Data.privateClosureRequestEncoder (PrivateClosureRequest Nothing msg)) + } + ) else HE.postJsonTask @@ -2249,6 +2254,52 @@ actualupdate msg model = , Cmd.none ) + ( ReceiveSocketMsg jd, _ ) -> + case JD.decodeValue WebSocket.decodeMsg jd of + Ok (WebSocket.Error wsm) -> + ( displayMessageDialog model <| "websocket error: " ++ wsm.error + , Cmd.none + ) + + Ok (WebSocket.Data wsm) -> + if wsm.name == "private" then + case JD.decodeString (makeTDDecoder Data.privateClosureReplyDecoder) wsm.data of + Ok td -> + let + _ = + Debug.log "td data" td + in + case td.data.closureId of + Just id -> + case Dict.get id model.ziClosures of + Just closure -> + let + cmsg = + closure (Ok ( td.utc, td.data.reply )) + in + actualupdate cmsg model + + Nothing -> + actualupdate (ZkReplyData (Ok ( td.utc, td.data.reply ))) model + + Nothing -> + actualupdate (ZkReplyData (Ok ( td.utc, td.data.reply ))) model + + Err e -> + ( displayMessageDialog model <| JD.errorToString e ++ "\n" ++ JE.encode 2 jd + , Cmd.none + ) + + else + ( displayMessageDialog model <| "unknown websocket connection - \"" ++ wsm.name ++ "\"" + , Cmd.none + ) + + Err e -> + ( displayMessageDialog model <| JD.errorToString e ++ "\n" ++ JE.encode 2 jd + , Cmd.none + ) + ( TauriUserReplyData jd, _ ) -> case JD.decodeValue OD.userResponseDecoder jd of Ok d -> diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index ad6699a4..a5858b1d 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -39,16 +39,17 @@ pub use orgauth::{ }; pub use rusqlite; use rusqlite::Connection; +use serde::{Deserialize, Serialize}; use serde_json; use simple_error::simple_error; use sqldata::{get_single_value, local_server_id, LapinInfo}; -use std::fs::File; use std::io::{stdin, Write}; use std::path::Path; use std::path::PathBuf; use std::str::FromStr; use std::{env, sync::RwLock}; use std::{error::Error, sync::Arc}; +use std::{fs::File, time::SystemTime}; use timer; use tokio_util::io::StreamReader; use tracing_actix_web::TracingLogger; @@ -58,7 +59,9 @@ pub use zkprotocol::content as zc; pub use zkprotocol::messages::PrivateStreamingMessage; pub use zkprotocol::search as zs; use zkprotocol::{ - private::{PrivateError, PrivateReply, PrivateRequest}, + private::{ + PrivateClosureReply, PrivateClosureRequest, PrivateError, PrivateReply, PrivateRequest, + }, public::{PublicError, PublicReply, PublicRequest}, upload::UploadReply, }; @@ -237,7 +240,7 @@ fn session_user( None => Err(zkerr::Error::NotLoggedIn)?, Some(token) => orgauth::dbfun::read_user_by_token_api( &conn, - token, + &token, state.config.orgauth_config.login_token_expiration_ms, state.config.orgauth_config.regen_login_tokens, ) @@ -430,6 +433,11 @@ fn get_cookie_id(req: &HttpRequest) -> Option { req.cookie("id").map(|c| c.value().to_string()) } +#[derive(Serialize, Deserialize)] +pub struct PrivateTimedData { + utcmillis: u128, + data: PrivateClosureReply, +} async fn private_ws( session: Session, data: web::Data, @@ -440,6 +448,17 @@ async fn private_ws( let mut state = data.clone(); let token = get_cookie_id(&req); + info!("private_ws token {:?}", token); + + let token_uuid = match session.get::("token")? { + Some(tu) => tu, + None => { + return Ok( + HttpResponse::Ok().json(PrivateReply::PvyServerError(PrivateError::PveNotLoggedIn)), + ) + } + }; + info!("private_ws sess token {:?}", token_uuid); let (res, mut wssession, stream) = actix_ws::handle(&req, stream)?; @@ -453,11 +472,34 @@ async fn private_ws( match async { match msg { Ok(AggregatedMessage::Text(t)) => { - let prq: PrivateRequest = serde_json::from_str(&t)?; - - match zk_interface_check(&session, &mut state, &token, prq).await { - Ok(sr) => wssession.text(serde_json::to_string(&sr)?).await?, - Err(e) => { + let pcr: PrivateClosureRequest = serde_json::from_str(&t)?; + info!("private_ws sess token 2 {:?}", session.get::("token")); + match ( + zk_interface_check(&token_uuid, &mut state, &token, pcr.request).await, + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map(|n| n.as_millis()), + ) { + (Ok(sr), Ok(t)) => { + let pcreply = PrivateTimedData { + utcmillis: t, + data: PrivateClosureReply { + closure_id: pcr.closure_id, + reply: sr, + }, + }; + + wssession.text(serde_json::to_string(&pcreply)?).await? + } + (_, Err(e)) => { + error!("'private' err: {:?}", e); + wssession + .text(serde_json::to_string(&PrivateReply::PvyServerError( + PrivateError::PveString(e.to_string()), + ))?) + .await?; + } + (Err(e), _) => { error!("'private' err: {:?}", e); wssession .text(serde_json::to_string(&PrivateReply::PvyServerError( @@ -516,8 +558,19 @@ async fn private( let mut state = data.clone(); let token = get_cookie_id(&req); + let token_uuid = match session.get::("token") { + Ok(None) => { + return HttpResponse::Ok().json(PrivateReply::PvyServerError(PrivateError::PveNotLoggedIn)) + } + Ok(Some(token_uuid)) => token_uuid, + Err(e) => { + return HttpResponse::Ok().json(PrivateReply::PvyServerError(PrivateError::PveString( + e.to_string(), + ))) + } + }; - match zk_interface_check(&session, &mut state, &token, item.into_inner()).await { + match zk_interface_check(&token_uuid, &mut state, &token, item.into_inner()).await { Ok(sr) => HttpResponse::Ok().json(sr), Err(e) => { error!("'private' err: {:?}", e); @@ -545,33 +598,30 @@ async fn private_streaming( } async fn zk_interface_check( - session: &Session, + token_uuid: &Uuid, state: &State, token: &Option, msg: PrivateRequest, ) -> Result { - match session.get::("token")? { - None => Ok(PrivateReply::PvyServerError(PrivateError::PveNotLoggedIn)), - Some(token_uuid) => { - let conn = sqldata::connection_open(state.config.orgauth_config.db.as_path())?; - match orgauth::dbfun::read_user_by_token_api( - &conn, - token_uuid, - state.config.orgauth_config.login_token_expiration_ms, - state.config.orgauth_config.regen_login_tokens, - ) { - Err(e) => { - info!("read_user_by_token_api error2: {:?}, {:?}", token_uuid, e); + // println!("zkcheck token: {:?}", session.get::("token")?); - Ok(PrivateReply::PvyServerError(PrivateError::PveLoginError( - e.to_string(), - ))) - } - Ok(userdata) => { - // finally! processing messages as logged in user. - interfaces::zk_interface_loggedin(state, &conn, token, userdata.id, &msg).await - } - } + let conn = sqldata::connection_open(state.config.orgauth_config.db.as_path())?; + match orgauth::dbfun::read_user_by_token_api( + &conn, + token_uuid, + state.config.orgauth_config.login_token_expiration_ms, + state.config.orgauth_config.regen_login_tokens, + ) { + Err(e) => { + info!("read_user_by_token_api error2: {:?}, {:?}", token_uuid, e); + + Ok(PrivateReply::PvyServerError(PrivateError::PveLoginError( + e.to_string(), + ))) + } + Ok(userdata) => { + // finally! processing messages as logged in user. + interfaces::zk_interface_loggedin(state, &conn, token, userdata.id, &msg).await } } } @@ -588,7 +638,7 @@ async fn zk_interface_check_streaming( let conn = sqldata::connection_open(config.orgauth_config.db.as_path())?; match orgauth::dbfun::read_user_by_token_api( &conn, - token, + &token, config.orgauth_config.login_token_expiration_ms, config.orgauth_config.regen_login_tokens, ) { @@ -647,7 +697,7 @@ async fn zk_interface_check_upstreaming( match orgauth::dbfun::read_user_by_token_api( &conn, - token, + &token, config.orgauth_config.login_token_expiration_ms, config.orgauth_config.regen_login_tokens, ) { From c06b310a493a20b7187e84d634664aeff218b613 Mon Sep 17 00:00:00 2001 From: Ben Burdette <> Date: Wed, 22 Jul 2026 15:50:32 -0600 Subject: [PATCH 04/40] bump orgauth --- orgauth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgauth b/orgauth index 2ab12256..766b86fe 160000 --- a/orgauth +++ b/orgauth @@ -1 +1 @@ -Subproject commit 2ab122564b991f20186257cef3298d323b102d97 +Subproject commit 766b86fed2f30ea0dcd6d6e811ebe874a0abf293 From 9d4f2bd3a35e74ea84d58e3d335b28ee509a0066 Mon Sep 17 00:00:00 2001 From: Ben Burdette <> Date: Fri, 24 Jul 2026 10:38:59 -0600 Subject: [PATCH 05/40] fix warning --- server-lib/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index a5858b1d..75a4b7f3 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -537,7 +537,8 @@ async fn private_ws( let se = UserResponse::UrpServerError(e.to_string()); match serde_json::to_string(&se) { Ok(s) => { - wssession.text(s).await; + // we're ignoring it if reporting the error fails too. + let _ = wssession.text(s).await; } Err(_) => (), } From 1461c724ae79973382309909395fffc6588bc797 Mon Sep 17 00:00:00 2001 From: Ben Burdette <> Date: Fri, 24 Jul 2026 11:53:43 -0600 Subject: [PATCH 06/40] debuggin --- elm/src/Main.elm | 25 +++++++++++++++++++++---- server/static/index.html | 14 +++++++------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index 8c4649c3..c1e50a61 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -1925,11 +1925,17 @@ displayMessageNLinkDialog model message url text = onZkNoteStateEditWhat : Model -> Time.Posix -> DataUtil.ZkNoteAndStateWhat -> ( Model, Cmd Msg ) onZkNoteStateEditWhat model pt znew = let + _ = + Debug.log "onZkNoteStateEditWhat what=?" "" + state = model.state in if znew.what == "cache" then let + _ = + Debug.log "onZkNoteStateEditWhat what=cache" "" + noteCache = NC.addNote pt znew.znl.znal.zknote.id (NC.ZNAL znew.znl) model.noteCache |> NC.purgeNotes @@ -2265,10 +2271,10 @@ actualupdate msg model = if wsm.name == "private" then case JD.decodeString (makeTDDecoder Data.privateClosureReplyDecoder) wsm.data of Ok td -> - let - _ = - Debug.log "td data" td - in + -- let + -- _ = + -- Debug.log "td data" ( td.data.reply.what, td.reply.zknote.id ) + -- in case td.data.closureId of Just id -> case Dict.get id model.ziClosures of @@ -2280,9 +2286,17 @@ actualupdate msg model = actualupdate cmsg model Nothing -> + let + _ = + Debug.log "actualupdate 1" "" + in actualupdate (ZkReplyData (Ok ( td.utc, td.data.reply ))) model Nothing -> + let + _ = + Debug.log "actualupdate 2" "" + in actualupdate (ZkReplyData (Ok ( td.utc, td.data.reply ))) model Err e -> @@ -3318,6 +3332,9 @@ actualupdate msg model = Data.PvyZkNoteAndLinksWhat znew -> let + _ = + Debug.log "Data.PvyZkNoteAndLinksWhat znew" znew.what + action = \mbstate -> let diff --git a/server/static/index.html b/server/static/index.html index 7170d0a3..b3b5dcde 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -103,20 +103,20 @@ var sockQueue = {}; function sendSocketCommand(wat) { - console.log( "ssc: " + JSON.stringify(wat, null, 4)); + // console.log( "ssc: " + JSON.stringify(wat, null, 4)); if (wat.cmd == "connect") { - console.log("connecting!"); + // console.log("connecting!"); // let socket = new WebSocket(wat.address, wat.protocol); let socket = new WebSocket(wat.address); socket.onmessage = function (event) { - console.log( "onmessage: " + JSON.stringify(event.data, null, 4)); + // console.log( "onmessage: " + JSON.stringify(event.data, null, 4)); app.ports.receiveSocketMsg.send({ name : wat.name , msg : "data" , data : event.data} ); } socket.onopen = function (event) { - console.log( "onopen: " + JSON.stringify(event.data, null, 4)); + // console.log( "onopen: " + JSON.stringify(event.data, null, 4)); // send queued messages. let sq = sockQueue[wat.name]; if (sq) { @@ -130,12 +130,12 @@ } else if (wat.cmd == "send") { - console.log("sending to socket: " + wat.name, mySockets[wat.name].readyState); + // console.log("sending to socket: " + wat.name, mySockets[wat.name].readyState); if (mySockets[wat.name].readyState) { mySockets[wat.name].send(wat.content); } else { - console.log("queuing message", wat.name, wat.content); + // console.log("queuing message", wat.name, wat.content); let sq = sockQueue[wat.name]; if (sq) { sq.push(wat.content); @@ -147,7 +147,7 @@ } else if (wat.cmd == "close") { - console.log("closing socket: " + wat.name); + // console.log("closing socket: " + wat.name); mySockets[wat.name].close(); delete mySockets[wat.name]; } From 9079571468c8716749cb831f9ee163d051651797 Mon Sep 17 00:00:00 2001 From: Ben Burdette <> Date: Fri, 24 Jul 2026 12:47:11 -0600 Subject: [PATCH 07/40] deal with simultaneous localValActions; remove model from closure --- elm/src/Main.elm | 50 +++++++++++++++++++++++++------------- server/static/localvals.js | 2 +- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index c1e50a61..bbe92770 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -53,6 +53,7 @@ import ShowMessage import SlideShow import SpecialNotes as SN import SpecialNotesGui as SNG +import TDict import TSet import TagAThing import TagFiles @@ -212,7 +213,7 @@ type LocalValAction = LocalValAction { for : String , name : String - , action : Maybe String -> ( Model, Cmd Msg ) + , action : Model -> Maybe String -> ( Model, Cmd Msg ) } @@ -237,7 +238,7 @@ type alias Model = , ziClosures : Dict Int (Result Http.Error ( Time.Posix, Data.PrivateReply ) -> Msg) , mobile : Bool , spmodel : SP.Model - , localValAction : Maybe LocalValAction + , localValAction : Dict String LocalValAction , zknSearchResult : Data.ZkListNoteSearchResult } @@ -2225,15 +2226,30 @@ actualupdate msg model = ( nmd, cmd ) ( ReceiveLocalVal lv, _ ) -> - case model.localValAction of + let + _ = + Debug.log "recievelocalval" lv + + _ = + Debug.log "localValAction keys" <| Dict.keys model.localValAction + in + case Dict.get lv.name model.localValAction of Just (LocalValAction lva) -> + let + _ = + Debug.log "recievelocalval got action " lv + in if lv.for == lva.for && lv.name == lva.name then - lva.action lv.value + lva.action { model | localValAction = Dict.remove lv.name model.localValAction } lv.value else ( model, Cmd.none ) Nothing -> + let + _ = + Debug.log "recievelocalval nothing " lv + in ( model, Cmd.none ) ( TauriZkReplyData jd, _ ) -> @@ -2800,7 +2816,7 @@ actualupdate msg model = Data.PbyZkNoteAndLinks znl -> let action = - \mbstate -> + \amodel mbstate -> let znas = { znal = znl, mbstate = mbstate } @@ -2821,21 +2837,21 @@ actualupdate msg model = ngets = makePubNoteCacheGets model znl.zknote.content in - ( { model | state = vstate } + ( { amodel | state = vstate } , Cmd.batch ngets ) lid = SNG.localDataId znl.zknote.id in - ( { model | localValAction = Just <| LocalValAction { for = "lva", name = lid, action = action } } - , LS.getLocalVal { for = "lva", name = lid } + ( { model | localValAction = Dict.insert lid (LocalValAction { for = "lva", name = lid, action = action }) model.localValAction } + , LS.getLocalVal { for = "lva", name = Debug.log "getlocalval1" lid } ) Data.PbyZkNoteAndLinksWhat znlw -> let action = - \mbstate -> + \amodel mbstate -> let znas : DataUtil.ZkNoteAndStateWhat znas = @@ -2843,13 +2859,13 @@ actualupdate msg model = , what = znlw.what } in - onZkNoteStatePbWhat model pt znas + onZkNoteStatePbWhat amodel pt znas lid = SNG.localDataId znlw.znl.zknote.id in - ( { model | localValAction = Just <| LocalValAction { for = "lva", name = lid, action = action } } - , LS.getLocalVal { for = "lva", name = lid } + ( { model | localValAction = Dict.insert lid (LocalValAction { for = "lva", name = lid, action = action }) model.localValAction } + , LS.getLocalVal { for = "lva", name = Debug.log "getlocalval2" lid } ) Data.PbyNoop -> @@ -3336,7 +3352,7 @@ actualupdate msg model = Debug.log "Data.PvyZkNoteAndLinksWhat znew" znew.what action = - \mbstate -> + \amodel mbstate -> let znas : DataUtil.ZkNoteAndStateWhat znas = @@ -3344,13 +3360,13 @@ actualupdate msg model = , what = znew.what } in - onZkNoteStateEditWhat model pt znas + onZkNoteStateEditWhat amodel pt znas lid = SNG.localDataId znew.znl.zknote.id in - ( { model | localValAction = Just <| LocalValAction { for = "lva", name = lid, action = action } } - , LS.getLocalVal { for = "lva", name = lid } + ( { model | localValAction = Dict.insert lid (LocalValAction { for = "lva", name = lid, action = action }) model.localValAction } + , LS.getLocalVal { for = "lva", name = Debug.log "getlocalval3" lid } ) Data.PvyZkNoteComments zc -> @@ -5323,7 +5339,7 @@ init flags url key zone fontsize = , ziClosures = Dict.empty , mobile = flags.mobile , spmodel = SP.initModel - , localValAction = Nothing + , localValAction = Dict.empty , zknSearchResult = { notes = [] , offset = 0 diff --git a/server/static/localvals.js b/server/static/localvals.js index 69cb34d9..66832da0 100644 --- a/server/static/localvals.js +++ b/server/static/localvals.js @@ -1,4 +1,4 @@ -let localvalsdebug = false; +let localvalsdebug = true; function storeVal( nv ) { localStorage.setItem(nv.name, nv.value); From 49ef9e733ab967fe6e30a8d2cab7963bc30a3155 Mon Sep 17 00:00:00 2001 From: Ben Burdette <> Date: Fri, 24 Jul 2026 12:48:42 -0600 Subject: [PATCH 08/40] remove debugs --- elm/src/Main.elm | 42 +++--------------------------------------- 1 file changed, 3 insertions(+), 39 deletions(-) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index bbe92770..58a35a77 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -1926,17 +1926,11 @@ displayMessageNLinkDialog model message url text = onZkNoteStateEditWhat : Model -> Time.Posix -> DataUtil.ZkNoteAndStateWhat -> ( Model, Cmd Msg ) onZkNoteStateEditWhat model pt znew = let - _ = - Debug.log "onZkNoteStateEditWhat what=?" "" - state = model.state in if znew.what == "cache" then let - _ = - Debug.log "onZkNoteStateEditWhat what=cache" "" - noteCache = NC.addNote pt znew.znl.znal.zknote.id (NC.ZNAL znew.znl) model.noteCache |> NC.purgeNotes @@ -2226,19 +2220,8 @@ actualupdate msg model = ( nmd, cmd ) ( ReceiveLocalVal lv, _ ) -> - let - _ = - Debug.log "recievelocalval" lv - - _ = - Debug.log "localValAction keys" <| Dict.keys model.localValAction - in case Dict.get lv.name model.localValAction of Just (LocalValAction lva) -> - let - _ = - Debug.log "recievelocalval got action " lv - in if lv.for == lva.for && lv.name == lva.name then lva.action { model | localValAction = Dict.remove lv.name model.localValAction } lv.value @@ -2246,10 +2229,6 @@ actualupdate msg model = ( model, Cmd.none ) Nothing -> - let - _ = - Debug.log "recievelocalval nothing " lv - in ( model, Cmd.none ) ( TauriZkReplyData jd, _ ) -> @@ -2287,10 +2266,6 @@ actualupdate msg model = if wsm.name == "private" then case JD.decodeString (makeTDDecoder Data.privateClosureReplyDecoder) wsm.data of Ok td -> - -- let - -- _ = - -- Debug.log "td data" ( td.data.reply.what, td.reply.zknote.id ) - -- in case td.data.closureId of Just id -> case Dict.get id model.ziClosures of @@ -2302,17 +2277,9 @@ actualupdate msg model = actualupdate cmsg model Nothing -> - let - _ = - Debug.log "actualupdate 1" "" - in actualupdate (ZkReplyData (Ok ( td.utc, td.data.reply ))) model Nothing -> - let - _ = - Debug.log "actualupdate 2" "" - in actualupdate (ZkReplyData (Ok ( td.utc, td.data.reply ))) model Err e -> @@ -2845,7 +2812,7 @@ actualupdate msg model = SNG.localDataId znl.zknote.id in ( { model | localValAction = Dict.insert lid (LocalValAction { for = "lva", name = lid, action = action }) model.localValAction } - , LS.getLocalVal { for = "lva", name = Debug.log "getlocalval1" lid } + , LS.getLocalVal { for = "lva", name = lid } ) Data.PbyZkNoteAndLinksWhat znlw -> @@ -2865,7 +2832,7 @@ actualupdate msg model = SNG.localDataId znlw.znl.zknote.id in ( { model | localValAction = Dict.insert lid (LocalValAction { for = "lva", name = lid, action = action }) model.localValAction } - , LS.getLocalVal { for = "lva", name = Debug.log "getlocalval2" lid } + , LS.getLocalVal { for = "lva", name = lid } ) Data.PbyNoop -> @@ -3348,9 +3315,6 @@ actualupdate msg model = Data.PvyZkNoteAndLinksWhat znew -> let - _ = - Debug.log "Data.PvyZkNoteAndLinksWhat znew" znew.what - action = \amodel mbstate -> let @@ -3366,7 +3330,7 @@ actualupdate msg model = SNG.localDataId znew.znl.zknote.id in ( { model | localValAction = Dict.insert lid (LocalValAction { for = "lva", name = lid, action = action }) model.localValAction } - , LS.getLocalVal { for = "lva", name = Debug.log "getlocalval3" lid } + , LS.getLocalVal { for = "lva", name = lid } ) Data.PvyZkNoteComments zc -> From 100f1a6586bb1d85c54a60821723df0a24df894a Mon Sep 17 00:00:00 2001 From: Ben Burdette <> Date: Fri, 24 Jul 2026 13:05:31 -0600 Subject: [PATCH 09/40] turn off debug; delete some infos; comment --- elm/src/Main.elm | 2 +- server-lib/src/lib.rs | 5 ----- server/static/index.html | 1 - server/static/localvals.js | 2 +- 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index 58a35a77..a353115c 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -5340,7 +5340,7 @@ init flags url key zone fontsize = { name = "private" , address = flags.location - -- |> String.replace "https" "ws" + -- http -> ws, https -> wss |> String.replace "http" "ws" |> (\s -> s ++ "/privatews") , protocol = "" diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index 75a4b7f3..f62e3169 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -448,7 +448,6 @@ async fn private_ws( let mut state = data.clone(); let token = get_cookie_id(&req); - info!("private_ws token {:?}", token); let token_uuid = match session.get::("token")? { Some(tu) => tu, @@ -458,7 +457,6 @@ async fn private_ws( ) } }; - info!("private_ws sess token {:?}", token_uuid); let (res, mut wssession, stream) = actix_ws::handle(&req, stream)?; @@ -473,7 +471,6 @@ async fn private_ws( match msg { Ok(AggregatedMessage::Text(t)) => { let pcr: PrivateClosureRequest = serde_json::from_str(&t)?; - info!("private_ws sess token 2 {:?}", session.get::("token")); match ( zk_interface_check(&token_uuid, &mut state, &token, pcr.request).await, SystemTime::now() @@ -604,8 +601,6 @@ async fn zk_interface_check( token: &Option, msg: PrivateRequest, ) -> Result { - // println!("zkcheck token: {:?}", session.get::("token")?); - let conn = sqldata::connection_open(state.config.orgauth_config.db.as_path())?; match orgauth::dbfun::read_user_by_token_api( &conn, diff --git a/server/static/index.html b/server/static/index.html index b3b5dcde..697d7e72 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -107,7 +107,6 @@ if (wat.cmd == "connect") { // console.log("connecting!"); - // let socket = new WebSocket(wat.address, wat.protocol); let socket = new WebSocket(wat.address); socket.onmessage = function (event) { // console.log( "onmessage: " + JSON.stringify(event.data, null, 4)); diff --git a/server/static/localvals.js b/server/static/localvals.js index 66832da0..69cb34d9 100644 --- a/server/static/localvals.js +++ b/server/static/localvals.js @@ -1,4 +1,4 @@ -let localvalsdebug = true; +let localvalsdebug = false; function storeVal( nv ) { localStorage.setItem(nv.name, nv.value); From 5d78d0adadd7cca383768471f45be705fd8c0344 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 6 Oct 2026 17:50:03 -0600 Subject: [PATCH 10/40] merge master; token uuids by reference --- elm/src/EditZkNote.elm | 17 ++++++----------- elm/src/Main.elm | 1 + elm/src/MdCommon.elm | 36 ++++++++++++++++++------------------ orgauth | 2 +- server-lib/src/interfaces.rs | 3 ++- server-lib/src/sqldata.rs | 10 ++++++---- 6 files changed, 34 insertions(+), 35 deletions(-) diff --git a/elm/src/EditZkNote.elm b/elm/src/EditZkNote.elm index c2de5ea0..9c2f4c25 100644 --- a/elm/src/EditZkNote.elm +++ b/elm/src/EditZkNote.elm @@ -3225,18 +3225,13 @@ onWkKeyPress noteCache key model = Toop.T4 "Enter" False False False -> ( model, SPMod SP.onEnter ) - Toop.T4 "Escape" False False False -> - -- close blockedit if no changes. - case model.blockEdit of - Just (Text t) -> - if t.s == t.original then - ( { model | blockEdit = Nothing }, None ) - - else - ( model, None ) + Toop.T4 "Enter" True False False -> + -- ctrl-enter to merge block changes. + ( mergeEditBlock model, None ) - Nothing -> - ( model, None ) + Toop.T4 "Escape" False False False -> + -- close blockedit, cancel changes. + ( { model | blockEdit = Nothing }, None ) _ -> ( model, None ) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index a353115c..eaeea9c0 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -5328,6 +5328,7 @@ init flags url key zone fontsize = , { key = "r", ctrl = True, alt = True, shift = False, preventDefault = True } , { key = "v", ctrl = True, alt = True, shift = False, preventDefault = True } , { key = "Enter", ctrl = False, alt = False, shift = False, preventDefault = False } + , { key = "Enter", ctrl = True, alt = False, shift = False, preventDefault = False } , { key = "Escape", ctrl = False, alt = False, shift = False, preventDefault = False } , { key = "l", ctrl = True, alt = True, shift = False, preventDefault = True } ] diff --git a/elm/src/MdCommon.elm b/elm/src/MdCommon.elm index b0f386fd..5f78b1f9 100644 --- a/elm/src/MdCommon.elm +++ b/elm/src/MdCommon.elm @@ -178,10 +178,10 @@ link destination body = String.contains ":" destination in (if locallink then - E.newTabLink + E.link else - E.link + E.newTabLink ) [ E.htmlAttribute (HA.style "display" "inline-flex") ] { url = destination @@ -191,7 +191,12 @@ link destination body = , E.htmlAttribute (HA.style "overflow-wrap" "break-word") , E.htmlAttribute (HA.style "word-break" "break-word") ] - (if locallink then body else (body ++ [E.text "⇗"]) ) + (if locallink then + body + + else + body ++ [ E.text " ⇗" ] + ) } @@ -223,7 +228,12 @@ nooplink dirty destination body noop = , E.htmlAttribute (HA.style "overflow-wrap" "break-word") , E.htmlAttribute (HA.style "word-break" "break-word") ] - (if locallink then body else (body ++ [E.text " ⇗"]) ) + (if locallink then + body + + else + body ++ [ E.text " ⇗" ] + ) } @@ -639,11 +649,11 @@ audioNoteView fui autoplay mbOnEnded zkn = , if fui.tauri || List.filter (\i -> i == DataUtil.sysids.publicid) zkn.sysids /= [] then link ("https://29a.ch/timestretch/#a=" ++ fui.location ++ "/file/" ++ zkNoteIdToString zkn.id) - [ E.text "ts↗" ] + [ E.text "ts" ] else E.el [ Util.addToolTip E.below (ZkCommon.stringToolTip "disabled for private notes") ] - (E.el [ EF.color TC.darkGrey ] <| E.text "ts↗") + (E.el [ EF.color TC.darkGrey ] <| E.text "ts⇗") ] ] @@ -753,12 +763,7 @@ yeetView args url audioOnly mbid show text _ = ++ (audioOnly |> Maybe.map (\_ -> " -x") |> Maybe.withDefault "") , nooplink args.isDirty url - [ E.el - [ E.inFront (E.el [ E.centerY ] <| E.text "↗") - ] - <| - E.text "☐" - ] + [] args.noop ] @@ -768,12 +773,7 @@ yeetView args url audioOnly mbid show text _ = [ E.text <| "yeet " ++ url , nooplink args.isDirty url - [ E.el - [ E.inFront (E.el [ E.centerY ] <| E.text "↗") - ] - <| - E.text "☐" - ] + [] args.noop , E.text (audioOnly |> Maybe.map (\_ -> " -x") |> Maybe.withDefault "") ] diff --git a/orgauth b/orgauth index 766b86fe..e8e40147 160000 --- a/orgauth +++ b/orgauth @@ -1 +1 @@ -Subproject commit 766b86fed2f30ea0dcd6d6e811ebe874a0abf293 +Subproject commit e8e4014717e59ffd92a5ff7ff74e50ffeb671874 diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index e41970e1..3b43f112 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -40,6 +40,7 @@ use zkprotocol::private::PrivateReply; use zkprotocol::private::PrivateRequest; use zkprotocol::public::{PublicReply, PublicRequest}; use zkprotocol::search::{ZkListNoteSearchResult, ZkNoteSearch}; + pub fn login_data_for_token( session: Session, config: &Config, @@ -54,7 +55,7 @@ pub fn login_data_for_token( match orgauth::dbfun::read_user_with_token_pageload( &mut conn, &session, - token, + &token, config.orgauth_config.regen_login_tokens, config.orgauth_config.login_token_expiration_ms, ) { diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index 064939ed..35a1e6a2 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -3101,10 +3101,12 @@ pub fn update_filetable(conn: &Connection, files_dir: &Path) -> Result<(), zkerr } } - conn.execute( - format!("insert into files_dir (filename) values {}", fns).as_str(), - params![], - )?; + if !fns.is_empty() { + conn.execute( + format!("insert into files_dir (filename) values {}", fns).as_str(), + params![], + )?; + } Ok(()) } From 6147edc78d91661da0232a96cc5a1a9441ab1caf Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 9 Oct 2026 14:05:42 -0600 Subject: [PATCH 11/40] onclose support --- .gitmodules | 3 +++ elm/elm.json | 4 ++-- elm/src/Main.elm | 11 ++++++++--- server/static/index.html | 37 +++++++++++++++++++++++++++++-------- websocket | 1 + 5 files changed, 43 insertions(+), 13 deletions(-) create mode 160000 websocket diff --git a/.gitmodules b/.gitmodules index 56818778..3fe77e51 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "orgauth"] path = orgauth url = https://github.com/bburdette/orgauth.git +[submodule "websocket"] + path = websocket + url = git@github.com:bburdette/websocket.git diff --git a/elm/elm.json b/elm/elm.json index a7f5aa0f..dd0979fa 100644 --- a/elm/elm.json +++ b/elm/elm.json @@ -3,7 +3,8 @@ "source-directories": [ "src", "../orgauth/elm-common/src/", - "../orgauth/src/" + "../orgauth/src/", + "../websocket/src/" ], "elm-version": "0.19.1", "dependencies": { @@ -15,7 +16,6 @@ "bburdette/schelme": "3.0.0", "bburdette/toop": "1.2.0", "bburdette/typed-collections": "1.0.2", - "bburdette/websocket": "1.0.3", "bburdette/windowkeys": "1.0.1", "ceddlyburge/dnd-list": "1.0.0", "dillonkearns/elm-markdown": "7.0.0", diff --git a/elm/src/Main.elm b/elm/src/Main.elm index eaeea9c0..a64377fa 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -2257,12 +2257,17 @@ actualupdate msg model = ( ReceiveSocketMsg jd, _ ) -> case JD.decodeValue WebSocket.decodeMsg jd of - Ok (WebSocket.Error wsm) -> - ( displayMessageDialog model <| "websocket error: " ++ wsm.error + Ok (WebSocket.OnError wsm) -> + ( displayMessageDialog model <| "websocket \"" ++ wsm.name ++ "\" error: " ++ wsm.error , Cmd.none ) - Ok (WebSocket.Data wsm) -> + Ok (WebSocket.OnClose wsm) -> + ( displayMessageDialog model <| "websocket closed: \"" ++ wsm.name ++ "\" with code: " ++ String.fromInt wsm.code + , Cmd.none + ) + + Ok (WebSocket.OnData wsm) -> if wsm.name == "private" then case JD.decodeString (makeTDDecoder Data.privateClosureReplyDecoder) wsm.data of Ok td -> diff --git a/server/static/index.html b/server/static/index.html index 697d7e72..2d5c17b1 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -109,14 +109,17 @@ // console.log("connecting!"); let socket = new WebSocket(wat.address); socket.onmessage = function (event) { - // console.log( "onmessage: " + JSON.stringify(event.data, null, 4)); - app.ports.receiveSocketMsg.send({ name : wat.name - , msg : "data" - , data : event.data} ); - } + console.log( "onmessage"); + + app.ports.receiveSocketMsg.send( + { name : wat.name + , msg : "data" + , data : event.data} + ); + }; socket.onopen = function (event) { - // console.log( "onopen: " + JSON.stringify(event.data, null, 4)); - // send queued messages. + console.log( "onopen"); + let sq = sockQueue[wat.name]; if (sq) { for (let msg of sq) { @@ -124,7 +127,24 @@ } sockQueue[wat.name] = []; } - } + }; + + socket.addEventListener("close", function (event) { + console.log( "socket close: ", event); + app.ports.receiveSocketMsg.send( + { name : wat.name + , msg : "close" + , code : event.code } + ); + }); + socket.addEventListener("error", function (event) { + console.log( "socket error: " + JSON.stringify(event.data, null, 4)); + app.ports.receiveSocketMsg.send( + { name : wat.name + , msg : "error" + , error : event.data} + ); + }); mySockets[wat.name] = socket; } else if (wat.cmd == "send") @@ -147,6 +167,7 @@ else if (wat.cmd == "close") { // console.log("closing socket: " + wat.name); + mySockets[wat.name].onclose = Null; mySockets[wat.name].close(); delete mySockets[wat.name]; } diff --git a/websocket b/websocket new file mode 160000 index 00000000..8b3e49d4 --- /dev/null +++ b/websocket @@ -0,0 +1 @@ +Subproject commit 8b3e49d45d39e59c6ef01a499862006d373ae2a1 From 2745ebde4ee11790a4c292c949a8f7e3aee577dd Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 9 Oct 2026 14:18:15 -0600 Subject: [PATCH 12/40] removeEventListener --- server/static/index.html | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/server/static/index.html b/server/static/index.html index 2d5c17b1..cfc4a2c3 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -109,8 +109,7 @@ // console.log("connecting!"); let socket = new WebSocket(wat.address); socket.onmessage = function (event) { - console.log( "onmessage"); - + // console.log( "onmessage: " + JSON.stringify(event.data, null, 4)); app.ports.receiveSocketMsg.send( { name : wat.name , msg : "data" @@ -118,8 +117,7 @@ ); }; socket.onopen = function (event) { - console.log( "onopen"); - + console.log( "onopen, websocket: \"" + wat.name + "\""); let sq = sockQueue[wat.name]; if (sq) { for (let msg of sq) { @@ -128,9 +126,8 @@ sockQueue[wat.name] = []; } }; - socket.addEventListener("close", function (event) { - console.log( "socket close: ", event); + // console.log( "socket close: ", event); app.ports.receiveSocketMsg.send( { name : wat.name , msg : "close" @@ -138,7 +135,7 @@ ); }); socket.addEventListener("error", function (event) { - console.log( "socket error: " + JSON.stringify(event.data, null, 4)); + // console.log( "socket error: " + JSON.stringify(event.data, null, 4)); app.ports.receiveSocketMsg.send( { name : wat.name , msg : "error" @@ -166,8 +163,8 @@ } else if (wat.cmd == "close") { - // console.log("closing socket: " + wat.name); - mySockets[wat.name].onclose = Null; + // don't trigger close event when initiated from elm. + mySockets[wat.name].removeEventListener("close"); mySockets[wat.name].close(); delete mySockets[wat.name]; } From fb6f9f9b74e3cb7569d980acfaa435e5b55c4417 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 9 Oct 2026 14:28:16 -0600 Subject: [PATCH 13/40] revise sockets js --- server/static/index.html | 342 +++++++++++++++++++-------------------- websocket | 2 +- 2 files changed, 172 insertions(+), 172 deletions(-) diff --git a/server/static/index.html b/server/static/index.html index cfc4a2c3..5faf49b4 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -1,174 +1,174 @@ - - - - - - - - - - -
- - + + + + + + + + + + +
+ + diff --git a/websocket b/websocket index 8b3e49d4..94daabfb 160000 --- a/websocket +++ b/websocket @@ -1 +1 @@ -Subproject commit 8b3e49d45d39e59c6ef01a499862006d373ae2a1 +Subproject commit 94daabfbf51bbbcb0c38dc4dcad280d30a3fe11e From 5591fa04037437ed76b77d8408fd46555aa3a26d Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 9 Oct 2026 15:34:55 -0600 Subject: [PATCH 14/40] onopen --- elm/src/Main.elm | 5 +++++ server/static/index.html | 5 ++++- websocket | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index a64377fa..ad035791 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -2267,6 +2267,11 @@ actualupdate msg model = , Cmd.none ) + Ok (WebSocket.OnOpen wsm) -> + ( displayMessageDialog model <| "websocket opened: \"" ++ wsm.name ++ "\"" + , Cmd.none + ) + Ok (WebSocket.OnData wsm) -> if wsm.name == "private" then case JD.decodeString (makeTDDecoder Data.privateClosureReplyDecoder) wsm.data of diff --git a/server/static/index.html b/server/static/index.html index 5faf49b4..61246864 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -119,6 +119,9 @@ socket.onopen = function (event) { // console.log( "onopen, websocket: \"" + sc.name + "\""); let sq = sockQueue[sc.name]; + app.ports.receiveSocketMsg.send( + { name : sc.name + , msg : "open" }); if (sq) { for (let msg of sq) { mySockets[sc.name].send(msg); @@ -154,7 +157,7 @@ // console.log("queuing message", sc.name, sc.content); let sq = sockQueue[sc.name]; if (sq) { - sq.push(sc.content); + sq.push(sc.content); } else { sq = [sc.content]; } diff --git a/websocket b/websocket index 94daabfb..9bba75cc 160000 --- a/websocket +++ b/websocket @@ -1 +1 @@ -Subproject commit 94daabfbf51bbbcb0c38dc4dcad280d30a3fe11e +Subproject commit 9bba75ccb660f77d8c0665f70b1eb724b4ed0169 From 9b21540d67877e544c1da62ff5aa9309678fb5b7 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 9 Oct 2026 16:03:39 -0600 Subject: [PATCH 15/40] show socket closure reason --- elm/src/Main.elm | 12 +++++++++++- websocket | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index ad035791..fd3bd29f 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -2263,7 +2263,17 @@ actualupdate msg model = ) Ok (WebSocket.OnClose wsm) -> - ( displayMessageDialog model <| "websocket closed: \"" ++ wsm.name ++ "\" with code: " ++ String.fromInt wsm.code + ( displayMessageDialog model <| + "websocket \"" + ++ wsm.name + ++ "\" closed: " + ++ " with code: " + ++ String.fromInt wsm.code + ++ " string: " + ++ (WebSocket.socketClosure wsm.code + |> Maybe.map WebSocket.showSocketClosure + |> Maybe.withDefault "unknown code" + ) , Cmd.none ) diff --git a/websocket b/websocket index 9bba75cc..256e0f89 160000 --- a/websocket +++ b/websocket @@ -1 +1 @@ -Subproject commit 9bba75ccb660f77d8c0665f70b1eb724b4ed0169 +Subproject commit 256e0f890bc61610624d701adaeb9e8e025f32eb From 4a7803762e56c00924403fc36ef89bbe0e3d7953 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Sat, 10 Oct 2026 09:05:27 -0600 Subject: [PATCH 16/40] socket state display; reconnect --- elm/src/Main.elm | 270 +++++++++++++++++++++++---------------- server/static/index.html | 2 +- 2 files changed, 164 insertions(+), 108 deletions(-) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index fd3bd29f..4e864351 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -217,6 +217,29 @@ type LocalValAction } +type SocketState + = SsOpenRequested + | SsIsOpen + | SsIsClosed Int + | SsError + + +opensocket : String -> Cmd Msg +opensocket location = + sendSocketCommand + (WebSocket.encodeCmd <| + WebSocket.Connect + { name = "private" + , address = + location + -- http -> ws, https -> wss + |> String.replace "http" "ws" + |> (\s -> s ++ "/privatews") + , protocol = "" + } + ) + + type alias Model = { state : State , size : Util.Size @@ -240,6 +263,7 @@ type alias Model = , spmodel : SP.Model , localValAction : Dict String LocalValAction , zknSearchResult : Data.ZkListNoteSearchResult + , socketState : Maybe SocketState } @@ -1610,94 +1634,128 @@ view model = _ -> routeTitle model.savedRoute.route , body = - [ case model.state of - DisplayMessage dm _ -> - Html.map DisplayMessageMsg <| - GD.layout - (Just { width = min 600 model.size.width, height = min 500 model.size.height }) - dm - - MessageNLink dm _ -> - Html.map MessageNLinkMsg <| - GD.layout - (Just { width = min 600 model.size.width, height = min 500 model.size.height }) - dm - - SelectDialog sdm _ -> - if model.mobile then - E.layout [] <| - E.map SelectDialogMsg <| - GD.dialogView Nothing sdm + (case model.socketState of + Nothing -> + [] - else - Html.map SelectDialogMsg <| - GD.layout - (Just { width = min 600 model.size.width, height = min 500 model.size.height }) - sdm - - ChangePasswordDialog cdm _ -> - Html.map ChangePasswordDialogMsg <| - GD.layout - (Just { width = min 600 model.size.width, height = min 200 model.size.height }) - cdm - - ChangeEmailDialog cdm _ -> - Html.map ChangeEmailDialogMsg <| - GD.layout - (Just { width = min 600 model.size.width, height = min 200 model.size.height }) - cdm - - ChangeRemoteUrlDialog cdm _ -> - Html.map ChangeRemoteUrlDialogMsg <| - GD.layout - (Just { width = min 600 model.size.width, height = min 200 model.size.height }) - cdm - - JobsDialog dm _ -> - if model.mobile then - E.layout [] <| - E.map JobsDialogMsg <| - GD.dialogView Nothing { dm | model = model.jobs } + Just ss -> + case ss of + SsOpenRequested -> + [ Html.text "connecting websocket" ] - else - Html.map JobsDialogMsg <| - GD.layout - (Just { width = min 600 model.size.width, height = min 500 model.size.height }) - { dm | model = model.jobs } + SsIsOpen -> + [] - MdInlineXform gdm _ -> - if model.mobile then - E.layout [] <| - E.map MdInlineXformMsg <| - GD.dialogView Nothing gdm + SsIsClosed code -> + [ Html.div [] + [ Html.div [] + [ Html.text <| + "websocket closed with code: " + ++ String.fromInt code + ++ " string: " + ++ (WebSocket.socketClosure code + |> Maybe.map WebSocket.showSocketClosure + |> Maybe.withDefault "unknown code" + ) + ] + , Html.div [] + [ Html.text + "reconnecting..." + ] + ] + ] - else - Html.map MdInlineXformMsg <| - GD.layout - (Just { width = min 600 model.size.width, height = min 500 model.size.height }) - gdm + SsError -> + [ Html.text <| "websocket error" ] + ) + ++ [ case model.state of + DisplayMessage dm _ -> + Html.map DisplayMessageMsg <| + GD.layout + (Just { width = min 600 model.size.width, height = min 500 model.size.height }) + dm + + MessageNLink dm _ -> + Html.map MessageNLinkMsg <| + GD.layout + (Just { width = min 600 model.size.width, height = min 500 model.size.height }) + dm + + SelectDialog sdm _ -> + if model.mobile then + E.layout [] <| + E.map SelectDialogMsg <| + GD.dialogView Nothing sdm - RequestsDialog dm _ -> - if model.mobile then - E.layout [] <| - E.map RequestsDialogMsg <| - GD.dialogView Nothing { dm | model = model.trackedRequests } + else + Html.map SelectDialogMsg <| + GD.layout + (Just { width = min 600 model.size.width, height = min 500 model.size.height }) + sdm + + ChangePasswordDialog cdm _ -> + Html.map ChangePasswordDialogMsg <| + GD.layout + (Just { width = min 600 model.size.width, height = min 200 model.size.height }) + cdm + + ChangeEmailDialog cdm _ -> + Html.map ChangeEmailDialogMsg <| + GD.layout + (Just { width = min 600 model.size.width, height = min 200 model.size.height }) + cdm + + ChangeRemoteUrlDialog cdm _ -> + Html.map ChangeRemoteUrlDialogMsg <| + GD.layout + (Just { width = min 600 model.size.width, height = min 200 model.size.height }) + cdm + + JobsDialog dm _ -> + if model.mobile then + E.layout [] <| + E.map JobsDialogMsg <| + GD.dialogView Nothing { dm | model = model.jobs } - else - Html.map RequestsDialogMsg <| - GD.layout - (Just { width = min 600 model.size.width, height = min 500 model.size.height }) - -- use the live-updated model - { dm | model = model.trackedRequests } + else + Html.map JobsDialogMsg <| + GD.layout + (Just { width = min 600 model.size.width, height = min 500 model.size.height }) + { dm | model = model.jobs } - _ -> - E.layout - ([ EF.size model.stylePalette.fontSize, E.width E.fill ] - ++ dndif - ) - <| - viewState model.size model.state model - ] + MdInlineXform gdm _ -> + if model.mobile then + E.layout [] <| + E.map MdInlineXformMsg <| + GD.dialogView Nothing gdm + + else + Html.map MdInlineXformMsg <| + GD.layout + (Just { width = min 600 model.size.width, height = min 500 model.size.height }) + gdm + + RequestsDialog dm _ -> + if model.mobile then + E.layout [] <| + E.map RequestsDialogMsg <| + GD.dialogView Nothing { dm | model = model.trackedRequests } + + else + Html.map RequestsDialogMsg <| + GD.layout + (Just { width = min 600 model.size.width, height = min 500 model.size.height }) + -- use the live-updated model + { dm | model = model.trackedRequests } + + _ -> + E.layout + ([ EF.size model.stylePalette.fontSize, E.width E.fill ] + ++ dndif + ) + <| + viewState model.size model.state model + ] } @@ -2258,27 +2316,30 @@ actualupdate msg model = ( ReceiveSocketMsg jd, _ ) -> case JD.decodeValue WebSocket.decodeMsg jd of Ok (WebSocket.OnError wsm) -> - ( displayMessageDialog model <| "websocket \"" ++ wsm.name ++ "\" error: " ++ wsm.error + ( { model | socketState = Just <| SsError } + -- ( displayMessageDialog model <| "websocket \"" ++ wsm.name ++ "\" error: " ++ wsm.error , Cmd.none ) Ok (WebSocket.OnClose wsm) -> - ( displayMessageDialog model <| - "websocket \"" - ++ wsm.name - ++ "\" closed: " - ++ " with code: " - ++ String.fromInt wsm.code - ++ " string: " - ++ (WebSocket.socketClosure wsm.code - |> Maybe.map WebSocket.showSocketClosure - |> Maybe.withDefault "unknown code" - ) - , Cmd.none + ( { model | socketState = Just <| SsIsClosed wsm.code } + -- ( displayMessageDialog model <| + -- "websocket \"" + -- ++ wsm.name + -- ++ "\" closed: " + -- ++ " with code: " + -- ++ String.fromInt wsm.code + -- ++ " string: " + -- ++ (WebSocket.socketClosure wsm.code + -- |> Maybe.map WebSocket.showSocketClosure + -- |> Maybe.withDefault "unknown code" + -- ) + , opensocket model.fui.location ) Ok (WebSocket.OnOpen wsm) -> - ( displayMessageDialog model <| "websocket opened: \"" ++ wsm.name ++ "\"" + ( { model | socketState = Just SsIsOpen } + -- ( displayMessageDialog model <| "websocket opened: \"" ++ wsm.name ++ "\"" , Cmd.none ) @@ -5329,6 +5390,12 @@ init flags url key zone fontsize = , offset = 0 , what = "" } + , socketState = + if flags.websockets then + Just SsOpenRequested + + else + Nothing } geterrornote = @@ -5355,18 +5422,7 @@ init flags url key zone fontsize = opensock = if flags.websockets then - sendSocketCommand - (WebSocket.encodeCmd <| - WebSocket.Connect - { name = "private" - , address = - flags.location - -- http -> ws, https -> wss - |> String.replace "http" "ws" - |> (\s -> s ++ "/privatews") - , protocol = "" - } - ) + opensocket flags.location else Cmd.none diff --git a/server/static/index.html b/server/static/index.html index 61246864..dae00378 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -138,7 +138,7 @@ ); }); socket.addEventListener("error", function (event) { - // console.log( "socket error: " + JSON.stringify(event.data, null, 4)); + console.log( "socket error: " + event); app.ports.receiveSocketMsg.send( { name : sc.name , msg : "error" From 37b82a7f9361b9ccd0f3b2dd70d08d5b4efff635 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Sun, 11 Oct 2026 08:17:51 -0600 Subject: [PATCH 17/40] cleanup --- elm/src/Main.elm | 13 ------------- server/static/index.html | 5 ++--- websocket | 2 +- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index 4e864351..2d880262 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -2317,29 +2317,16 @@ actualupdate msg model = case JD.decodeValue WebSocket.decodeMsg jd of Ok (WebSocket.OnError wsm) -> ( { model | socketState = Just <| SsError } - -- ( displayMessageDialog model <| "websocket \"" ++ wsm.name ++ "\" error: " ++ wsm.error , Cmd.none ) Ok (WebSocket.OnClose wsm) -> ( { model | socketState = Just <| SsIsClosed wsm.code } - -- ( displayMessageDialog model <| - -- "websocket \"" - -- ++ wsm.name - -- ++ "\" closed: " - -- ++ " with code: " - -- ++ String.fromInt wsm.code - -- ++ " string: " - -- ++ (WebSocket.socketClosure wsm.code - -- |> Maybe.map WebSocket.showSocketClosure - -- |> Maybe.withDefault "unknown code" - -- ) , opensocket model.fui.location ) Ok (WebSocket.OnOpen wsm) -> ( { model | socketState = Just SsIsOpen } - -- ( displayMessageDialog model <| "websocket opened: \"" ++ wsm.name ++ "\"" , Cmd.none ) diff --git a/server/static/index.html b/server/static/index.html index dae00378..a73e6107 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -138,11 +138,10 @@ ); }); socket.addEventListener("error", function (event) { - console.log( "socket error: " + event); + // console.log( "socket error: " + event); app.ports.receiveSocketMsg.send( { name : sc.name - , msg : "error" - , error : event.data} + , msg : "error" } ); }); mySockets[sc.name] = socket; diff --git a/websocket b/websocket index 256e0f89..b5bc30d6 160000 --- a/websocket +++ b/websocket @@ -1 +1 @@ -Subproject commit 256e0f890bc61610624d701adaeb9e8e025f32eb +Subproject commit b5bc30d68d66312c6be760e214d973de37514fed From 071088233b0bfd1c3229ac5da7d04f6db9064fdd Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 13 Oct 2026 14:17:58 -0600 Subject: [PATCH 18/40] cancel websocket if not logged in; connect websocket after login --- elm/src/Main.elm | 28 ++++++++++++++++++++-------- server-lib/src/lib.rs | 38 ++++++++++++++++++++++++++------------ server/static/index.html | 4 +--- websocket | 2 +- 4 files changed, 48 insertions(+), 24 deletions(-) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index 2d880262..b5dc7d17 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -220,7 +220,7 @@ type LocalValAction type SocketState = SsOpenRequested | SsIsOpen - | SsIsClosed Int + | SsIsClosed WebSocket.SocketClosure | SsError @@ -1646,17 +1646,17 @@ view model = SsIsOpen -> [] - SsIsClosed code -> + SsIsClosed WebSocket.PolicyViolation -> + [] + + SsIsClosed sc -> [ Html.div [] [ Html.div [] [ Html.text <| "websocket closed with code: " - ++ String.fromInt code + ++ (String.fromInt <| WebSocket.socketClosureCode sc) ++ " string: " - ++ (WebSocket.socketClosure code - |> Maybe.map WebSocket.showSocketClosure - |> Maybe.withDefault "unknown code" - ) + ++ WebSocket.showSocketClosure sc ] , Html.div [] [ Html.text @@ -2322,7 +2322,18 @@ actualupdate msg model = Ok (WebSocket.OnClose wsm) -> ( { model | socketState = Just <| SsIsClosed wsm.code } - , opensocket model.fui.location + , case wsm.code of + WebSocket.PolicyViolation -> + -- don't keep trying, bad login. + sendSocketCommand + (WebSocket.encodeCmd <| + WebSocket.Close + { name = "private" + } + ) + + _ -> + opensocket model.fui.location ) Ok (WebSocket.OnOpen wsm) -> @@ -5316,6 +5327,7 @@ initToRoute model route = model.navkey (routeUrl (stateRoute initialstate).route) , Just c + , Just <| opensocket model.fui.location ] ) diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index f62e3169..440cc995 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -21,16 +21,18 @@ use actix_web::{ dev::Server, rt, web, App, HttpRequest, HttpResponse, HttpServer, ResponseError, Result, }; -use actix_ws::AggregatedMessage; +use actix_ws::{AggregatedMessage, CloseReason}; use chrono; use clap::Arg; use config::Config; +use either::IntoEither; +use futures::SinkExt; use futures_util::StreamExt as _; use futures_util::TryStreamExt as _; use girlboss::Girlboss; use interfaces::connect_and_make_lapin_info; use lapin; -use log::{error, info}; +use log::{error, info, warn}; pub use orgauth; use orgauth::{data::UserId, util}; pub use orgauth::{ @@ -43,14 +45,18 @@ use serde::{Deserialize, Serialize}; use serde_json; use simple_error::simple_error; use sqldata::{get_single_value, local_server_id, LapinInfo}; -use std::io::{stdin, Write}; use std::path::Path; use std::path::PathBuf; use std::str::FromStr; use std::{env, sync::RwLock}; use std::{error::Error, sync::Arc}; use std::{fs::File, time::SystemTime}; +use std::{ + io::{stdin, Write}, + time::Duration, +}; use timer; +use tokio::time::sleep; use tokio_util::io::StreamReader; use tracing_actix_web::TracingLogger; use uuid::Uuid; @@ -449,15 +455,7 @@ async fn private_ws( let token = get_cookie_id(&req); - let token_uuid = match session.get::("token")? { - Some(tu) => tu, - None => { - return Ok( - HttpResponse::Ok().json(PrivateReply::PvyServerError(PrivateError::PveNotLoggedIn)), - ) - } - }; - + info!("getting handle"); let (res, mut wssession, stream) = actix_ws::handle(&req, stream)?; let mut stream = stream @@ -465,6 +463,22 @@ async fn private_ws( // aggregate continuation frames up to 1MiB .max_continuation_size(2_usize.pow(20)); + let token_uuid = match session.get::("token")? { + Some(tu) => tu, + None => { + warn!("closing socket"); + wssession + .close(Some(actix_ws::CloseReason { + code: actix_ws::CloseCode::Policy, + description: Some("not logged in".to_string()), + })) + .await + .map_err(|e| zkerr::Error::String(e.to_string()))?; + + return Ok(res); // have to return this special 'res' for the above to work. + } + }; + rt::spawn(async move { while let Some(msg) = stream.next().await { match async { diff --git a/server/static/index.html b/server/static/index.html index a73e6107..cb5fd680 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -165,9 +165,7 @@ } else if (sc.cmd == "close") { - // don't trigger close event when initiated from elm. - mySockets[sc.name].removeEventListener("close"); - mySockets[sc.name].close(); + // console.log("removing socket: ", sc.name) delete mySockets[sc.name]; } } diff --git a/websocket b/websocket index b5bc30d6..3bab6300 160000 --- a/websocket +++ b/websocket @@ -1 +1 @@ -Subproject commit b5bc30d68d66312c6be760e214d973de37514fed +Subproject commit 3bab63000d0630e482af3c494b693de9265cf1b1 From 8adc4fedebd1a0b51bd8c80fc63a36400e5a6ef3 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 13 Oct 2026 15:52:08 -0600 Subject: [PATCH 19/40] don't double open socket --- elm/src/Main.elm | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/elm/src/Main.elm b/elm/src/Main.elm index b5dc7d17..674ca4be 100644 --- a/elm/src/Main.elm +++ b/elm/src/Main.elm @@ -5327,7 +5327,11 @@ initToRoute model route = model.navkey (routeUrl (stateRoute initialstate).route) , Just c - , Just <| opensocket model.fui.location + , if model.fui.websockets then + Just <| opensocket model.fui.location + + else + Nothing ] ) @@ -5419,13 +5423,6 @@ init flags url key zone fontsize = , { key = "l", ctrl = True, alt = True, shift = False, preventDefault = True } ] - opensock = - if flags.websockets then - opensocket flags.location - - else - Cmd.none - ( m, c ) = initToRoute imodel imodel.initialRoute in @@ -5434,7 +5431,6 @@ init flags url key zone fontsize = [ c , geterrornote , setkeys - , opensock ] ) From 7bc5414e347da7b8ce7047859b782298186e792a Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 19 Oct 2026 13:31:17 -0600 Subject: [PATCH 20/40] improved sync with indexes; add tracing and etc --- orgauth | 2 +- server-lib/Cargo.toml | 2 +- server-lib/src/interfaces.rs | 39 ++++-- server-lib/src/lib.rs | 53 +++++--- server-lib/src/migrations.rs | 14 ++ server-lib/src/search.rs | 3 + server-lib/src/sqldata.rs | 36 ++++- server-lib/src/sync.rs | 246 ++++++++++++++++++++++++++++++----- server-lib/src/synctest.rs | 10 +- server/default.toml | 2 +- 10 files changed, 332 insertions(+), 75 deletions(-) diff --git a/orgauth b/orgauth index e8e40147..f92e19b3 160000 --- a/orgauth +++ b/orgauth @@ -1 +1 @@ -Subproject commit e8e4014717e59ffd92a5ff7ff74e50ffeb671874 +Subproject commit f92e19b3db304f4b5e72e3b4cf71c022c9f70497 diff --git a/server-lib/Cargo.toml b/server-lib/Cargo.toml index 153d6ac9..26a0a8f5 100644 --- a/server-lib/Cargo.toml +++ b/server-lib/Cargo.toml @@ -9,7 +9,7 @@ zkprotocol = { path = "../zkprotocol" } orgauth = { path = "../orgauth/rustlib/" } env_logger = "0.11.8" # env_logger = { version = "0.11.5", features = ["anstream"] } -rusqlite = { version = "0.37.0", features = ["bundled"] } +rusqlite = { version = "0.37.0", features = ["bundled", "trace"] } log = "0.4.28" actix-web = "4.11.0" actix-files = "0.6.7" diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index 3b43f112..02985baa 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -45,7 +45,8 @@ pub fn login_data_for_token( session: Session, config: &Config, ) -> Result, Box> { - let mut conn = sqldata::connection_open(config.orgauth_config.db.as_path())?; + let mut conn = + sqldata::connection_open("login_data_for_token", config.orgauth_config.db.as_path())?; conn.busy_timeout(Duration::from_millis(500))?; let mut cb = sqldata::zknotes_callbacks(); @@ -178,6 +179,7 @@ pub async fn zk_interface_loggedin_streaming( PrivateStreamingRequests::SearchZkNotes => { let search: ZkNoteSearch = serde_json::from_value(msgdata.clone())?; let conn = Arc::new(sqldata::connection_open( + "PrivateStreamingRequests::SearchZkNotes", config.orgauth_config.db.as_path(), )?); let znsstream = @@ -188,6 +190,7 @@ pub async fn zk_interface_loggedin_streaming( PrivateStreamingRequests::GetArchiveZkLinks => { let rq: GetArchiveZkLinks = serde_json::from_value(msgdata.clone())?; let conn = Arc::new(sqldata::connection_open( + "psr:GetArchiveZkLinks", config.orgauth_config.db.as_path(), )?); let bstream = sqldata::read_archivezklinks_stream(conn, uid, rq.createddate_after, None) @@ -197,6 +200,7 @@ pub async fn zk_interface_loggedin_streaming( PrivateStreamingRequests::GetZkLinksSince => { let rq: GetZkLinksSince = serde_json::from_value(msgdata.clone())?; let conn = Arc::new(sqldata::connection_open( + "psr:GetZkLinksSince", config.orgauth_config.db.as_path(), )?); let bstream = sqldata::read_zklinks_since_stream(conn, uid, rq.createddate_after, None) @@ -205,6 +209,7 @@ pub async fn zk_interface_loggedin_streaming( } PrivateStreamingRequests::Sync => { let conn = Arc::new(sqldata::connection_open( + "psr:Sync", config.orgauth_config.db.as_path(), )?); let rq: SyncSince = serde_json::from_value(msgdata.clone())?; @@ -236,7 +241,7 @@ pub async fn zk_interface_loggedin_streaming( pub async fn zk_interface_loggedin( state: &State, - conn: &Connection, + conn: Connection, token: &Option, uid: UserId, msg: &PrivateRequest, @@ -356,6 +361,8 @@ pub async fn zk_interface_loggedin( let lapin_channelx = li.map(|li| li.channel).clone(); let token = token.clone(); + conn.close().map_err(|(_conn, err)| err)?; + std::thread::spawn(move || { let rt = actix_rt::System::new(); @@ -422,11 +429,12 @@ pub async fn zk_interface_loggedin( .unwrap() }); - Ok(PrivateReply::PvyJobStatus(JobStatus { - jobno: jid.jobno, - state: JobState::Started, - message: "".to_string(), - })) + // Ok(PrivateReply::PvyJobStatus(JobStatus { + // jobno: jid.jobno, + // state: JobState::Started, + // message: "".to_string(), + // })) + Ok(PrivateReply::PvyNoop) } PrivateRequest::PvqSyncFiles(znsrq) => { let dbpath: PathBuf = state.config.orgauth_config.db.to_path_buf(); @@ -463,7 +471,7 @@ pub async fn zk_interface_loggedin( write!(gbm, "starting file sync"); let r = async { - let conn = sqldata::connection_open(&dbpath.as_path())?; + let conn = sqldata::connection_open("startit", &dbpath.as_path())?; let _dv = sync::sync_files_down( &conn, &lapin_info, @@ -560,7 +568,10 @@ pub fn public_interface( ) -> Result { match msg { PublicRequest::PbrGetZkNoteAndLinks(gzne) => { - let conn = sqldata::connection_open(config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open( + "PublicRequest::PbrGetZkNoteAndLinks", + config.orgauth_config.db.as_path(), + )?; let (_, note) = sqldata::read_zknote(&conn, &config.file_path, None, &gzne.zknote)?; info!( "public#getzknote: {:?} - {} - {:?}", @@ -580,7 +591,10 @@ pub fn public_interface( "user#getzneifchanged: {:?} - {}", gzic.zknote, gzic.changeddate ); - let conn = sqldata::connection_open(config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open( + "PublicRequest::PbrGetZknIfChanged", + config.orgauth_config.db.as_path(), + )?; let ozkne = sqldata::read_zneifchanged(&conn, &config.file_path, None, &gzic)?; match ozkne { @@ -589,7 +603,10 @@ pub fn public_interface( } } PublicRequest::PbrGetZkNotePubId(pubid) => { - let conn = sqldata::connection_open(config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open( + "PublicRequest::PbrGetZkNotePubId", + config.orgauth_config.db.as_path(), + )?; let note = sqldata::read_zknotepubid(&conn, &config.file_path, None, pubid.as_str())?; info!( "public#getzknotepubid: {} - {} - {:?}", diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index 440cc995..9a3ee46f 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -40,16 +40,16 @@ pub use orgauth::{ endpoints::ActixTokener, }; pub use rusqlite; -use rusqlite::Connection; +use rusqlite::{trace::config_log, Connection}; use serde::{Deserialize, Serialize}; use serde_json; use simple_error::simple_error; use sqldata::{get_single_value, local_server_id, LapinInfo}; -use std::path::Path; use std::path::PathBuf; use std::str::FromStr; use std::{env, sync::RwLock}; use std::{error::Error, sync::Arc}; +use std::{ffi::c_int, path::Path}; use std::{fs::File, time::SystemTime}; use std::{ io::{stdin, Write}, @@ -189,7 +189,7 @@ async fn user( // req.connection_info() // ); match async { - let conn = sqldata::connection_open(data.config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open("user", data.config.orgauth_config.db.as_path())?; interfaces::user_interface( &conn, @@ -265,7 +265,7 @@ pub fn get_tauri_uid(conn: &Connection) -> Result, zkerr::Error> } async fn file(session: Session, state: web::Data, req: HttpRequest) -> HttpResponse { - let conn = match sqldata::connection_open(state.config.orgauth_config.db.as_path()) { + let conn = match sqldata::connection_open("file", state.config.orgauth_config.db.as_path()) { Ok(c) => c, Err(e) => return HttpResponse::InternalServerError().body(format!("{:?}", e)), }; @@ -347,7 +347,7 @@ async fn make_file_notes( lapin_info: &Option, payload: &mut Multipart, ) -> Result> { - let conn = sqldata::connection_open(state.config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open("make_file_notes", state.config.orgauth_config.db.as_path())?; let uid = if state.config.tauri_mode { get_tauri_uid(&conn)?.ok_or(zkerr::Error::NotLoggedIn)? } else { @@ -444,6 +444,7 @@ pub struct PrivateTimedData { utcmillis: u128, data: PrivateClosureReply, } + async fn private_ws( session: Session, data: web::Data, @@ -455,7 +456,6 @@ async fn private_ws( let token = get_cookie_id(&req); - info!("getting handle"); let (res, mut wssession, stream) = actix_ws::handle(&req, stream)?; let mut stream = stream @@ -482,9 +482,11 @@ async fn private_ws( rt::spawn(async move { while let Some(msg) = stream.next().await { match async { + println!("after stream.next()"); match msg { Ok(AggregatedMessage::Text(t)) => { let pcr: PrivateClosureRequest = serde_json::from_str(&t)?; + println!("after let pcr: PrivateClosureRequest ="); match ( zk_interface_check(&token_uuid, &mut state, &token, pcr.request).await, SystemTime::now() @@ -615,7 +617,11 @@ async fn zk_interface_check( token: &Option, msg: PrivateRequest, ) -> Result { - let conn = sqldata::connection_open(state.config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open( + "zk_interface_check", + state.config.orgauth_config.db.as_path(), + )?; + println!("before read_user_by_token_api"); match orgauth::dbfun::read_user_by_token_api( &conn, token_uuid, @@ -630,8 +636,9 @@ async fn zk_interface_check( ))) } Ok(userdata) => { + println!("after read_user_by_token_api"); // finally! processing messages as logged in user. - interfaces::zk_interface_loggedin(state, &conn, token, userdata.id, &msg).await + interfaces::zk_interface_loggedin(state, conn, token, userdata.id, &msg).await } } } @@ -645,7 +652,10 @@ async fn zk_interface_check_streaming( None => Ok(HttpResponse::Ok().json(PrivateReply::PvyServerError(PrivateError::PveNotLoggedIn))), Some(token) => { - let conn = sqldata::connection_open(config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open( + "zk_interface_check_streaming", + config.orgauth_config.db.as_path(), + )?; match orgauth::dbfun::read_user_by_token_api( &conn, &token, @@ -653,7 +663,7 @@ async fn zk_interface_check_streaming( config.orgauth_config.regen_login_tokens, ) { Err(e) => { - info!("read_user_by_token_api error2: {:?}, {:?}", token, e); + info!("read_user_by_token_api error3: {:?}, {:?}", token, e); Ok( HttpResponse::Ok().json(PrivateReply::PvyServerError(PrivateError::PveLoginError( format!("{:?}", e), @@ -687,7 +697,7 @@ async fn private_upstreaming( } fn convert_bodyerr(err: actix_web::error::PayloadError) -> std::io::Error { - error!("convert_err {:?}", err); + // error!("convert_err {:?}", err); todo!() } @@ -701,7 +711,10 @@ async fn zk_interface_check_upstreaming( None => Ok(HttpResponse::Ok().json(PrivateReply::PvyServerError(PrivateError::PveNotLoggedIn))), Some(token) => { - let conn = sqldata::connection_open(config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open( + "zk_interface_check_upstreaming", + config.orgauth_config.db.as_path(), + )?; let server = sqldata::local_server_id(&conn)?; @@ -712,7 +725,7 @@ async fn zk_interface_check_upstreaming( config.orgauth_config.regen_login_tokens, ) { Err(e) => { - info!("read_user_by_token_api error2: {:?}, {:?}", token, e); + info!("read_user_by_token_api error4: {:?}, {:?}", token, e); Ok( HttpResponse::Ok().json(PrivateReply::PvyServerError(PrivateError::PveLoginError( @@ -804,11 +817,19 @@ async fn new_email(data: web::Data, req: HttpRequest) -> HttpResponse { orgauth::endpoints::new_email(&data.config.orgauth_config, req) } +fn rulog(cint: c_int, meh: &str) { + info!("rulog: {}; {}", cint, meh); +} + #[actix_web::main] pub async fn err_main( oconfig: Option, logfile: Option, ) -> Result<(), Box> { + unsafe { + config_log(Some(rulog))?; + } + match logfile { Some(lf) => { let target = Box::new(File::create(lf).expect("Can't create file")); @@ -924,7 +945,7 @@ pub async fn err_main( // promoting a user to admin? if let Some(uid) = matches.value_of("promote_to_admin") { - let conn = sqldata::connection_open(config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open("promote to admin", config.orgauth_config.db.as_path())?; let mut user = orgauth::dbfun::read_user_by_name(&conn, uid)?; user.admin = true; orgauth::dbfun::update_user(&conn, &user)?; @@ -941,7 +962,7 @@ pub async fn err_main( stdin().read_line(&mut pwd)?; let mut cb = sqldata::zknotes_callbacks(); - let conn = sqldata::connection_open(config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open("creating admin", config.orgauth_config.db.as_path())?; // make new registration i let rd = orgauth::data::RegistrationData { uid: username.to_string(), @@ -1032,7 +1053,7 @@ pub async fn init_server(mut config: Config) -> Result> { }; { - let conn = sqldata::connection_open(config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open("init_server", config.orgauth_config.db.as_path())?; let tr = conn.unchecked_transaction()?; sqldata::update_filetable(&conn, &config.file_path)?; diff --git a/server-lib/src/migrations.rs b/server-lib/src/migrations.rs index 7180d7b6..29fcd33d 100644 --- a/server-lib/src/migrations.rs +++ b/server-lib/src/migrations.rs @@ -3199,3 +3199,17 @@ pub fn udpate43(dbfile: &Path) -> Result<(), zkerr::Error> { tr.commit()?; Ok(()) } + +pub fn udpate44(dbfile: &Path) -> Result<(), zkerr::Error> { + // db connection without foreign key checking. + let conn = Connection::open(dbfile)?; + let tr = conn.unchecked_transaction()?; + + conn.execute( + "CREATE INDEX \"idx_zkarchzknote\" ON \"zkarch\" (\"zknote\")", + params![], + )?; + + tr.commit()?; + Ok(()) +} diff --git a/server-lib/src/search.rs b/server-lib/src/search.rs index 6eec4d23..39966e6e 100644 --- a/server-lib/src/search.rs +++ b/server-lib/src/search.rs @@ -5,6 +5,7 @@ use crate::sqldata::server_id; use crate::sqldata::{delete_zknote, get_sysids, note_id}; use async_stream::try_stream; use futures::Stream; +use log::{debug, error, info}; use orgauth::data::UserId; use orgauth::dbfun::user_id; use rusqlite::Connection; @@ -238,9 +239,11 @@ pub fn search_zknotes_stream( try_stream! { let (sql, args) = build_sql(&conn, user, &search, exclude_notes)?; + debug!("search_zknotes_stream sql\n {}", sql); let mut stmt = conn.prepare(sql.as_str())?; let mut rows = stmt.query(rusqlite::params_from_iter(args.iter()))?; + yield SyncMessage::from(ZkSearchResultHeader { what: search.what, resulttype: search.resulttype, diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index 35a1e6a2..11fa91eb 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -6,7 +6,7 @@ use barrel::backend::Sqlite; use lapin::options::QueueDeclareOptions; use lapin::types::FieldTable; use lapin::Channel; -use log::{error, info}; +use log::{debug, error, info}; use orgauth::data::{RegistrationData, UserId}; use orgauth::dbfun::user_id; use orgauth::endpoints::Callbacks; @@ -169,8 +169,21 @@ pub fn set_homenote( Ok(()) } -pub fn connection_open(dbfile: &Path) -> Result { - let conn = Connection::open(dbfile)?; +fn sqlog(meh: &str) { + info!("sqlog: {}", meh); +} +// unsafe extern "C" fn trace_callback( +// evt: c_uint, +// ctx: *mut c_void, +// p: *mut c_void, +// x: *mut c_void, +// ) -> c_int { + +pub fn connection_open(connid: &str, dbfile: &Path) -> Result { + debug!("connection id: {}", connid); + let mut conn = Connection::open(dbfile)?; + + debug!("after connection_open"); // conn.busy_timeout(Duration::from_millis(500))?; conn.busy_handler(Some(|count| { @@ -180,12 +193,18 @@ pub fn connection_open(dbfile: &Path) -> Result { true }))?; + conn.trace(Some(sqlog)); + + debug!("connection_open 2"); conn.execute("PRAGMA foreign_keys = true;", params![])?; + debug!("connection_open 3"); // give this a go with sync! // doesn't get the busy handle message at least. - conn.query_row("PRAGMA journal_mode = WAL;", params![], |_row| Ok(()))?; + // conn.query_row("PRAGMA journal_mode = WAL;", params![], |_row| Ok(()))?; + // conn.execute("PRAGMA journal_mode = WAL;", params![])?; + debug!("end of connection_open"); Ok(conn) } @@ -240,7 +259,7 @@ pub fn update_single_value( pub fn dbinit(dbfile: &Path, token_expiration_ms: Option) -> Result { let exists = dbfile.exists(); - let conn = connection_open(dbfile)?; + let conn = connection_open("dbinit", dbfile)?; if !exists { info!("initialdb"); @@ -474,6 +493,11 @@ pub fn dbinit(dbfile: &Path, token_expiration_ms: Option) -> Result, ) -> Result<(), zkerr::Error> { - let conn = connection_open(dbfile)?; + let conn = connection_open("save_zklinks", dbfile)?; for zklink in zklinks.iter() { // TODO: integrate into sql instead of separate queries. diff --git a/server-lib/src/sync.rs b/server-lib/src/sync.rs index e85d2e6a..8c7e5323 100644 --- a/server-lib/src/sync.rs +++ b/server-lib/src/sync.rs @@ -11,13 +11,15 @@ use actix_multipart_rfc7578 as multipart; use actix_web::error::PayloadError; use async_stream::try_stream; use awc; -use bytes::Bytes; +use awc::http::uri::Scheme; +use awc::ws::Frame; +use bytes::{Buf, Bytes}; use futures::executor::block_on; use futures::future; use futures::Stream; use futures_util::TryStreamExt; -use futures_util::{StreamExt, TryFutureExt}; -use log::{error, info}; +use futures_util::{SinkExt, StreamExt, TryFutureExt}; +use log::{debug, error, info}; use orgauth; use orgauth::data::{User, UserId}; use orgauth::dbfun::user_id; @@ -33,6 +35,7 @@ use std::time::Duration; use tokio::io::AsyncBufReadExt; use tokio::io::AsyncWriteExt; use tokio_util::io::StreamReader; +use tracing::instrument::WithSubscriber; use uuid::Uuid; use zkprotocol::constants::{PrivateStreamingRequests, SpecialUuids}; use zkprotocol::content::{FileStatus, OnMakeFileNote, SaveZkNote, Server, SyncSince, ZkNoteId}; @@ -183,6 +186,15 @@ pub fn temp_tables(conn: &Connection) -> Result { params![], )?; + conn.execute( + format!( + "CREATE UNIQUE INDEX \"{}unq\" ON \"{}\" (\"id\")", + notetemp, notetemp + ) + .as_str(), + params![], + )?; + conn.execute( format!( "create temporary table {} (\"id\" integer primary key not null)", @@ -192,6 +204,15 @@ pub fn temp_tables(conn: &Connection) -> Result { params![], )?; + conn.execute( + format!( + "CREATE UNIQUE INDEX \"{}unq\" ON \"{}\" (\"id\")", + archivenotetemp, archivenotetemp + ) + .as_str(), + params![], + )?; + conn.execute( format!( "create temporary table {} ( @@ -223,6 +244,15 @@ pub fn temp_tables(conn: &Connection) -> Result { params![], )?; + conn.execute( + format!( + "CREATE UNIQUE INDEX \"{}unq\" ON \"{}\" (\"id\")", + archivelinktemp, archivelinktemp + ) + .as_str(), + params![], + )?; + Ok(TempTableNames { notetemp, archivenotetemp, @@ -240,7 +270,7 @@ pub async fn sync( callbacks: &mut Callbacks, monitor: &dyn JobMonitor, ) -> Result> { - let conn = Arc::new(sqldata::connection_open(dbpath)?); + let conn = Arc::new(sqldata::connection_open("sync", dbpath)?); let user = orgauth::dbfun::read_user_by_id(&conn, uid)?; // TODO pass this in from calling ftn? let extra_login_data = sqldata::read_extra_login_data(&conn, user.id)?; // let mut callbacks = &mut zknotes_callbacks(); @@ -276,19 +306,19 @@ pub async fn sync( match res { PrivateReply::PvySyncComplete => { write!(monitor, "starting sync to remote"); - let _reply = sync_to_remote( - conn.clone(), - file_path, - &user, - Some(ttn.notetemp.clone()), - Some(ttn.archivenotetemp.clone()), - Some(ttn.linktemp.clone()), - Some(ttn.archivelinktemp.clone()), - after, - callbacks, - monitor, - ) - .await?; + // let _reply = sync_to_remote( + // conn.clone(), + // file_path, + // &user, + // Some(ttn.notetemp.clone()), + // Some(ttn.archivenotetemp.clone()), + // Some(ttn.linktemp.clone()), + // Some(ttn.archivelinktemp.clone()), + // after, + // callbacks, + // monitor, + // ) + // .await?; tr.commit()?; @@ -687,6 +717,110 @@ pub async fn sync_files_up( Ok(resvec) } +// pub async fn read_sync_message_ws(line: &mut String, br: &S) -> Result +// where +// // S: Stream> + Unpin, +// S: Stream> + Unpin, +// { +// line.clear(); +// if br.read_line(line).await? == 0 { +// return Err::(zkerr::Error::EmptyStream); +// } +// let sm = serde_json::from_str(line.trim())?; +// Ok(sm) +// } + +pub async fn sync_from_remote_ws( + conn: &Connection, + lapin_info: &Option, + server: &Server, + user: &User, + after: Option, + file_path: &Path, + notetemp: &String, + archivenotetemp: &String, + linktemp: &String, + archivelinktemp: &String, + callbacks: &mut Callbacks, + monitor: &dyn JobMonitor, +) -> Result> { + let (c, url) = match (user.cookie.clone(), user.remote_url.clone()) { + (Some(c), Some(url)) => (c, url), + _ => return Err("can't remote sync - not a remote user".into()), + }; + + let user_url = awc::http::Uri::try_from(url).map_err(|x| zkerr::Error::String(x.to_string()))?; + let mut parts = awc::http::uri::Parts::default(); + parts.scheme = if user_url.scheme() == Some(&Scheme::HTTP) { + Some("ws".parse().unwrap()) + } else { + Some("wss".parse().unwrap()) + }; + parts.authority = user_url.authority().cloned(); + parts.path_and_query = Some(awc::http::uri::PathAndQuery::from_static("/stream")); + let uri = awc::http::Uri::from_parts(parts).map_err(|x| zkerr::Error::String(x.to_string()))?; + + info!("syncing from uri: {}", uri); + write!(monitor, "syncing from uri: {}", uri); + + let cookie = cookie::Cookie::parse_encoded(c)?; + // ClientResponse, Framed return Ok(PrivateReply::PvyNoop), + // will this be on a line, or totally arbitrary? + Some(Ok(Frame::Text(s))) => serde_json::from_str(&*String::from_utf8_lossy(s.chunk()))?, + // Binary frame. + Some(Ok(Frame::Binary(_b))) => SyncMessage::SyncEnd, + + // Continuation. + Some(Ok(Frame::Continuation(_i))) => SyncMessage::SyncEnd, + + // Ping message. + Some(Ok(Frame::Ping(_b))) => SyncMessage::SyncEnd, + + // Pong message. + Some(Ok(Frame::Pong(_b))) => SyncMessage::SyncEnd, + + // Close message with optional reason. + Some(Ok(Frame::Close(_scr))) => SyncMessage::SyncEnd, + Some(Err(_e)) => SyncMessage::SyncEnd, + }; + + // let (snk, strm) = bs.split(); + // let mut line = String::new(); + // // nope! + // let sr = StreamReader::new(strm); + // read_sync_message_ws(&mut line, &strm); + + // let mut sr = StreamReader::new(bs.map_err(convert_payloaderr)); + // let mut sr = StreamReader::new(bs); + // + // + + // let _ = bs.send(rq); + // + // let mut r = bs.io_ref(); + + // let reply = sync_from_stream( + // conn, + // lapin_info, + // &server, + // &user, + // file_path, + // Some(notetemp), + // Some(archivenotetemp), + // Some(linktemp), + // Some(archivelinktemp), + // callbacks, + // &mut sr, + // ) + // .await?; + // Ok(reply) + Ok(PrivateReply::PvyNoop) +} + pub async fn sync_from_remote( conn: &Connection, lapin_info: &Option, @@ -850,6 +984,7 @@ where // ---------------------------------------------------------------------------------- // current zknotes // ---------------------------------------------------------------------------------- + debug!("current notes"); // First should be the current notes. if let SyncMessage::ZkSearchResultHeader(ref _zsrh) = sm { } else { @@ -864,20 +999,26 @@ where sm = read_sync_message(&mut line, br).await?; - let serverhash = HashMap::::new(); + let mut serverhash = HashMap::::new(); + + let mut notecount: u64 = 0; while let SyncMessage::ZkNote(ref note, ref mbf) = sm { + notecount = notecount + 1; + debug!("notecount {}", notecount); let uid = UserId::Uid( *userhash .get(note.user.to_i64()) .ok_or_else(|| zkerr::Error::String("user not found".to_string()))?, ); + debug!("note5"); let file_id: Option = match mbf { None => None, Some(fileinfo) => // already have a file for this hash? { + debug!("fileinfo {:?}", fileinfo); match conn.query_row( "select id from file where hash = ?1", params![fileinfo.hash], @@ -923,23 +1064,54 @@ where } }; - let server_id = match serverhash.get(¬e.server).ok_or_else(|| { - match server_id(&conn, note.server.as_str()) { - Ok(id) => Ok(id), - Err(_e) => conn - .execute( - "insert into server (uuid, createdate) values (?1, ?2)", - params![note.server, now], - ) - .map(|_| conn.last_insert_rowid()) - .map_err(|e| zkerr::Error::from(e)), + debug!("note4"); + let server_id: i64 = match serverhash.get(¬e.server) { + Some(id) => id.clone(), + None => { + debug!("getting server id"); + match server_id(&conn, note.server.as_str()) { + Ok(id) => { + serverhash.insert(note.server.clone(), id); + id.clone() + } + Err(_e) => { + match conn.execute( + "insert into server (uuid, createdate) values (?1, ?2)", + params![note.server, now], + ) { + Ok(_) => { + let id = conn.last_insert_rowid(); + serverhash.insert(note.server.clone(), id); + id.clone() + } + Err(e) => Err(zkerr::Error::from(e))?, + } + } + } } - }) { - Ok(id) => Ok(id.clone()), - Err(Ok(id)) => Ok(id), - Err(Err(e)) => Err(e), - }?; - + }; + // let server_id = match serverhash.get(¬e.server).ok_or_else(|| { + // match server_id(&conn, note.server.as_str()) { + // Ok(id) => Ok(id), + // Err(_e) => conn + // .execute( + // "insert into server (uuid, createdate) values (?1, ?2)", + // params![note.server, now], + // ) + // .map(|_| { + // let id = conn.last_insert_rowid(); + // serverhash.insert(note.server.clone(), id); + // id + // }) + // .map_err(|e| zkerr::Error::from(e)), + // } + // }) { + // Ok(id) => Ok(id.clone()), // Ok(id.clone()), + // Err(Ok(id)) => Ok(id), + // Err(Err(e)) => Err(e), + // }?; + + debug!("note3"); let ex = conn.execute( "insert into zknote (title, content, user, pubid, editable, showtitle, deleted, uuid, file, server, createdate, changeddate) values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", @@ -958,6 +1130,7 @@ where note.changeddate, ] ); + debug!("note2"); let id: i64 = match ex { Ok(_) => Ok(conn.last_insert_rowid()), Err(rusqlite::Error::SqliteFailure(e, Some(s))) => { @@ -1018,12 +1191,14 @@ where Err(e) => Err(e.into()), }?; + debug!("note1"); if let Some(ref nt) = notetemp { conn.execute( format!("insert into {} values (?1)", nt).as_str(), params![id], )?; } + debug!("note0"); sm = read_sync_message(&mut line, br).await?; } @@ -1032,6 +1207,7 @@ where // archive notes // ---------------------------------------------------------------------------------- // After the current notes are the archivenotes. + debug!("archive notes"); if let SyncMessage::ZkSearchResultHeader(ref _zsrh) = sm { } else { return Err( @@ -1137,6 +1313,7 @@ where // ---------------------------------------------------------------------------------- // archive links // ---------------------------------------------------------------------------------- + debug!("archive links"); sm = read_sync_message(&mut line, br).await?; let mut count = 0; @@ -1216,6 +1393,7 @@ where // ---------------------------------------------------------------------------------- // current links // ---------------------------------------------------------------------------------- + debug!("current links"); if let SyncMessage::UuidZkLinkHeader = sm { } else { return Err( diff --git a/server-lib/src/synctest.rs b/server-lib/src/synctest.rs index 68addffa..83c797c3 100644 --- a/server-lib/src/synctest.rs +++ b/server-lib/src/synctest.rs @@ -584,7 +584,7 @@ mod tests { let mut br = StreamReader::new(ss); println!("0.27"); - match sync_from_stream( + match sync_from_ws( &caconn, &None, &caserver, @@ -785,7 +785,7 @@ mod tests { pin_mut!(ss); let mut br = StreamReader::new(ss); - sync_from_stream( + sync_from_ws( &caconn, &None, &caserver, @@ -823,7 +823,7 @@ mod tests { pin_mut!(cs); let mut cbr = StreamReader::new(cs); - sync_from_stream( + sync_from_ws( &saconn, &None, &saserver, @@ -1152,7 +1152,7 @@ mod tests { pin_mut!(ss); let mut br = StreamReader::new(ss); - sync_from_stream( + sync_from_ws( &caconn, &None, &caserver, @@ -1186,7 +1186,7 @@ mod tests { pin_mut!(cs); let mut cbr = StreamReader::new(cs); - sync_from_stream( + sync_from_ws( &saconn, &None, &saserver, diff --git a/server/default.toml b/server/default.toml index 053dedf1..a53a6605 100644 --- a/server/default.toml +++ b/server/default.toml @@ -21,4 +21,4 @@ invite_token_expiration_ms = 604800000 open_registration = true send_emails = false non_admin_invite = true -remote_registration = false +remote_registration = true From 3ac802d6741468469fe0bea3062592793829253d Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 19 Oct 2026 13:47:01 -0600 Subject: [PATCH 21/40] cleanup --- onsave/src/main.rs | 3 - server-lib/src/interfaces.rs | 12 +-- server-lib/src/sqldata.rs | 2 - server-lib/src/sync.rs | 157 ++++------------------------------- 4 files changed, 22 insertions(+), 152 deletions(-) diff --git a/onsave/src/main.rs b/onsave/src/main.rs index 45c93053..340f1370 100644 --- a/onsave/src/main.rs +++ b/onsave/src/main.rs @@ -147,7 +147,6 @@ async fn err_main() -> Result<(), Box> { loop { let r: Result<(), Box> = async { - println!("here!!"); let conn = match (amqp_uid.clone(), amqp_pwd.clone()) { (Some(uid), Some(pwd)) => { // Hack in the uid and pwd to the url. @@ -429,8 +428,6 @@ pub async fn yeet_service( match res { Ok(res) => { - println!("res: {:?}", res); - if !res.status().is_success() { error!("error saving yeet error note: {}, {:?}", zkn.id, res); } diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index 02985baa..26765b5f 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -361,6 +361,7 @@ pub async fn zk_interface_loggedin( let lapin_channelx = li.map(|li| li.channel).clone(); let token = token.clone(); + // not really necessary. trying to ensure no sqlite blocking. conn.close().map_err(|(_conn, err)| err)?; std::thread::spawn(move || { @@ -429,12 +430,11 @@ pub async fn zk_interface_loggedin( .unwrap() }); - // Ok(PrivateReply::PvyJobStatus(JobStatus { - // jobno: jid.jobno, - // state: JobState::Started, - // message: "".to_string(), - // })) - Ok(PrivateReply::PvyNoop) + Ok(PrivateReply::PvyJobStatus(JobStatus { + jobno: jid.jobno, + state: JobState::Started, + message: "".to_string(), + })) } PrivateRequest::PvqSyncFiles(znsrq) => { let dbpath: PathBuf = state.config.orgauth_config.db.to_path_buf(); diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index 11fa91eb..938ea502 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -1867,13 +1867,11 @@ pub fn save_savelzlinks( where fromid = ?2 and toid = ?3 and user = ?4", params![now, from, to, uid.to_i64()], )?; - println!("inserted {} archive lzlink", ac); // delete the link. let dc = conn.execute( "delete from zklink where fromid = ?1 and toid = ?2 and user = ?3", params![from, to, uid.to_i64()], )?; - println!("deleted {} lzlink", dc); } else { save_zklink(&conn, from, to, uid, Some(linkzknote))?; } diff --git a/server-lib/src/sync.rs b/server-lib/src/sync.rs index 8c7e5323..3a7eeb3a 100644 --- a/server-lib/src/sync.rs +++ b/server-lib/src/sync.rs @@ -306,19 +306,19 @@ pub async fn sync( match res { PrivateReply::PvySyncComplete => { write!(monitor, "starting sync to remote"); - // let _reply = sync_to_remote( - // conn.clone(), - // file_path, - // &user, - // Some(ttn.notetemp.clone()), - // Some(ttn.archivenotetemp.clone()), - // Some(ttn.linktemp.clone()), - // Some(ttn.archivelinktemp.clone()), - // after, - // callbacks, - // monitor, - // ) - // .await?; + let _reply = sync_to_remote( + conn.clone(), + file_path, + &user, + Some(ttn.notetemp.clone()), + Some(ttn.archivenotetemp.clone()), + Some(ttn.linktemp.clone()), + Some(ttn.archivelinktemp.clone()), + after, + callbacks, + monitor, + ) + .await?; tr.commit()?; @@ -392,7 +392,7 @@ pub async fn download_file( .map_err(|e| zkerr::Error::String(e.to_string()))?; if res.status() == 404 { - println!("404 for {}!", title); + info!("404 for {}!", title); return Ok(DownloadResult::DownloadFailed); } @@ -717,110 +717,6 @@ pub async fn sync_files_up( Ok(resvec) } -// pub async fn read_sync_message_ws(line: &mut String, br: &S) -> Result -// where -// // S: Stream> + Unpin, -// S: Stream> + Unpin, -// { -// line.clear(); -// if br.read_line(line).await? == 0 { -// return Err::(zkerr::Error::EmptyStream); -// } -// let sm = serde_json::from_str(line.trim())?; -// Ok(sm) -// } - -pub async fn sync_from_remote_ws( - conn: &Connection, - lapin_info: &Option, - server: &Server, - user: &User, - after: Option, - file_path: &Path, - notetemp: &String, - archivenotetemp: &String, - linktemp: &String, - archivelinktemp: &String, - callbacks: &mut Callbacks, - monitor: &dyn JobMonitor, -) -> Result> { - let (c, url) = match (user.cookie.clone(), user.remote_url.clone()) { - (Some(c), Some(url)) => (c, url), - _ => return Err("can't remote sync - not a remote user".into()), - }; - - let user_url = awc::http::Uri::try_from(url).map_err(|x| zkerr::Error::String(x.to_string()))?; - let mut parts = awc::http::uri::Parts::default(); - parts.scheme = if user_url.scheme() == Some(&Scheme::HTTP) { - Some("ws".parse().unwrap()) - } else { - Some("wss".parse().unwrap()) - }; - parts.authority = user_url.authority().cloned(); - parts.path_and_query = Some(awc::http::uri::PathAndQuery::from_static("/stream")); - let uri = awc::http::Uri::from_parts(parts).map_err(|x| zkerr::Error::String(x.to_string()))?; - - info!("syncing from uri: {}", uri); - write!(monitor, "syncing from uri: {}", uri); - - let cookie = cookie::Cookie::parse_encoded(c)?; - // ClientResponse, Framed return Ok(PrivateReply::PvyNoop), - // will this be on a line, or totally arbitrary? - Some(Ok(Frame::Text(s))) => serde_json::from_str(&*String::from_utf8_lossy(s.chunk()))?, - // Binary frame. - Some(Ok(Frame::Binary(_b))) => SyncMessage::SyncEnd, - - // Continuation. - Some(Ok(Frame::Continuation(_i))) => SyncMessage::SyncEnd, - - // Ping message. - Some(Ok(Frame::Ping(_b))) => SyncMessage::SyncEnd, - - // Pong message. - Some(Ok(Frame::Pong(_b))) => SyncMessage::SyncEnd, - - // Close message with optional reason. - Some(Ok(Frame::Close(_scr))) => SyncMessage::SyncEnd, - Some(Err(_e)) => SyncMessage::SyncEnd, - }; - - // let (snk, strm) = bs.split(); - // let mut line = String::new(); - // // nope! - // let sr = StreamReader::new(strm); - // read_sync_message_ws(&mut line, &strm); - - // let mut sr = StreamReader::new(bs.map_err(convert_payloaderr)); - // let mut sr = StreamReader::new(bs); - // - // - - // let _ = bs.send(rq); - // - // let mut r = bs.io_ref(); - - // let reply = sync_from_stream( - // conn, - // lapin_info, - // &server, - // &user, - // file_path, - // Some(notetemp), - // Some(archivenotetemp), - // Some(linktemp), - // Some(archivelinktemp), - // callbacks, - // &mut sr, - // ) - // .await?; - // Ok(reply) - Ok(PrivateReply::PvyNoop) -} - pub async fn sync_from_remote( conn: &Connection, lapin_info: &Option, @@ -1090,26 +986,6 @@ where } } }; - // let server_id = match serverhash.get(¬e.server).ok_or_else(|| { - // match server_id(&conn, note.server.as_str()) { - // Ok(id) => Ok(id), - // Err(_e) => conn - // .execute( - // "insert into server (uuid, createdate) values (?1, ?2)", - // params![note.server, now], - // ) - // .map(|_| { - // let id = conn.last_insert_rowid(); - // serverhash.insert(note.server.clone(), id); - // id - // }) - // .map_err(|e| zkerr::Error::from(e)), - // } - // }) { - // Ok(id) => Ok(id.clone()), // Ok(id.clone()), - // Err(Ok(id)) => Ok(id), - // Err(Err(e)) => Err(e), - // }?; debug!("note3"); let ex = conn.execute( @@ -1345,7 +1221,7 @@ where // actually there is no unique index on zklinkarchive so can this even happen? if e.code == rusqlite::ErrorCode::ConstraintViolation { // if duplicate record, just ignore and go on. - println!("duplicate archivelink: {:?}", l); + warning!("duplicate archivelink: {:?}", l); None } else { return Err(rusqlite::Error::SqliteFailure(e, s).into()); @@ -1367,7 +1243,7 @@ where // actually there is no unique index on zklinkarchive so can this even happen? if e.code == rusqlite::ErrorCode::ConstraintViolation { // if duplicate record, just ignore and go on. - println!("duplicate archivelink: {:?}", l); + warning!("duplicate archivelink: {:?}", l); None } else { return Err(rusqlite::Error::SqliteFailure(e, s).into()); @@ -1612,7 +1488,6 @@ pub async fn sync_to_remote( .await?; // TODO: send back Sync results... how many records and etc. - Ok(PrivateReply::PvySyncComplete) } From 7615f34c1d8fe62082f9467a3af6283550867b05 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 19 Oct 2026 13:50:09 -0600 Subject: [PATCH 22/40] remove printlns --- server-lib/src/lib.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index 9a3ee46f..d2a74cca 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -482,11 +482,9 @@ async fn private_ws( rt::spawn(async move { while let Some(msg) = stream.next().await { match async { - println!("after stream.next()"); match msg { Ok(AggregatedMessage::Text(t)) => { let pcr: PrivateClosureRequest = serde_json::from_str(&t)?; - println!("after let pcr: PrivateClosureRequest ="); match ( zk_interface_check(&token_uuid, &mut state, &token, pcr.request).await, SystemTime::now() @@ -621,7 +619,6 @@ async fn zk_interface_check( "zk_interface_check", state.config.orgauth_config.db.as_path(), )?; - println!("before read_user_by_token_api"); match orgauth::dbfun::read_user_by_token_api( &conn, token_uuid, @@ -636,7 +633,6 @@ async fn zk_interface_check( ))) } Ok(userdata) => { - println!("after read_user_by_token_api"); // finally! processing messages as logged in user. interfaces::zk_interface_loggedin(state, conn, token, userdata.id, &msg).await } From 6b243d41d7ecc1fa77e3af6827a243f969d14c8a Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 19 Oct 2026 14:00:42 -0600 Subject: [PATCH 23/40] more cleanup --- server-lib/src/lib.rs | 23 +++++++++++------------ server-lib/src/search.rs | 4 +--- server-lib/src/sqldata.rs | 17 ++++++++--------- server-lib/src/sync.rs | 13 +++++-------- 4 files changed, 25 insertions(+), 32 deletions(-) diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index d2a74cca..b196b07e 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -19,14 +19,12 @@ use actix_session::{ use actix_web::{ cookie::{self, Key}, dev::Server, - rt, web, App, HttpRequest, HttpResponse, HttpServer, ResponseError, Result, + rt, web, App, HttpRequest, HttpResponse, HttpServer, Result, }; -use actix_ws::{AggregatedMessage, CloseReason}; +use actix_ws::AggregatedMessage; use chrono; use clap::Arg; use config::Config; -use either::IntoEither; -use futures::SinkExt; use futures_util::StreamExt as _; use futures_util::TryStreamExt as _; use girlboss::Girlboss; @@ -45,18 +43,14 @@ use serde::{Deserialize, Serialize}; use serde_json; use simple_error::simple_error; use sqldata::{get_single_value, local_server_id, LapinInfo}; +use std::io::{stdin, Write}; use std::path::PathBuf; use std::str::FromStr; use std::{env, sync::RwLock}; use std::{error::Error, sync::Arc}; use std::{ffi::c_int, path::Path}; use std::{fs::File, time::SystemTime}; -use std::{ - io::{stdin, Write}, - time::Duration, -}; use timer; -use tokio::time::sleep; use tokio_util::io::StreamReader; use tracing_actix_web::TracingLogger; use uuid::Uuid; @@ -693,7 +687,7 @@ async fn private_upstreaming( } fn convert_bodyerr(err: actix_web::error::PayloadError) -> std::io::Error { - // error!("convert_err {:?}", err); + error!("convert_bodyerr {:?}", err); todo!() } @@ -817,13 +811,18 @@ fn rulog(cint: c_int, meh: &str) { info!("rulog: {}; {}", cint, meh); } +const SQLITE_LOGGING: bool = false; + #[actix_web::main] pub async fn err_main( oconfig: Option, logfile: Option, ) -> Result<(), Box> { - unsafe { - config_log(Some(rulog))?; + // sqlite logging + if SQLITE_LOGGING { + unsafe { + config_log(Some(rulog))?; + } } match logfile { diff --git a/server-lib/src/search.rs b/server-lib/src/search.rs index 39966e6e..eca67319 100644 --- a/server-lib/src/search.rs +++ b/server-lib/src/search.rs @@ -5,7 +5,7 @@ use crate::sqldata::server_id; use crate::sqldata::{delete_zknote, get_sysids, note_id}; use async_stream::try_stream; use futures::Stream; -use log::{debug, error, info}; +use log::debug; use orgauth::data::UserId; use orgauth::dbfun::user_id; use rusqlite::Connection; @@ -239,8 +239,6 @@ pub fn search_zknotes_stream( try_stream! { let (sql, args) = build_sql(&conn, user, &search, exclude_notes)?; - debug!("search_zknotes_stream sql\n {}", sql); - let mut stmt = conn.prepare(sql.as_str())?; let mut rows = stmt.query(rusqlite::params_from_iter(args.iter()))?; diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index 938ea502..7c88c31f 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -172,12 +172,8 @@ pub fn set_homenote( fn sqlog(meh: &str) { info!("sqlog: {}", meh); } -// unsafe extern "C" fn trace_callback( -// evt: c_uint, -// ctx: *mut c_void, -// p: *mut c_void, -// x: *mut c_void, -// ) -> c_int { + +const SQLITE_TRACING: bool = false; pub fn connection_open(connid: &str, dbfile: &Path) -> Result { debug!("connection id: {}", connid); @@ -193,7 +189,10 @@ pub fn connection_open(connid: &str, dbfile: &Path) -> Result Date: Mon, 19 Oct 2026 14:06:59 -0600 Subject: [PATCH 24/40] remove sync debugs --- server-lib/src/sqldata.rs | 6 ----- server-lib/src/sync.rs | 56 +++++++++++++++++---------------------- 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index 7c88c31f..4ec184ec 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -176,11 +176,8 @@ fn sqlog(meh: &str) { const SQLITE_TRACING: bool = false; pub fn connection_open(connid: &str, dbfile: &Path) -> Result { - debug!("connection id: {}", connid); let mut conn = Connection::open(dbfile)?; - debug!("after connection_open"); - // conn.busy_timeout(Duration::from_millis(500))?; conn.busy_handler(Some(|count| { info!("busy_handler: {}", count); @@ -194,16 +191,13 @@ pub fn connection_open(connid: &str, dbfile: &Path) -> Result = match mbf { None => None, Some(fileinfo) => // already have a file for this hash? { - debug!("fileinfo {:?}", fileinfo); match conn.query_row( "select id from file where hash = ?1", params![fileinfo.hash], @@ -957,34 +953,29 @@ where } }; - debug!("note4"); let server_id: i64 = match serverhash.get(¬e.server) { Some(id) => id.clone(), - None => { - debug!("getting server id"); - match server_id(&conn, note.server.as_str()) { - Ok(id) => { - serverhash.insert(note.server.clone(), id); - id.clone() - } - Err(_e) => { - match conn.execute( - "insert into server (uuid, createdate) values (?1, ?2)", - params![note.server, now], - ) { - Ok(_) => { - let id = conn.last_insert_rowid(); - serverhash.insert(note.server.clone(), id); - id.clone() - } - Err(e) => Err(zkerr::Error::from(e))?, + None => match server_id(&conn, note.server.as_str()) { + Ok(id) => { + serverhash.insert(note.server.clone(), id); + id.clone() + } + Err(_e) => { + match conn.execute( + "insert into server (uuid, createdate) values (?1, ?2)", + params![note.server, now], + ) { + Ok(_) => { + let id = conn.last_insert_rowid(); + serverhash.insert(note.server.clone(), id); + id.clone() } + Err(e) => Err(zkerr::Error::from(e))?, } } - } + }, }; - debug!("note3"); let ex = conn.execute( "insert into zknote (title, content, user, pubid, editable, showtitle, deleted, uuid, file, server, createdate, changeddate) values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", @@ -1003,7 +994,6 @@ where note.changeddate, ] ); - debug!("note2"); let id: i64 = match ex { Ok(_) => Ok(conn.last_insert_rowid()), Err(rusqlite::Error::SqliteFailure(e, Some(s))) => { @@ -1064,23 +1054,24 @@ where Err(e) => Err(e.into()), }?; - debug!("note1"); if let Some(ref nt) = notetemp { conn.execute( format!("insert into {} values (?1)", nt).as_str(), params![id], )?; } - debug!("note0"); sm = read_sync_message(&mut line, br).await?; } + info!("synced {} zknotes", notecount); + // ---------------------------------------------------------------------------------- // archive notes // ---------------------------------------------------------------------------------- // After the current notes are the archivenotes. - debug!("archive notes"); + // + notecount = 0; if let SyncMessage::ZkSearchResultHeader(ref _zsrh) = sm { } else { return Err( @@ -1094,6 +1085,7 @@ where sm = read_sync_message(&mut line, br).await?; while let SyncMessage::ZkNote(ref note, ref _mbf) = sm { + notecount = notecount + 1; // TODO: make file source record (?) let uid = UserId::Uid( @@ -1172,6 +1164,8 @@ where sm = read_sync_message(&mut line, br).await?; } + info!("synced {} archive zknotes", notecount); + if let SyncMessage::ArchiveZkLinkHeader = sm { } else { return Err( @@ -1186,7 +1180,6 @@ where // ---------------------------------------------------------------------------------- // archive links // ---------------------------------------------------------------------------------- - debug!("archive links"); sm = read_sync_message(&mut line, br).await?; let mut count = 0; @@ -1266,7 +1259,6 @@ where // ---------------------------------------------------------------------------------- // current links // ---------------------------------------------------------------------------------- - debug!("current links"); if let SyncMessage::UuidZkLinkHeader = sm { } else { return Err( From 900388262f343d2e784b847fd35b10f51201bd9e Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 19 Oct 2026 14:34:45 -0600 Subject: [PATCH 25/40] remove infos --- server-lib/src/lib.rs | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index b196b07e..d76f2c76 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -619,13 +619,9 @@ async fn zk_interface_check( state.config.orgauth_config.login_token_expiration_ms, state.config.orgauth_config.regen_login_tokens, ) { - Err(e) => { - info!("read_user_by_token_api error2: {:?}, {:?}", token_uuid, e); - - Ok(PrivateReply::PvyServerError(PrivateError::PveLoginError( - e.to_string(), - ))) - } + Err(e) => Ok(PrivateReply::PvyServerError(PrivateError::PveLoginError( + e.to_string(), + ))), Ok(userdata) => { // finally! processing messages as logged in user. interfaces::zk_interface_loggedin(state, conn, token, userdata.id, &msg).await @@ -652,14 +648,9 @@ async fn zk_interface_check_streaming( config.orgauth_config.login_token_expiration_ms, config.orgauth_config.regen_login_tokens, ) { - Err(e) => { - info!("read_user_by_token_api error3: {:?}, {:?}", token, e); - Ok( - HttpResponse::Ok().json(PrivateReply::PvyServerError(PrivateError::PveLoginError( - format!("{:?}", e), - ))), - ) - } + Err(e) => Ok(HttpResponse::Ok().json(PrivateReply::PvyServerError( + PrivateError::PveLoginError(format!("{:?}", e)), + ))), Ok(userdata) => { // finally! processing messages as logged in user. interfaces::zk_interface_loggedin_streaming(&config, userdata.id, &msg).await @@ -714,15 +705,9 @@ async fn zk_interface_check_upstreaming( config.orgauth_config.login_token_expiration_ms, config.orgauth_config.regen_login_tokens, ) { - Err(e) => { - info!("read_user_by_token_api error4: {:?}, {:?}", token, e); - - Ok( - HttpResponse::Ok().json(PrivateReply::PvyServerError(PrivateError::PveLoginError( - format!("{:?}", e), - ))), - ) - } + Err(e) => Ok(HttpResponse::Ok().json(PrivateReply::PvyServerError( + PrivateError::PveLoginError(format!("{:?}", e)), + ))), Ok(user) => { // finally! processing messages as logged in user. let rstream = From 7ab51408a537cdde8ee7c9e486312abb724da3f8 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 19 Oct 2026 14:36:20 -0600 Subject: [PATCH 26/40] WAL mode again --- server-lib/src/sqldata.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index 4ec184ec..ade3016a 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -195,8 +195,7 @@ pub fn connection_open(connid: &str, dbfile: &Path) -> Result Date: Mon, 19 Oct 2026 14:52:19 -0600 Subject: [PATCH 27/40] fix tests --- server-lib/src/search.rs | 1 - server-lib/src/sqldata.rs | 4 ++-- server-lib/src/sqltest.rs | 12 ++++++------ server-lib/src/synctest.rs | 14 +++++++------- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/server-lib/src/search.rs b/server-lib/src/search.rs index eca67319..4c7ed1af 100644 --- a/server-lib/src/search.rs +++ b/server-lib/src/search.rs @@ -5,7 +5,6 @@ use crate::sqldata::server_id; use crate::sqldata::{delete_zknote, get_sysids, note_id}; use async_stream::try_stream; use futures::Stream; -use log::debug; use orgauth::data::UserId; use orgauth::dbfun::user_id; use rusqlite::Connection; diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index ade3016a..4f4ce0bd 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -6,7 +6,7 @@ use barrel::backend::Sqlite; use lapin::options::QueueDeclareOptions; use lapin::types::FieldTable; use lapin::Channel; -use log::{debug, error, info}; +use log::{error, info}; use orgauth::data::{RegistrationData, UserId}; use orgauth::dbfun::user_id; use orgauth::endpoints::Callbacks; @@ -175,7 +175,7 @@ fn sqlog(meh: &str) { const SQLITE_TRACING: bool = false; -pub fn connection_open(connid: &str, dbfile: &Path) -> Result { +pub fn connection_open(_connid: &str, dbfile: &Path) -> Result { let mut conn = Connection::open(dbfile)?; // conn.busy_timeout(Duration::from_millis(500))?; diff --git a/server-lib/src/sqltest.rs b/server-lib/src/sqltest.rs index 618bd16f..7628eefe 100644 --- a/server-lib/src/sqltest.rs +++ b/server-lib/src/sqltest.rs @@ -40,7 +40,7 @@ mod tests { dbinit(dbp, None)?; - let conn = connection_open(dbp)?; + let conn = connection_open("dbp", dbp)?; let server = local_server_id(&conn)?; @@ -754,10 +754,10 @@ mod tests { } println!("25"); - // should be 4 notes tagged with et'user' - 3 users and 1 'system'. + // should be 3 notes tagged with e>'user' - 3 users let u1pubnote2_exact_search = ZkNoteSearch { tagsearch: vec![TagSearch::SearchTerm { - mods: vec![SearchMod::ExactMatch, SearchMod::Tag], + mods: vec![SearchMod::ExactMatch, SearchMod::TagTo], term: "user".to_string(), }], offset: 0, @@ -771,10 +771,10 @@ mod tests { match search_zknotes(&conn, filesdir, uid1, &u1pubnote2_exact_search)? { SearchResult::SrListNote(zklr) => { - if zklr.notes.len() == 4 { + if zklr.notes.len() == 3 { () } else { - panic!("test failed") + panic!("test failed, len was {}", zklr.notes.len()) } } _ => panic!("test failed"), @@ -784,7 +784,7 @@ mod tests { // should be 9 notes for 'user1' let u1pubnote2_exact_search = ZkNoteSearch { tagsearch: vec![TagSearch::SearchTerm { - mods: vec![SearchMod::ExactMatch, SearchMod::Tag, SearchMod::User], + mods: vec![SearchMod::ExactMatch, SearchMod::TagTo, SearchMod::User], term: "user1".to_string(), }], offset: 0, diff --git a/server-lib/src/synctest.rs b/server-lib/src/synctest.rs index 83c797c3..e642128a 100644 --- a/server-lib/src/synctest.rs +++ b/server-lib/src/synctest.rs @@ -480,7 +480,7 @@ mod tests { fs::copy(dbp_server, Path::new("server.db.saved"))?; } } - let server_conn = connection_open(dbp_server)?; + let server_conn = connection_open("dbp_server", dbp_server)?; let server_ts = setup_db(&server_conn, &mut cb, "".to_string(), None, None, "server").await?; println!("0.1"); @@ -506,7 +506,7 @@ mod tests { fs::copy(dbp_client, Path::new("client.db.saved"))?; } } - let client_conn = connection_open(dbp_client)?; + let client_conn = connection_open("dbp_client", dbp_client)?; let client_ts = setup_db( &client_conn, &mut cb, @@ -584,7 +584,7 @@ mod tests { let mut br = StreamReader::new(ss); println!("0.27"); - match sync_from_ws( + match sync_from_stream( &caconn, &None, &caserver, @@ -785,7 +785,7 @@ mod tests { pin_mut!(ss); let mut br = StreamReader::new(ss); - sync_from_ws( + sync_from_stream( &caconn, &None, &caserver, @@ -823,7 +823,7 @@ mod tests { pin_mut!(cs); let mut cbr = StreamReader::new(cs); - sync_from_ws( + sync_from_stream( &saconn, &None, &saserver, @@ -1152,7 +1152,7 @@ mod tests { pin_mut!(ss); let mut br = StreamReader::new(ss); - sync_from_ws( + sync_from_stream( &caconn, &None, &caserver, @@ -1186,7 +1186,7 @@ mod tests { pin_mut!(cs); let mut cbr = StreamReader::new(cs); - sync_from_ws( + sync_from_stream( &saconn, &None, &saserver, From 060423215e2381779f55c5a573beefe8c654898d Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 19 Oct 2026 16:59:36 -0600 Subject: [PATCH 28/40] orgauth bump --- orgauth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgauth b/orgauth index f92e19b3..78cd395c 160000 --- a/orgauth +++ b/orgauth @@ -1 +1 @@ -Subproject commit f92e19b3db304f4b5e72e3b4cf71c022c9f70497 +Subproject commit 78cd395cf3cb45e15ba1bdcfd70eac036abe67dd From f1c501e80a9cec9b4cd1985150dbce556e1a3a6e Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 19 Oct 2026 18:40:00 -0600 Subject: [PATCH 29/40] read-only connections; more sync monitor messages; uuid index for zkarch --- server-lib/src/interfaces.rs | 64 ++++++++++++++++++++++++++++++++++-- server-lib/src/lib.rs | 27 +++++++++++---- server-lib/src/migrations.rs | 14 ++++++++ server-lib/src/sqldata.rs | 34 ++++++++++++++++--- server-lib/src/sync.rs | 12 ++++++- server/static/index.html | 2 +- 6 files changed, 137 insertions(+), 16 deletions(-) diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index 26765b5f..3707f9a9 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -45,8 +45,11 @@ pub fn login_data_for_token( session: Session, config: &Config, ) -> Result, Box> { - let mut conn = - sqldata::connection_open("login_data_for_token", config.orgauth_config.db.as_path())?; + let mut conn = sqldata::connection_open( + "login_data_for_token", + true, + config.orgauth_config.db.as_path(), + )?; conn.busy_timeout(Duration::from_millis(500))?; let mut cb = sqldata::zknotes_callbacks(); @@ -180,6 +183,7 @@ pub async fn zk_interface_loggedin_streaming( let search: ZkNoteSearch = serde_json::from_value(msgdata.clone())?; let conn = Arc::new(sqldata::connection_open( "PrivateStreamingRequests::SearchZkNotes", + true, config.orgauth_config.db.as_path(), )?); let znsstream = @@ -191,6 +195,7 @@ pub async fn zk_interface_loggedin_streaming( let rq: GetArchiveZkLinks = serde_json::from_value(msgdata.clone())?; let conn = Arc::new(sqldata::connection_open( "psr:GetArchiveZkLinks", + true, config.orgauth_config.db.as_path(), )?); let bstream = sqldata::read_archivezklinks_stream(conn, uid, rq.createddate_after, None) @@ -201,6 +206,7 @@ pub async fn zk_interface_loggedin_streaming( let rq: GetZkLinksSince = serde_json::from_value(msgdata.clone())?; let conn = Arc::new(sqldata::connection_open( "psr:GetZkLinksSince", + true, config.orgauth_config.db.as_path(), )?); let bstream = sqldata::read_zklinks_since_stream(conn, uid, rq.createddate_after, None) @@ -210,6 +216,7 @@ pub async fn zk_interface_loggedin_streaming( PrivateStreamingRequests::Sync => { let conn = Arc::new(sqldata::connection_open( "psr:Sync", + false, config.orgauth_config.db.as_path(), )?); let rq: SyncSince = serde_json::from_value(msgdata.clone())?; @@ -317,23 +324,53 @@ pub async fn zk_interface_loggedin( } } PrivateRequest::PvqPowerDelete(search) => { + conn.close(); + let conn = sqldata::connection_open( + "PvqPowerDelete", + false, + state.config.orgauth_config.db.as_path(), + )?; let res = search::power_delete_zknotes(&conn, state.config.file_path.clone(), uid, &search)?; Ok(PrivateReply::PvyPowerDeleteComplete(res)) } PrivateRequest::PvqDeleteZkNote(id) => { + conn.close(); + let conn = sqldata::connection_open( + "PvqPowerDelete", + false, + state.config.orgauth_config.db.as_path(), + )?; sqldata::delete_zknote(&conn, state.config.file_path.clone(), uid, &id)?; Ok(PrivateReply::PvyDeletedZkNote(id.clone())) } PrivateRequest::PvqSaveZkNote(sbe) => { + conn.close(); + let conn = sqldata::connection_open( + "PvqPowerDelete", + false, + state.config.orgauth_config.db.as_path(), + )?; let li = connect_and_make_lapin_info(state, token).await; let (_id, s) = sqldata::save_zknote(&conn, &li, &state.server, uid, &sbe, None).await?; Ok(PrivateReply::PvySavedZkNote(s)) } PrivateRequest::PvqSaveZkLinks(msg) => { + conn.close(); + let conn = sqldata::connection_open( + "PvqPowerDelete", + false, + state.config.orgauth_config.db.as_path(), + )?; let _ = sqldata::save_zklinks(&state.config.orgauth_config.db.as_path(), uid, &msg.links)?; Ok(PrivateReply::PvySavedZkLinks) } PrivateRequest::PvqSaveZkNoteAndLinks(sznpl) => { + conn.close(); + let conn = sqldata::connection_open( + "PvqPowerDelete", + false, + state.config.orgauth_config.db.as_path(), + )?; let li = connect_and_make_lapin_info(state, token).await; let (_, szkn) = sqldata::save_zknote(&conn, &li, &state.server, uid, &sznpl.note, None).await?; @@ -342,15 +379,33 @@ pub async fn zk_interface_loggedin( Ok(PrivateReply::PvySavedZkNoteAndLinks(szkn)) } PrivateRequest::PvqSaveImportZkNotes(gzl) => { + conn.close(); + let conn = sqldata::connection_open( + "PvqPowerDelete", + false, + state.config.orgauth_config.db.as_path(), + )?; let li = connect_and_make_lapin_info(state, token).await; sqldata::save_importzknotes(&conn, &li, &state.server, uid, gzl).await?; Ok(PrivateReply::PvySavedImportZkNotes) } PrivateRequest::PvqSetHomeNote(hn) => { + conn.close(); + let conn = sqldata::connection_open( + "PvqPowerDelete", + false, + state.config.orgauth_config.db.as_path(), + )?; sqldata::set_homenote(&conn, uid, hn)?; Ok(PrivateReply::PvyHomeNoteSet(hn.clone())) } PrivateRequest::PvqSyncRemote => { + conn.close(); + let conn = sqldata::connection_open( + "PvqPowerDelete", + false, + state.config.orgauth_config.db.as_path(), + )?; let dbpath: PathBuf = state.config.orgauth_config.db.to_path_buf(); let file_path: PathBuf = state.config.file_path.to_path_buf(); let uid: UserId = uid; @@ -471,7 +526,7 @@ pub async fn zk_interface_loggedin( write!(gbm, "starting file sync"); let r = async { - let conn = sqldata::connection_open("startit", &dbpath.as_path())?; + let conn = sqldata::connection_open("startit", false, &dbpath.as_path())?; let _dv = sync::sync_files_down( &conn, &lapin_info, @@ -570,6 +625,7 @@ pub fn public_interface( PublicRequest::PbrGetZkNoteAndLinks(gzne) => { let conn = sqldata::connection_open( "PublicRequest::PbrGetZkNoteAndLinks", + true, config.orgauth_config.db.as_path(), )?; let (_, note) = sqldata::read_zknote(&conn, &config.file_path, None, &gzne.zknote)?; @@ -593,6 +649,7 @@ pub fn public_interface( ); let conn = sqldata::connection_open( "PublicRequest::PbrGetZknIfChanged", + true, config.orgauth_config.db.as_path(), )?; let ozkne = sqldata::read_zneifchanged(&conn, &config.file_path, None, &gzic)?; @@ -605,6 +662,7 @@ pub fn public_interface( PublicRequest::PbrGetZkNotePubId(pubid) => { let conn = sqldata::connection_open( "PublicRequest::PbrGetZkNotePubId", + true, config.orgauth_config.db.as_path(), )?; let note = sqldata::read_zknotepubid(&conn, &config.file_path, None, pubid.as_str())?; diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index d76f2c76..e874667b 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -183,7 +183,7 @@ async fn user( // req.connection_info() // ); match async { - let conn = sqldata::connection_open("user", data.config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open("user", false, data.config.orgauth_config.db.as_path())?; interfaces::user_interface( &conn, @@ -259,7 +259,8 @@ pub fn get_tauri_uid(conn: &Connection) -> Result, zkerr::Error> } async fn file(session: Session, state: web::Data, req: HttpRequest) -> HttpResponse { - let conn = match sqldata::connection_open("file", state.config.orgauth_config.db.as_path()) { + let conn = match sqldata::connection_open("file", true, state.config.orgauth_config.db.as_path()) + { Ok(c) => c, Err(e) => return HttpResponse::InternalServerError().body(format!("{:?}", e)), }; @@ -341,7 +342,11 @@ async fn make_file_notes( lapin_info: &Option, payload: &mut Multipart, ) -> Result> { - let conn = sqldata::connection_open("make_file_notes", state.config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open( + "make_file_notes", + false, + state.config.orgauth_config.db.as_path(), + )?; let uid = if state.config.tauri_mode { get_tauri_uid(&conn)?.ok_or(zkerr::Error::NotLoggedIn)? } else { @@ -611,8 +616,10 @@ async fn zk_interface_check( ) -> Result { let conn = sqldata::connection_open( "zk_interface_check", + true, state.config.orgauth_config.db.as_path(), )?; + info!("pre read_user_by_token_api"); match orgauth::dbfun::read_user_by_token_api( &conn, token_uuid, @@ -640,6 +647,7 @@ async fn zk_interface_check_streaming( Some(token) => { let conn = sqldata::connection_open( "zk_interface_check_streaming", + true, config.orgauth_config.db.as_path(), )?; match orgauth::dbfun::read_user_by_token_api( @@ -694,6 +702,7 @@ async fn zk_interface_check_upstreaming( Some(token) => { let conn = sqldata::connection_open( "zk_interface_check_upstreaming", + true, config.orgauth_config.db.as_path(), )?; @@ -727,6 +736,7 @@ async fn zk_interface_check_upstreaming( None, &mut sqldata::zknotes_callbacks(), &mut br, + None, ) .await?; @@ -925,7 +935,11 @@ pub async fn err_main( // promoting a user to admin? if let Some(uid) = matches.value_of("promote_to_admin") { - let conn = sqldata::connection_open("promote to admin", config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open( + "promote to admin", + false, + config.orgauth_config.db.as_path(), + )?; let mut user = orgauth::dbfun::read_user_by_name(&conn, uid)?; user.admin = true; orgauth::dbfun::update_user(&conn, &user)?; @@ -942,7 +956,8 @@ pub async fn err_main( stdin().read_line(&mut pwd)?; let mut cb = sqldata::zknotes_callbacks(); - let conn = sqldata::connection_open("creating admin", config.orgauth_config.db.as_path())?; + let conn = + sqldata::connection_open("creating admin", false, config.orgauth_config.db.as_path())?; // make new registration i let rd = orgauth::data::RegistrationData { uid: username.to_string(), @@ -1033,7 +1048,7 @@ pub async fn init_server(mut config: Config) -> Result> { }; { - let conn = sqldata::connection_open("init_server", config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open("init_server", false, config.orgauth_config.db.as_path())?; let tr = conn.unchecked_transaction()?; sqldata::update_filetable(&conn, &config.file_path)?; diff --git a/server-lib/src/migrations.rs b/server-lib/src/migrations.rs index 29fcd33d..db4066f6 100644 --- a/server-lib/src/migrations.rs +++ b/server-lib/src/migrations.rs @@ -3213,3 +3213,17 @@ pub fn udpate44(dbfile: &Path) -> Result<(), zkerr::Error> { tr.commit()?; Ok(()) } + +pub fn udpate45(dbfile: &Path) -> Result<(), zkerr::Error> { + // db connection without foreign key checking. + let conn = Connection::open(dbfile)?; + let tr = conn.unchecked_transaction()?; + + conn.execute( + "CREATE INDEX \"idx_zkarchuuid\" ON \"zkarch\" (\"uuid\")", + params![], + )?; + + tr.commit()?; + Ok(()) +} diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index 4f4ce0bd..4dd45f5e 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -11,8 +11,8 @@ use orgauth::data::{RegistrationData, UserId}; use orgauth::dbfun::user_id; use orgauth::endpoints::Callbacks; use orgauth::util::now; -use rusqlite::Row; use rusqlite::{params, Connection}; +use rusqlite::{OpenFlags, Row}; use serde_derive::{Deserialize, Serialize}; use simple_error::bail; use std::iter::FromIterator; @@ -175,8 +175,27 @@ fn sqlog(meh: &str) { const SQLITE_TRACING: bool = false; -pub fn connection_open(_connid: &str, dbfile: &Path) -> Result { - let mut conn = Connection::open(dbfile)?; +pub fn connection_open( + _connid: &str, + read_only: bool, + dbfile: &Path, +) -> Result { + let mut conn = if read_only { + info!("beefore"); + let c = Connection::open_with_flags( + dbfile, + OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_NO_MUTEX + | OpenFlags::SQLITE_OPEN_URI, // OpenFlags::SQLITE_OPEN_READ_ONLY + // | OpenFlags::SQLITE_OPEN_CREATE + // | OpenFlags::SQLITE_OPEN_URI + // | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + info!("eefter"); + c + } else { + Connection::open(dbfile)? + }; // conn.busy_timeout(Duration::from_millis(500))?; conn.busy_handler(Some(|count| { @@ -251,7 +270,7 @@ pub fn update_single_value( pub fn dbinit(dbfile: &Path, token_expiration_ms: Option) -> Result { let exists = dbfile.exists(); - let conn = connection_open("dbinit", dbfile)?; + let conn = connection_open("dbinit", false, dbfile)?; if !exists { info!("initialdb"); @@ -490,6 +509,11 @@ pub fn dbinit(dbfile: &Path, token_expiration_ms: Option) -> Result, ) -> Result<(), zkerr::Error> { - let conn = connection_open("save_zklinks", dbfile)?; + let conn = connection_open("save_zklinks", false, dbfile)?; for zklink in zklinks.iter() { // TODO: integrate into sql instead of separate queries. diff --git a/server-lib/src/sync.rs b/server-lib/src/sync.rs index 5333277b..70a7786d 100644 --- a/server-lib/src/sync.rs +++ b/server-lib/src/sync.rs @@ -267,7 +267,7 @@ pub async fn sync( callbacks: &mut Callbacks, monitor: &dyn JobMonitor, ) -> Result> { - let conn = Arc::new(sqldata::connection_open("sync", dbpath)?); + let conn = Arc::new(sqldata::connection_open("sync", false, dbpath)?); let user = orgauth::dbfun::read_user_by_id(&conn, uid)?; // TODO pass this in from calling ftn? let extra_login_data = sqldata::read_extra_login_data(&conn, user.id)?; // let mut callbacks = &mut zknotes_callbacks(); @@ -768,6 +768,7 @@ pub async fn sync_from_remote( Some(archivelinktemp), callbacks, &mut sr, + Some(monitor), ) .await?; Ok(reply) @@ -800,6 +801,7 @@ pub async fn sync_from_stream( archivelinktemp: Option<&str>, callbacks: &mut Callbacks, br: &mut StreamReader, + monitor: Option<&dyn JobMonitor>, ) -> Result> where S: Stream> + Unpin, @@ -897,6 +899,9 @@ where while let SyncMessage::ZkNote(ref note, ref mbf) = sm { notecount = notecount + 1; + if notecount % 100 == 0 { + monitor.map(|m| write!(m, "sync downloaded {} notes", notecount)); + } let uid = UserId::Uid( *userhash .get(note.user.to_i64()) @@ -1065,6 +1070,7 @@ where } info!("synced {} zknotes", notecount); + monitor.map(|m| write!(m, "sync downloaded {} notes total", notecount)); // ---------------------------------------------------------------------------------- // archive notes @@ -1086,6 +1092,9 @@ where sm = read_sync_message(&mut line, br).await?; while let SyncMessage::ZkNote(ref note, ref _mbf) = sm { notecount = notecount + 1; + if notecount % 100 == 0 { + monitor.map(|m| write!(m, "sync downloaded {} archive notes", notecount)); + } // TODO: make file source record (?) let uid = UserId::Uid( @@ -1165,6 +1174,7 @@ where } info!("synced {} archive zknotes", notecount); + monitor.map(|m| write!(m, "sync downloaded {} archive notes total", notecount)); if let SyncMessage::ArchiveZkLinkHeader = sm { } else { diff --git a/server/static/index.html b/server/static/index.html index cb5fd680..70c29e88 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -68,7 +68,7 @@ login : login, tauri : false, mobile : false, - websockets : true, + websockets : false, adminsettings : adminsettings} } ); From c8ad5d5568bd2930973274a290c01ae94b2d9d0b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 19 Oct 2026 19:14:09 -0600 Subject: [PATCH 30/40] remove infossh root@zknotes.com --- server-lib/src/sqldata.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index 4dd45f5e..77153322 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -181,7 +181,6 @@ pub fn connection_open( dbfile: &Path, ) -> Result { let mut conn = if read_only { - info!("beefore"); let c = Connection::open_with_flags( dbfile, OpenFlags::SQLITE_OPEN_READ_ONLY @@ -191,7 +190,6 @@ pub fn connection_open( // | OpenFlags::SQLITE_OPEN_URI // | OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; - info!("eefter"); c } else { Connection::open(dbfile)? From 564006dd60a14ea6065ed48469e4cc9f20903dce Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 20 Oct 2026 06:32:44 -0600 Subject: [PATCH 31/40] fix warnings --- server-lib/src/interfaces.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index 3707f9a9..693a77dd 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -324,7 +324,8 @@ pub async fn zk_interface_loggedin( } } PrivateRequest::PvqPowerDelete(search) => { - conn.close(); + conn.close().map_err(|e| e.1)?; + let conn = sqldata::connection_open( "PvqPowerDelete", false, @@ -334,7 +335,8 @@ pub async fn zk_interface_loggedin( Ok(PrivateReply::PvyPowerDeleteComplete(res)) } PrivateRequest::PvqDeleteZkNote(id) => { - conn.close(); + conn.close().map_err(|e| e.1)?; + let conn = sqldata::connection_open( "PvqPowerDelete", false, @@ -344,7 +346,8 @@ pub async fn zk_interface_loggedin( Ok(PrivateReply::PvyDeletedZkNote(id.clone())) } PrivateRequest::PvqSaveZkNote(sbe) => { - conn.close(); + conn.close().map_err(|e| e.1)?; + let conn = sqldata::connection_open( "PvqPowerDelete", false, @@ -355,7 +358,8 @@ pub async fn zk_interface_loggedin( Ok(PrivateReply::PvySavedZkNote(s)) } PrivateRequest::PvqSaveZkLinks(msg) => { - conn.close(); + conn.close().map_err(|e| e.1)?; + let conn = sqldata::connection_open( "PvqPowerDelete", false, @@ -365,7 +369,8 @@ pub async fn zk_interface_loggedin( Ok(PrivateReply::PvySavedZkLinks) } PrivateRequest::PvqSaveZkNoteAndLinks(sznpl) => { - conn.close(); + conn.close().map_err(|e| e.1)?; + let conn = sqldata::connection_open( "PvqPowerDelete", false, @@ -379,7 +384,8 @@ pub async fn zk_interface_loggedin( Ok(PrivateReply::PvySavedZkNoteAndLinks(szkn)) } PrivateRequest::PvqSaveImportZkNotes(gzl) => { - conn.close(); + conn.close().map_err(|e| e.1)?; + let conn = sqldata::connection_open( "PvqPowerDelete", false, @@ -390,7 +396,7 @@ pub async fn zk_interface_loggedin( Ok(PrivateReply::PvySavedImportZkNotes) } PrivateRequest::PvqSetHomeNote(hn) => { - conn.close(); + conn.close().map_err(|e| e.1)?; let conn = sqldata::connection_open( "PvqPowerDelete", false, @@ -400,7 +406,7 @@ pub async fn zk_interface_loggedin( Ok(PrivateReply::PvyHomeNoteSet(hn.clone())) } PrivateRequest::PvqSyncRemote => { - conn.close(); + conn.close().map_err(|e| e.1)?; let conn = sqldata::connection_open( "PvqPowerDelete", false, From dbca3fa1998bca6110284792f3bd2bce3bedaf7d Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 20 Oct 2026 06:41:11 -0600 Subject: [PATCH 32/40] connection_open names --- server-lib/src/interfaces.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index 693a77dd..6d5cf331 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -325,7 +325,6 @@ pub async fn zk_interface_loggedin( } PrivateRequest::PvqPowerDelete(search) => { conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( "PvqPowerDelete", false, @@ -336,9 +335,8 @@ pub async fn zk_interface_loggedin( } PrivateRequest::PvqDeleteZkNote(id) => { conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqPowerDelete", + "PvqDeleteZkNote", false, state.config.orgauth_config.db.as_path(), )?; @@ -347,9 +345,8 @@ pub async fn zk_interface_loggedin( } PrivateRequest::PvqSaveZkNote(sbe) => { conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqPowerDelete", + "PvqSaveZkNote", false, state.config.orgauth_config.db.as_path(), )?; @@ -359,20 +356,23 @@ pub async fn zk_interface_loggedin( } PrivateRequest::PvqSaveZkLinks(msg) => { conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqPowerDelete", + "PvqSaveZkLinks", false, state.config.orgauth_config.db.as_path(), )?; - let _ = sqldata::save_zklinks(&state.config.orgauth_config.db.as_path(), uid, &msg.links)?; + let _ = sqldata::save_zklinks( + &conn, + &state.config.orgauth_config.db.as_path(), + uid, + &msg.links, + )?; Ok(PrivateReply::PvySavedZkLinks) } PrivateRequest::PvqSaveZkNoteAndLinks(sznpl) => { conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqPowerDelete", + "PvqSaveZkNoteAndLinks", false, state.config.orgauth_config.db.as_path(), )?; @@ -385,7 +385,6 @@ pub async fn zk_interface_loggedin( } PrivateRequest::PvqSaveImportZkNotes(gzl) => { conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( "PvqPowerDelete", false, From 72fcb4e7f089d085ae32ba9a472af828002bc5ed Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 20 Oct 2026 06:58:10 -0600 Subject: [PATCH 33/40] notate some errs; save_zklinks conn --- server-lib/src/lib.rs | 2 +- server-lib/src/sqldata.rs | 3 +-- server-lib/src/sync.rs | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index e874667b..da6ce3cf 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -678,7 +678,7 @@ async fn private_upstreaming( match zk_interface_check_upstreaming(&session, &data.config, &li, body).await { Ok(hr) => hr, Err(e) => { - error!("'private' err: {:?}", e); + error!("'private_upstreaming' err: {:?}", e); let se = PrivateReply::PvyServerError(PrivateError::PveString(e.to_string())); HttpResponse::Ok().json(se) } diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index 77153322..6fe547a8 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -1782,12 +1782,11 @@ pub fn delete_zknote( } pub fn save_zklinks( + conn: &Connection, dbfile: &Path, uid: UserId, zklinks: &Vec, ) -> Result<(), zkerr::Error> { - let conn = connection_open("save_zklinks", false, dbfile)?; - for zklink in zklinks.iter() { // TODO: integrate into sql instead of separate queries. let to = note_id_for_zknoteid(&conn, &zklink.to)?; diff --git a/server-lib/src/sync.rs b/server-lib/src/sync.rs index 70a7786d..7e7e7eda 100644 --- a/server-lib/src/sync.rs +++ b/server-lib/src/sync.rs @@ -1483,7 +1483,7 @@ pub async fn sync_to_remote( .cookie(cookie) .timeout(Duration::from_secs(60 * 60)) .send_body(awc::body::BodyStream::new(ss)) - .map_err(|e| zkerr::Error::String(e.to_string())) + .map_err(|e| zkerr::Error::String(format!("sync_to_remote send_body err {}", e.to_string()))) .await?; // TODO: send back Sync results... how many records and etc. From c9f7ea430ec961775f9df7332625a4db65c2fff5 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 20 Oct 2026 07:03:54 -0600 Subject: [PATCH 34/40] save_zklinks conn --- server-lib/src/interfaces.rs | 7 +------ server-lib/src/sqldata.rs | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index 6d5cf331..b0317e6e 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -361,12 +361,7 @@ pub async fn zk_interface_loggedin( false, state.config.orgauth_config.db.as_path(), )?; - let _ = sqldata::save_zklinks( - &conn, - &state.config.orgauth_config.db.as_path(), - uid, - &msg.links, - )?; + let _ = sqldata::save_zklinks(&conn, uid, &msg.links)?; Ok(PrivateReply::PvySavedZkLinks) } PrivateRequest::PvqSaveZkNoteAndLinks(sznpl) => { diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index 6fe547a8..a111fa13 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -1783,7 +1783,6 @@ pub fn delete_zknote( pub fn save_zklinks( conn: &Connection, - dbfile: &Path, uid: UserId, zklinks: &Vec, ) -> Result<(), zkerr::Error> { From 0a1ebdc6a5c2357b409f05d35504387629183bd9 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 20 Oct 2026 07:14:01 -0600 Subject: [PATCH 35/40] replace bool with enum --- server-lib/src/interfaces.rs | 36 ++++++++++++++++++---------------- server-lib/src/lib.rs | 38 ++++++++++++++++++++++++------------ server-lib/src/sqldata.rs | 36 ++++++++++++++++++++-------------- server-lib/src/sync.rs | 8 ++++++-- 4 files changed, 72 insertions(+), 46 deletions(-) diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index b0317e6e..b1b8b400 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -8,6 +8,7 @@ use crate::search; use crate::sqldata; use crate::sqldata::local_server_id; use crate::sqldata::zknotes_callbacks; +use crate::sqldata::ConnFig; use crate::sqldata::LapinInfo; use crate::state::new_jobid; use crate::state::State; @@ -47,7 +48,7 @@ pub fn login_data_for_token( ) -> Result, Box> { let mut conn = sqldata::connection_open( "login_data_for_token", - true, + ConnFig::ReadOnly, config.orgauth_config.db.as_path(), )?; conn.busy_timeout(Duration::from_millis(500))?; @@ -183,7 +184,7 @@ pub async fn zk_interface_loggedin_streaming( let search: ZkNoteSearch = serde_json::from_value(msgdata.clone())?; let conn = Arc::new(sqldata::connection_open( "PrivateStreamingRequests::SearchZkNotes", - true, + ConnFig::ReadOnly, config.orgauth_config.db.as_path(), )?); let znsstream = @@ -195,7 +196,7 @@ pub async fn zk_interface_loggedin_streaming( let rq: GetArchiveZkLinks = serde_json::from_value(msgdata.clone())?; let conn = Arc::new(sqldata::connection_open( "psr:GetArchiveZkLinks", - true, + ConnFig::ReadOnly, config.orgauth_config.db.as_path(), )?); let bstream = sqldata::read_archivezklinks_stream(conn, uid, rq.createddate_after, None) @@ -206,7 +207,7 @@ pub async fn zk_interface_loggedin_streaming( let rq: GetZkLinksSince = serde_json::from_value(msgdata.clone())?; let conn = Arc::new(sqldata::connection_open( "psr:GetZkLinksSince", - true, + ConnFig::ReadOnly, config.orgauth_config.db.as_path(), )?); let bstream = sqldata::read_zklinks_since_stream(conn, uid, rq.createddate_after, None) @@ -216,7 +217,7 @@ pub async fn zk_interface_loggedin_streaming( PrivateStreamingRequests::Sync => { let conn = Arc::new(sqldata::connection_open( "psr:Sync", - false, + ConnFig::ReadWrite, config.orgauth_config.db.as_path(), )?); let rq: SyncSince = serde_json::from_value(msgdata.clone())?; @@ -327,7 +328,7 @@ pub async fn zk_interface_loggedin( conn.close().map_err(|e| e.1)?; let conn = sqldata::connection_open( "PvqPowerDelete", - false, + ConnFig::ReadWrite, state.config.orgauth_config.db.as_path(), )?; let res = search::power_delete_zknotes(&conn, state.config.file_path.clone(), uid, &search)?; @@ -337,7 +338,7 @@ pub async fn zk_interface_loggedin( conn.close().map_err(|e| e.1)?; let conn = sqldata::connection_open( "PvqDeleteZkNote", - false, + ConnFig::ReadWrite, state.config.orgauth_config.db.as_path(), )?; sqldata::delete_zknote(&conn, state.config.file_path.clone(), uid, &id)?; @@ -347,7 +348,7 @@ pub async fn zk_interface_loggedin( conn.close().map_err(|e| e.1)?; let conn = sqldata::connection_open( "PvqSaveZkNote", - false, + ConnFig::ReadWrite, state.config.orgauth_config.db.as_path(), )?; let li = connect_and_make_lapin_info(state, token).await; @@ -358,7 +359,7 @@ pub async fn zk_interface_loggedin( conn.close().map_err(|e| e.1)?; let conn = sqldata::connection_open( "PvqSaveZkLinks", - false, + ConnFig::ReadWrite, state.config.orgauth_config.db.as_path(), )?; let _ = sqldata::save_zklinks(&conn, uid, &msg.links)?; @@ -368,7 +369,7 @@ pub async fn zk_interface_loggedin( conn.close().map_err(|e| e.1)?; let conn = sqldata::connection_open( "PvqSaveZkNoteAndLinks", - false, + ConnFig::ReadWrite, state.config.orgauth_config.db.as_path(), )?; let li = connect_and_make_lapin_info(state, token).await; @@ -382,7 +383,7 @@ pub async fn zk_interface_loggedin( conn.close().map_err(|e| e.1)?; let conn = sqldata::connection_open( "PvqPowerDelete", - false, + ConnFig::ReadWrite, state.config.orgauth_config.db.as_path(), )?; let li = connect_and_make_lapin_info(state, token).await; @@ -393,7 +394,7 @@ pub async fn zk_interface_loggedin( conn.close().map_err(|e| e.1)?; let conn = sqldata::connection_open( "PvqPowerDelete", - false, + ConnFig::ReadWrite, state.config.orgauth_config.db.as_path(), )?; sqldata::set_homenote(&conn, uid, hn)?; @@ -403,7 +404,7 @@ pub async fn zk_interface_loggedin( conn.close().map_err(|e| e.1)?; let conn = sqldata::connection_open( "PvqPowerDelete", - false, + ConnFig::ReadWrite, state.config.orgauth_config.db.as_path(), )?; let dbpath: PathBuf = state.config.orgauth_config.db.to_path_buf(); @@ -526,7 +527,8 @@ pub async fn zk_interface_loggedin( write!(gbm, "starting file sync"); let r = async { - let conn = sqldata::connection_open("startit", false, &dbpath.as_path())?; + let conn = + sqldata::connection_open("startit", ConnFig::ReadWrite, &dbpath.as_path())?; let _dv = sync::sync_files_down( &conn, &lapin_info, @@ -625,7 +627,7 @@ pub fn public_interface( PublicRequest::PbrGetZkNoteAndLinks(gzne) => { let conn = sqldata::connection_open( "PublicRequest::PbrGetZkNoteAndLinks", - true, + ConnFig::ReadOnly, config.orgauth_config.db.as_path(), )?; let (_, note) = sqldata::read_zknote(&conn, &config.file_path, None, &gzne.zknote)?; @@ -649,7 +651,7 @@ pub fn public_interface( ); let conn = sqldata::connection_open( "PublicRequest::PbrGetZknIfChanged", - true, + ConnFig::ReadOnly, config.orgauth_config.db.as_path(), )?; let ozkne = sqldata::read_zneifchanged(&conn, &config.file_path, None, &gzic)?; @@ -662,7 +664,7 @@ pub fn public_interface( PublicRequest::PbrGetZkNotePubId(pubid) => { let conn = sqldata::connection_open( "PublicRequest::PbrGetZkNotePubId", - true, + ConnFig::ReadOnly, config.orgauth_config.db.as_path(), )?; let note = sqldata::read_zknotepubid(&conn, &config.file_path, None, pubid.as_str())?; diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index da6ce3cf..a439c7ff 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -9,7 +9,7 @@ mod sqltest; pub mod state; mod sync; mod synctest; -use crate::{error as zkerr, state::State}; +use crate::{error as zkerr, sqldata::ConnFig, state::State}; use actix_cors::Cors; use actix_files::NamedFile; use actix_multipart::Multipart; @@ -183,7 +183,11 @@ async fn user( // req.connection_info() // ); match async { - let conn = sqldata::connection_open("user", false, data.config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open( + "user", + ConnFig::ReadWrite, + data.config.orgauth_config.db.as_path(), + )?; interfaces::user_interface( &conn, @@ -259,8 +263,11 @@ pub fn get_tauri_uid(conn: &Connection) -> Result, zkerr::Error> } async fn file(session: Session, state: web::Data, req: HttpRequest) -> HttpResponse { - let conn = match sqldata::connection_open("file", true, state.config.orgauth_config.db.as_path()) - { + let conn = match sqldata::connection_open( + "file", + ConnFig::ReadOnly, + state.config.orgauth_config.db.as_path(), + ) { Ok(c) => c, Err(e) => return HttpResponse::InternalServerError().body(format!("{:?}", e)), }; @@ -344,7 +351,7 @@ async fn make_file_notes( ) -> Result> { let conn = sqldata::connection_open( "make_file_notes", - false, + ConnFig::ReadWrite, state.config.orgauth_config.db.as_path(), )?; let uid = if state.config.tauri_mode { @@ -616,7 +623,7 @@ async fn zk_interface_check( ) -> Result { let conn = sqldata::connection_open( "zk_interface_check", - true, + ConnFig::ReadOnly, state.config.orgauth_config.db.as_path(), )?; info!("pre read_user_by_token_api"); @@ -647,7 +654,7 @@ async fn zk_interface_check_streaming( Some(token) => { let conn = sqldata::connection_open( "zk_interface_check_streaming", - true, + ConnFig::ReadOnly, config.orgauth_config.db.as_path(), )?; match orgauth::dbfun::read_user_by_token_api( @@ -702,7 +709,7 @@ async fn zk_interface_check_upstreaming( Some(token) => { let conn = sqldata::connection_open( "zk_interface_check_upstreaming", - true, + ConnFig::ReadOnly, config.orgauth_config.db.as_path(), )?; @@ -937,7 +944,7 @@ pub async fn err_main( if let Some(uid) = matches.value_of("promote_to_admin") { let conn = sqldata::connection_open( "promote to admin", - false, + ConnFig::ReadWrite, config.orgauth_config.db.as_path(), )?; let mut user = orgauth::dbfun::read_user_by_name(&conn, uid)?; @@ -956,8 +963,11 @@ pub async fn err_main( stdin().read_line(&mut pwd)?; let mut cb = sqldata::zknotes_callbacks(); - let conn = - sqldata::connection_open("creating admin", false, config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open( + "creating admin", + ConnFig::ReadWrite, + config.orgauth_config.db.as_path(), + )?; // make new registration i let rd = orgauth::data::RegistrationData { uid: username.to_string(), @@ -1048,7 +1058,11 @@ pub async fn init_server(mut config: Config) -> Result> { }; { - let conn = sqldata::connection_open("init_server", false, config.orgauth_config.db.as_path())?; + let conn = sqldata::connection_open( + "init_server", + ConnFig::ReadWrite, + config.orgauth_config.db.as_path(), + )?; let tr = conn.unchecked_transaction()?; sqldata::update_filetable(&conn, &config.file_path)?; diff --git a/server-lib/src/sqldata.rs b/server-lib/src/sqldata.rs index a111fa13..40a5bebd 100644 --- a/server-lib/src/sqldata.rs +++ b/server-lib/src/sqldata.rs @@ -175,24 +175,30 @@ fn sqlog(meh: &str) { const SQLITE_TRACING: bool = false; +pub enum ConnFig { + ReadWrite, + ReadOnly, +} + pub fn connection_open( _connid: &str, - read_only: bool, + conn_fig: ConnFig, dbfile: &Path, ) -> Result { - let mut conn = if read_only { - let c = Connection::open_with_flags( - dbfile, - OpenFlags::SQLITE_OPEN_READ_ONLY - | OpenFlags::SQLITE_OPEN_NO_MUTEX - | OpenFlags::SQLITE_OPEN_URI, // OpenFlags::SQLITE_OPEN_READ_ONLY - // | OpenFlags::SQLITE_OPEN_CREATE - // | OpenFlags::SQLITE_OPEN_URI - // | OpenFlags::SQLITE_OPEN_NO_MUTEX, - )?; - c - } else { - Connection::open(dbfile)? + let mut conn = match conn_fig { + ConnFig::ReadOnly => { + let c = Connection::open_with_flags( + dbfile, + OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_NO_MUTEX + | OpenFlags::SQLITE_OPEN_URI, // OpenFlags::SQLITE_OPEN_READ_ONLY + // | OpenFlags::SQLITE_OPEN_CREATE + // | OpenFlags::SQLITE_OPEN_URI + // | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + c + } + ConnFig::ReadWrite => Connection::open(dbfile)?, }; // conn.busy_timeout(Duration::from_millis(500))?; @@ -268,7 +274,7 @@ pub fn update_single_value( pub fn dbinit(dbfile: &Path, token_expiration_ms: Option) -> Result { let exists = dbfile.exists(); - let conn = connection_open("dbinit", false, dbfile)?; + let conn = connection_open("dbinit", ConnFig::ReadWrite, dbfile)?; if !exists { info!("initialdb"); diff --git a/server-lib/src/sync.rs b/server-lib/src/sync.rs index 7e7e7eda..e703e607 100644 --- a/server-lib/src/sync.rs +++ b/server-lib/src/sync.rs @@ -4,7 +4,7 @@ use crate::search::build_sql; use crate::search::{search_zknotes, search_zknotes_stream, sync_users, system_user, SearchResult}; use crate::sqldata::{ self, local_server_id, note_id_for_uuid, save_zklink, save_zknote, server_id, user_note_id, - LapinInfo, NoteDates, + ConnFig, LapinInfo, NoteDates, }; use crate::util::now; use actix_multipart_rfc7578 as multipart; @@ -267,7 +267,11 @@ pub async fn sync( callbacks: &mut Callbacks, monitor: &dyn JobMonitor, ) -> Result> { - let conn = Arc::new(sqldata::connection_open("sync", false, dbpath)?); + let conn = Arc::new(sqldata::connection_open( + "sync", + ConnFig::ReadWrite, + dbpath, + )?); let user = orgauth::dbfun::read_user_by_id(&conn, uid)?; // TODO pass this in from calling ftn? let extra_login_data = sqldata::read_extra_login_data(&conn, user.id)?; // let mut callbacks = &mut zknotes_callbacks(); From eec9f8b28e3720758b25aea1c3e54158cf04ac85 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 20 Oct 2026 07:25:52 -0600 Subject: [PATCH 36/40] comment --- server-lib/src/interfaces.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index b1b8b400..7073d2fe 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -249,7 +249,7 @@ pub async fn zk_interface_loggedin_streaming( pub async fn zk_interface_loggedin( state: &State, - conn: Connection, + conn: Connection, // assumed to be ConnFig::ReadOnly token: &Option, uid: UserId, msg: &PrivateRequest, From 056002e1173fd71056572711372d9ab0fea5d32c Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 20 Oct 2026 10:34:03 -0600 Subject: [PATCH 37/40] check for empty stream error --- server-lib/src/sync.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/server-lib/src/sync.rs b/server-lib/src/sync.rs index e703e607..12756b07 100644 --- a/server-lib/src/sync.rs +++ b/server-lib/src/sync.rs @@ -674,9 +674,13 @@ pub async fn sync_files_up( let mut line = String::new(); - let mut sm = read_sync_message(&mut line, &mut sr) - .map_err(|e| zkerr::Error::String(format!("read sync error: {:?}", e))) - .await?; + let mut resvec = Vec::new(); + + let mut sm = match read_sync_message(&mut line, &mut sr).await { + Err(zkerr::Error::EmptyStream) => return Ok(resvec), + Err(e) => return Err(zkerr::Error::String(format!("read sync error: {:?}", e))), + Ok(sm) => sm, + }; if let SyncMessage::ZkSearchResultHeader(ref _zsrh) = sm { } else { @@ -689,11 +693,12 @@ pub async fn sync_files_up( ); } - sm = read_sync_message(&mut line, &mut sr) - .map_err(|e| zkerr::Error::String(format!("read sync error: {:?}", e))) - .await?; + sm = match read_sync_message(&mut line, &mut sr).await { + Err(zkerr::Error::EmptyStream) => return Ok(resvec), + Err(e) => return Err(zkerr::Error::String(format!("read sync error: {:?}", e))), + Ok(sm) => sm, + }; - let mut resvec = Vec::new(); while let SyncMessage::ZkNoteId(ref id) = sm { let nid = note_id_for_uuid(&conn, &Uuid::from_str(id.as_str())?)?; From b86be52e0811b40a245df43bff9787ec71cebd50 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 20 Oct 2026 10:34:24 -0600 Subject: [PATCH 38/40] websockets back on --- server/static/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/static/index.html b/server/static/index.html index 70c29e88..cb5fd680 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -68,7 +68,7 @@ login : login, tauri : false, mobile : false, - websockets : false, + websockets : true, adminsettings : adminsettings} } ); From 8f44e36ae980aaaef7e08f8d40f3cf35e53eeb68 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 20 Oct 2026 10:34:48 -0600 Subject: [PATCH 39/40] back to readwrite connections because of login --- server-lib/src/interfaces.rs | 2 +- server-lib/src/lib.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index 7073d2fe..6a358128 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -48,7 +48,7 @@ pub fn login_data_for_token( ) -> Result, Box> { let mut conn = sqldata::connection_open( "login_data_for_token", - ConnFig::ReadOnly, + ConnFig::ReadWrite, config.orgauth_config.db.as_path(), )?; conn.busy_timeout(Duration::from_millis(500))?; diff --git a/server-lib/src/lib.rs b/server-lib/src/lib.rs index a439c7ff..3f890fbd 100644 --- a/server-lib/src/lib.rs +++ b/server-lib/src/lib.rs @@ -228,7 +228,7 @@ async fn admin( ) { Ok(sr) => HttpResponse::Ok().json(sr), Err(e) => { - error!("'user' err: {:?}", e); + error!("'admin' err: {:?}", e); let se = AdminResponse::ArpServerError(e.to_string()); HttpResponse::Ok().json(se) } @@ -623,7 +623,7 @@ async fn zk_interface_check( ) -> Result { let conn = sqldata::connection_open( "zk_interface_check", - ConnFig::ReadOnly, + ConnFig::ReadWrite, state.config.orgauth_config.db.as_path(), )?; info!("pre read_user_by_token_api"); @@ -654,7 +654,7 @@ async fn zk_interface_check_streaming( Some(token) => { let conn = sqldata::connection_open( "zk_interface_check_streaming", - ConnFig::ReadOnly, + ConnFig::ReadWrite, config.orgauth_config.db.as_path(), )?; match orgauth::dbfun::read_user_by_token_api( @@ -709,7 +709,7 @@ async fn zk_interface_check_upstreaming( Some(token) => { let conn = sqldata::connection_open( "zk_interface_check_upstreaming", - ConnFig::ReadOnly, + ConnFig::ReadWrite, config.orgauth_config.db.as_path(), )?; From 931e212dee20dd6662d55e39d6cffbc4db768156 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 21 Oct 2026 09:17:31 -0600 Subject: [PATCH 40/40] don't reopen conns as ReadWrite since already ReadWrite --- orgauth | 2 +- server-lib/src/interfaces.rs | 100 +++++++++++++++++------------------ 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/orgauth b/orgauth index 78cd395c..2492c7d0 160000 --- a/orgauth +++ b/orgauth @@ -1 +1 @@ -Subproject commit 78cd395cf3cb45e15ba1bdcfd70eac036abe67dd +Subproject commit 2492c7d05dba082dbdfc047adfe0ac2a2db9f498 diff --git a/server-lib/src/interfaces.rs b/server-lib/src/interfaces.rs index 6a358128..dbf81785 100644 --- a/server-lib/src/interfaces.rs +++ b/server-lib/src/interfaces.rs @@ -249,7 +249,7 @@ pub async fn zk_interface_loggedin_streaming( pub async fn zk_interface_loggedin( state: &State, - conn: Connection, // assumed to be ConnFig::ReadOnly + conn: Connection, token: &Option, uid: UserId, msg: &PrivateRequest, @@ -325,53 +325,53 @@ pub async fn zk_interface_loggedin( } } PrivateRequest::PvqPowerDelete(search) => { - conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqPowerDelete", - ConnFig::ReadWrite, - state.config.orgauth_config.db.as_path(), - )?; + // conn.close().map_err(|e| e.1)?; + // let conn = sqldata::connection_open( + // "PvqPowerDelete", + // ConnFig::ReadWrite, + // state.config.orgauth_config.db.as_path(), + // )?; let res = search::power_delete_zknotes(&conn, state.config.file_path.clone(), uid, &search)?; Ok(PrivateReply::PvyPowerDeleteComplete(res)) } PrivateRequest::PvqDeleteZkNote(id) => { - conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqDeleteZkNote", - ConnFig::ReadWrite, - state.config.orgauth_config.db.as_path(), - )?; + // conn.close().map_err(|e| e.1)?; + // let conn = sqldata::connection_open( + // "PvqDeleteZkNote", + // ConnFig::ReadWrite, + // state.config.orgauth_config.db.as_path(), + // )?; sqldata::delete_zknote(&conn, state.config.file_path.clone(), uid, &id)?; Ok(PrivateReply::PvyDeletedZkNote(id.clone())) } PrivateRequest::PvqSaveZkNote(sbe) => { - conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqSaveZkNote", - ConnFig::ReadWrite, - state.config.orgauth_config.db.as_path(), - )?; + // conn.close().map_err(|e| e.1)?; + // let conn = sqldata::connection_open( + // "PvqSaveZkNote", + // ConnFig::ReadWrite, + // state.config.orgauth_config.db.as_path(), + // )?; let li = connect_and_make_lapin_info(state, token).await; let (_id, s) = sqldata::save_zknote(&conn, &li, &state.server, uid, &sbe, None).await?; Ok(PrivateReply::PvySavedZkNote(s)) } PrivateRequest::PvqSaveZkLinks(msg) => { - conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqSaveZkLinks", - ConnFig::ReadWrite, - state.config.orgauth_config.db.as_path(), - )?; + // conn.close().map_err(|e| e.1)?; + // let conn = sqldata::connection_open( + // "PvqSaveZkLinks", + // ConnFig::ReadWrite, + // state.config.orgauth_config.db.as_path(), + // )?; let _ = sqldata::save_zklinks(&conn, uid, &msg.links)?; Ok(PrivateReply::PvySavedZkLinks) } PrivateRequest::PvqSaveZkNoteAndLinks(sznpl) => { - conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqSaveZkNoteAndLinks", - ConnFig::ReadWrite, - state.config.orgauth_config.db.as_path(), - )?; + // conn.close().map_err(|e| e.1)?; + // let conn = sqldata::connection_open( + // "PvqSaveZkNoteAndLinks", + // ConnFig::ReadWrite, + // state.config.orgauth_config.db.as_path(), + // )?; let li = connect_and_make_lapin_info(state, token).await; let (_, szkn) = sqldata::save_zknote(&conn, &li, &state.server, uid, &sznpl.note, None).await?; @@ -380,33 +380,33 @@ pub async fn zk_interface_loggedin( Ok(PrivateReply::PvySavedZkNoteAndLinks(szkn)) } PrivateRequest::PvqSaveImportZkNotes(gzl) => { - conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqPowerDelete", - ConnFig::ReadWrite, - state.config.orgauth_config.db.as_path(), - )?; + // conn.close().map_err(|e| e.1)?; + // let conn = sqldata::connection_open( + // "PvqPowerDelete", + // ConnFig::ReadWrite, + // state.config.orgauth_config.db.as_path(), + // )?; let li = connect_and_make_lapin_info(state, token).await; sqldata::save_importzknotes(&conn, &li, &state.server, uid, gzl).await?; Ok(PrivateReply::PvySavedImportZkNotes) } PrivateRequest::PvqSetHomeNote(hn) => { - conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqPowerDelete", - ConnFig::ReadWrite, - state.config.orgauth_config.db.as_path(), - )?; + // conn.close().map_err(|e| e.1)?; + // let conn = sqldata::connection_open( + // "PvqPowerDelete", + // ConnFig::ReadWrite, + // state.config.orgauth_config.db.as_path(), + // )?; sqldata::set_homenote(&conn, uid, hn)?; Ok(PrivateReply::PvyHomeNoteSet(hn.clone())) } PrivateRequest::PvqSyncRemote => { - conn.close().map_err(|e| e.1)?; - let conn = sqldata::connection_open( - "PvqPowerDelete", - ConnFig::ReadWrite, - state.config.orgauth_config.db.as_path(), - )?; + // conn.close().map_err(|e| e.1)?; + // let conn = sqldata::connection_open( + // "PvqPowerDelete", + // ConnFig::ReadWrite, + // state.config.orgauth_config.db.as_path(), + // )?; let dbpath: PathBuf = state.config.orgauth_config.db.to_path_buf(); let file_path: PathBuf = state.config.file_path.to_path_buf(); let uid: UserId = uid; @@ -417,7 +417,7 @@ pub async fn zk_interface_loggedin( let lapin_channelx = li.map(|li| li.channel).clone(); let token = token.clone(); - // not really necessary. trying to ensure no sqlite blocking. + // trying to reduce sqlite blocking. conn.close().map_err(|(_conn, err)| err)?; std::thread::spawn(move || {