diff --git a/desktop/AGENTS.md b/desktop/AGENTS.md index 02f2fe803..a8fc263a6 100644 --- a/desktop/AGENTS.md +++ b/desktop/AGENTS.md @@ -1,6 +1,6 @@ # ODE Desktop — AI and developer guide -**ODE Desktop** (`desktop/`) is the Tauri + React + Rust app for **Data management** (observations, sync, import) and the **Forms / app workbench** (bundles, form preview, custom app embed). User-facing overview: [desktop/README.md](README.md). +**ODE Desktop** (`desktop/`) is the Tauri + React + Rust app for **Data management** (observations, sync, import, **export**) and the **Forms / app workbench** (bundles, form preview, custom app embed). User-facing overview: [desktop/README.md](README.md). Published docs: [ODE Desktop developer mode](https://opendataensemble.org/docs/guides/ode-desktop-developer-mode) (local custom app iteration). diff --git a/desktop/README.md b/desktop/README.md index 47673495d..258792e4b 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -12,10 +12,23 @@ The internal Rust crate may still be named `custodian`; user-facing strings use ## Who it is for - **Field and data staff** managing observations and sync (Data management). +- **Analysts** exporting local Parquet from Data → **Export** for offline analysis (R, Python, etc.). - **Form and app authors** testing bundles and custom apps against Synkronus before mobile deploy (Workbench). Relationship to other ODE components: **Synkronus** (API), **Formulus** + **formplayer** (runtime parity), **Portal** and **CLI** (same public API — no privileged desktop channel). Long-form prose here is intended to be **copy-friendly** for [opendataensemble.org](https://opendataensemble.org/) documentation; keep user-facing sections free of repo-only trivia (put contributor notes under **Development**). +## Data management — Export + +From **Data → Export**, ODE Desktop writes analyst-oriented Parquet from the **active profile’s local workspace** (not from Synkronus over the network): + +1. Choose a destination parent folder (last location is remembered for the session, like Import). +2. Desktop creates a leaf folder named **`YYYYMMDD`** under that parent (overwrite confirmation if it already exists). +3. One **`.parquet`** per form type, plus **`export_manifest.json`**, and **`snippets/`** load scripts (R, Python, Stata, Julia). +4. Optional **Include pending observations** (local rows not yet synced). +5. Optional **Include attachments** — copies referenced files into `attachments/` under the export folder. Observation data stores attachment **basenames**; prefix them with the workspace attachments path shown on Profiles / Export (or the export `attachments/` path when copied) to open files on disk. + +The Export page also shows copyable load snippets with variables named after each form type. Transforms and study-specific ETL stay **outside** Desktop. For a full **server** dump as a ZIP, use Portal or `synk data export`. + ## Install Official builds are produced with **`pnpm tauri build`** (artifacts vary by OS: `.msi`, `.dmg`, `.app`, AppImage, `.deb`, etc.). When releases are published, attach those bundles to **GitHub Releases** and link them from project docs. diff --git a/desktop/docs/UI_FEEDBACK.md b/desktop/docs/UI_FEEDBACK.md index 69ef43ad9..d18330a53 100644 --- a/desktop/docs/UI_FEEDBACK.md +++ b/desktop/docs/UI_FEEDBACK.md @@ -4,7 +4,7 @@ Single source of truth for how the shell surfaces status, errors, and confirmati ## Progress banner (full-width) -**Use for:** long-running sync, import, and **app bundle download/apply** jobs. +**Use for:** long-running sync, import, **export**, and **app bundle download/apply** jobs. - Renders below the mode switch / dev bar in `Shell`. - Shows spinner + status text; optional determinate progress bar when `bundleActivity.total > 0`. diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index d089e5997..7ce2189c6 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -15,6 +15,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -68,6 +70,180 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arrow" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.23.1", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + +[[package]] +name = "arrow-ord" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" + +[[package]] +name = "arrow-select" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -222,6 +398,15 @@ dependencies = [ "system-deps", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -246,6 +431,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bit-set" version = "0.8.0" @@ -485,6 +676,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + [[package]] name = "combine" version = "4.6.7" @@ -504,6 +705,26 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -622,6 +843,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -1073,6 +1300,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.11.0", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.9" @@ -1582,6 +1819,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1612,6 +1861,12 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hashlink" version = "0.9.1" @@ -2156,6 +2411,63 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libappindicator" version = "0.9.0" @@ -2205,6 +2517,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.15" @@ -2438,12 +2756,40 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2451,6 +2797,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2602,9 +2949,11 @@ dependencies = [ name = "odedesktop" version = "1.2.1" dependencies = [ + "arrow", "chrono", "futures-util", "keyring", + "parquet", "rayon", "reqwest 0.12.28", "rusqlite", @@ -2755,6 +3104,32 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "parquet" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.23.1", + "bytes", + "chrono", + "half", + "hashbrown 0.17.1", + "num-bigint", + "num-integer", + "num-traits", + "seq-macro", + "snap", + "twox-hash", +] + [[package]] name = "pathdiff" version = "0.2.3" @@ -2797,6 +3172,15 @@ dependencies = [ "phf_shared 0.11.3", ] +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", +] + [[package]] name = "phf" version = "0.13.1" @@ -2945,6 +3329,15 @@ dependencies = [ "siphasher 1.0.2", ] +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher 1.0.2", +] + [[package]] name = "phf_shared" version = "0.13.1" @@ -3790,6 +4183,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.228" @@ -4026,6 +4425,12 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + [[package]] name = "socket2" version = "0.6.3" @@ -4692,6 +5097,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -4985,6 +5399,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + [[package]] name = "typeid" version = "1.0.3" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index e622ea386..d21d75122 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -36,4 +36,6 @@ walkdir = "2" url = "2" urlencoding = "2" tokio = { version = "1", features = ["sync", "time", "macros"] } +arrow = { version = "59", default-features = false, features = ["chrono-tz"] } +parquet = { version = "59", default-features = false, features = ["arrow", "snap"] } diff --git a/desktop/src-tauri/capabilities/default.json b/desktop/src-tauri/capabilities/default.json index 30932472f..de8ba96e8 100644 --- a/desktop/src-tauri/capabilities/default.json +++ b/desktop/src-tauri/capabilities/default.json @@ -6,7 +6,11 @@ "permissions": [ "core:default", "opener:default", - "opener:allow-open-path", + { + "identifier": "opener:allow-open-path", + "allow": [{ "path": "**" }, { "path": "/**" }] + }, + "opener:allow-reveal-item-in-dir", "dialog:default" ] } diff --git a/desktop/src-tauri/src/data_export.rs b/desktop/src-tauri/src/data_export.rs new file mode 100644 index 000000000..717453b75 --- /dev/null +++ b/desktop/src-tauri/src/data_export.rs @@ -0,0 +1,1054 @@ +//! Local Parquet export from the profile workspace SQLite. +//! +//! Produces analyst-compatible files (envelope + top-level `data_*` columns), +//! not bit-for-bit parity with Synkronus server export. + +use std::{ + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, + fs::{self, File}, + path::{Path, PathBuf}, + sync::Arc, +}; + +use arrow::array::{ArrayRef, BooleanBuilder, Float64Builder, Int64Builder, StringBuilder}; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use chrono::Utc; +use parquet::arrow::ArrowWriter; +use rusqlite::Connection; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{CustodianError, ObservationExtras, resolve_attachment_path}; + +const EXPORTER_VERSION: &str = env!("CARGO_PKG_VERSION"); + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportParquetRequest { + /// Parent directory chosen by the user; leaf `YYYYMMDD` is created underneath. + pub parent_dir: String, + #[serde(default)] + pub include_pending: bool, + #[serde(default)] + pub include_attachments: bool, + /// When true, replace an existing `YYYYMMDD` leaf folder. + #[serde(default)] + pub overwrite: bool, + #[serde(default)] + pub profile_label: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportParquetResult { + pub export_dir: String, + pub form_type_counts: BTreeMap, + /// Absolute paths keyed by form type (same stem as the `.parquet` file when possible). + pub parquet_files: BTreeMap, + pub total_rows: usize, + pub attachments_copied: usize, + pub attachments_missing: usize, + pub include_pending: bool, + pub include_attachments: bool, + pub workspace_attachments_path: String, + pub export_attachments_path: Option, + pub manifest_path: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct ExportManifest { + exported_at: String, + exporter_version: String, + include_pending: bool, + include_attachments: bool, + profile_label: Option, + workspace_attachments_path: String, + export_attachments_path: Option, + form_type_counts: BTreeMap, + total_rows: usize, + attachments_copied: usize, + attachments_missing: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ColKind { + Float64, + Boolean, + String, +} + +#[derive(Debug, Clone)] +pub(crate) struct ExportRow { + observation_id: String, + form_type: String, + form_version: String, + created_at: String, + updated_at: String, + synced_at: Option, + deleted: bool, + version: i64, + geolocation: Option, + author: Option, + device_id: Option, + tags: Option, + pending: bool, + data: BTreeMap, +} + +/// Sanitize a form type for use as a filename stem. +pub fn sanitize_filename(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + for ch in name.chars() { + if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' || ch == '.' { + out.push(ch); + } else { + out.push('_'); + } + } + if out.is_empty() { + "unknown".to_string() + } else { + out + } +} + +/// Identifier safe for R / Python / Julia / Stata frame names. +pub fn sanitize_identifier(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + for ch in name.chars() { + if ch.is_ascii_alphanumeric() || ch == '_' { + out.push(ch); + } else { + out.push('_'); + } + } + while out.contains("__") { + out = out.replace("__", "_"); + } + let out = out.trim_matches('_').to_string(); + if out.is_empty() { + "form".to_string() + } else if out.chars().next().is_some_and(|c| c.is_ascii_digit()) { + format!("f_{out}") + } else { + out + } +} + +/// Progress callback: (done, total, message). +pub type ExportProgressFn<'a> = dyn FnMut(usize, usize, &str) + 'a; + +/// Today's export leaf folder name (`YYYYMMDD`). +pub fn export_leaf_folder_name() -> String { + Utc::now().format("%Y%m%d").to_string() +} + +fn value_as_object_map(payload: &Value) -> BTreeMap { + match payload { + Value::Object(map) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + _ => BTreeMap::new(), + } +} + +fn json_value_to_export_cell(v: &Value) -> Value { + match v { + Value::Null => Value::Null, + Value::Bool(_) | Value::Number(_) | Value::String(_) => v.clone(), + other => Value::String(other.to_string()), + } +} + +fn infer_col_kind(values: &[&Value]) -> ColKind { + let mut saw_bool = false; + let mut saw_number = false; + let mut saw_other = false; + for v in values { + match v { + Value::Null => {} + Value::Bool(_) => saw_bool = true, + Value::Number(_) => saw_number = true, + Value::String(_) | Value::Array(_) | Value::Object(_) => saw_other = true, + } + } + if saw_other { + ColKind::String + } else if saw_bool && !saw_number { + ColKind::Boolean + } else if saw_number && !saw_bool { + ColKind::Float64 + } else { + // Mixed bool+number, or all null → string for analyst-safe parquet. + ColKind::String + } +} + +fn extras_deleted(extras: &Option) -> bool { + extras.as_ref().and_then(|e| e.deleted).unwrap_or(false) +} + +struct DbExportCandidate { + id: String, + payload: Value, + form_type: Option, + updated_at: Option, + dirty: bool, + sync_status: String, + last_saved_at: String, + extras: Option, +} + +fn row_from_db(c: DbExportCandidate) -> Option { + if c.sync_status == "conflict" { + return None; + } + if extras_deleted(&c.extras) { + return None; + } + let form_type = c + .form_type + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "unknown".to_string()); + let pending = c.dirty && c.sync_status == "dirty"; + let created_at = c + .extras + .as_ref() + .and_then(|e| e.created_at.clone()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| c.last_saved_at.clone()); + let updated = c + .updated_at + .filter(|s| !s.is_empty()) + .unwrap_or(c.last_saved_at); + let form_version = c + .extras + .as_ref() + .and_then(|e| e.form_version.clone()) + .unwrap_or_default(); + let synced_at = c.extras.as_ref().and_then(|e| e.synced_at.clone()); + let geolocation = c.extras.as_ref().and_then(|e| { + e.geolocation + .as_ref() + .map(|g| g.to_string()) + .filter(|s| s != "null") + }); + let author = c.extras.as_ref().and_then(|e| e.author.clone()); + let device_id = c.extras.as_ref().and_then(|e| e.device_id.clone()); + let tags = c + .extras + .as_ref() + .and_then(|e| e.tags.as_ref().and_then(|t| serde_json::to_string(t).ok())); + let data = value_as_object_map(&c.payload) + .into_iter() + .map(|(k, v)| (k, json_value_to_export_cell(&v))) + .collect(); + Some(ExportRow { + observation_id: c.id, + form_type, + form_version, + created_at, + updated_at: updated, + synced_at, + deleted: false, + version: 0, + geolocation, + author, + device_id, + tags, + pending, + data, + }) +} + +pub(crate) fn load_export_rows( + conn: &Connection, + include_pending: bool, +) -> Result, CustodianError> { + let sql = if include_pending { + "SELECT id, payload, form_type, updated_at, dirty, sync_status, last_saved_at, observation_extras + FROM observations + WHERE sync_status != 'conflict' + ORDER BY COALESCE(form_type, ''), id" + } else { + "SELECT id, payload, form_type, updated_at, dirty, sync_status, last_saved_at, observation_extras + FROM observations + WHERE sync_status != 'conflict' AND dirty = 0 + ORDER BY COALESCE(form_type, ''), id" + }; + let mut stmt = conn.prepare(sql)?; + let rows = stmt.query_map([], |row| { + let payload_raw: String = row.get(1)?; + let payload = serde_json::from_str::(&payload_raw).unwrap_or(Value::Null); + let dirty: i64 = row.get(4)?; + let sync_status: String = row.get(5)?; + let extras_raw: Option = row.get(7)?; + let extras = extras_raw.and_then(|s| { + let t = s.trim(); + if t.is_empty() { + None + } else { + serde_json::from_str(t).ok() + } + }); + Ok(( + row.get::<_, String>(0)?, + payload, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + dirty == 1, + sync_status, + row.get::<_, String>(6)?, + extras, + )) + })?; + + let mut out = Vec::new(); + for row in rows { + let (id, payload, form_type, updated_at, dirty, sync_status, last_saved_at, extras) = row?; + if let Some(r) = row_from_db(DbExportCandidate { + id, + payload, + form_type, + updated_at, + dirty, + sync_status, + last_saved_at, + extras, + }) { + out.push(r); + } + } + Ok(out) +} + +fn schema_for_rows(rows: &[ExportRow]) -> (Schema, Vec<(String, ColKind)>) { + let mut key_values: BTreeMap> = BTreeMap::new(); + for row in rows { + for (k, v) in &row.data { + key_values.entry(k.clone()).or_default().push(v); + } + } + let data_cols: Vec<(String, ColKind)> = key_values + .iter() + .map(|(k, vals)| (k.clone(), infer_col_kind(vals))) + .collect(); + + let mut fields = vec![ + Field::new("observation_id", DataType::Utf8, false), + Field::new("form_type", DataType::Utf8, false), + Field::new("form_version", DataType::Utf8, false), + Field::new("created_at", DataType::Utf8, false), + Field::new("updated_at", DataType::Utf8, false), + Field::new("synced_at", DataType::Utf8, true), + Field::new("deleted", DataType::Boolean, false), + Field::new("version", DataType::Int64, false), + Field::new("geolocation", DataType::Utf8, true), + Field::new("author", DataType::Utf8, true), + Field::new("device_id", DataType::Utf8, true), + Field::new("tags", DataType::Utf8, true), + Field::new("pending", DataType::Boolean, false), + ]; + for (key, kind) in &data_cols { + let dt = match kind { + ColKind::Float64 => DataType::Float64, + ColKind::Boolean => DataType::Boolean, + ColKind::String => DataType::Utf8, + }; + fields.push(Field::new(format!("data_{key}"), dt, true)); + } + (Schema::new(fields), data_cols) +} + +fn append_optional_string(builder: &mut StringBuilder, value: &Option) { + match value { + Some(s) => builder.append_value(s), + None => builder.append_null(), + } +} + +fn cell_as_f64(v: &Value) -> Option { + match v { + Value::Number(n) => n.as_f64(), + Value::String(s) => s.parse().ok(), + _ => None, + } +} + +fn cell_as_bool(v: &Value) -> Option { + match v { + Value::Bool(b) => Some(*b), + Value::String(s) => match s.to_ascii_lowercase().as_str() { + "true" | "1" | "yes" => Some(true), + "false" | "0" | "no" => Some(false), + _ => None, + }, + _ => None, + } +} + +fn cell_as_string(v: &Value) -> String { + match v { + Value::Null => String::new(), + Value::String(s) => s.clone(), + Value::Bool(b) => b.to_string(), + Value::Number(n) => n.to_string(), + other => other.to_string(), + } +} + +fn build_record_batch( + rows: &[ExportRow], + schema: &Schema, + data_cols: &[(String, ColKind)], +) -> Result { + let n = rows.len(); + let mut obs_id = StringBuilder::with_capacity(n, n * 36); + let mut form_type = StringBuilder::with_capacity(n, n * 16); + let mut form_version = StringBuilder::with_capacity(n, n * 8); + let mut created_at = StringBuilder::with_capacity(n, n * 24); + let mut updated_at = StringBuilder::with_capacity(n, n * 24); + let mut synced_at = StringBuilder::with_capacity(n, n * 24); + let mut deleted = BooleanBuilder::with_capacity(n); + let mut version = Int64Builder::with_capacity(n); + let mut geolocation = StringBuilder::with_capacity(n, n * 32); + let mut author = StringBuilder::with_capacity(n, n * 16); + let mut device_id = StringBuilder::with_capacity(n, n * 16); + let mut tags = StringBuilder::with_capacity(n, n * 16); + let mut pending = BooleanBuilder::with_capacity(n); + + for row in rows { + obs_id.append_value(&row.observation_id); + form_type.append_value(&row.form_type); + form_version.append_value(&row.form_version); + created_at.append_value(&row.created_at); + updated_at.append_value(&row.updated_at); + append_optional_string(&mut synced_at, &row.synced_at); + deleted.append_value(row.deleted); + version.append_value(row.version); + append_optional_string(&mut geolocation, &row.geolocation); + append_optional_string(&mut author, &row.author); + append_optional_string(&mut device_id, &row.device_id); + append_optional_string(&mut tags, &row.tags); + pending.append_value(row.pending); + } + + let mut columns: Vec = vec![ + Arc::new(obs_id.finish()), + Arc::new(form_type.finish()), + Arc::new(form_version.finish()), + Arc::new(created_at.finish()), + Arc::new(updated_at.finish()), + Arc::new(synced_at.finish()), + Arc::new(deleted.finish()), + Arc::new(version.finish()), + Arc::new(geolocation.finish()), + Arc::new(author.finish()), + Arc::new(device_id.finish()), + Arc::new(tags.finish()), + Arc::new(pending.finish()), + ]; + + for (key, kind) in data_cols { + match kind { + ColKind::Float64 => { + let mut b = Float64Builder::with_capacity(n); + for row in rows { + match row.data.get(key) { + None | Some(Value::Null) => b.append_null(), + Some(v) => match cell_as_f64(v) { + Some(f) => b.append_value(f), + None => b.append_null(), + }, + } + } + columns.push(Arc::new(b.finish())); + } + ColKind::Boolean => { + let mut b = BooleanBuilder::with_capacity(n); + for row in rows { + match row.data.get(key) { + None | Some(Value::Null) => b.append_null(), + Some(v) => match cell_as_bool(v) { + Some(x) => b.append_value(x), + None => b.append_null(), + }, + } + } + columns.push(Arc::new(b.finish())); + } + ColKind::String => { + let mut b = StringBuilder::with_capacity(n, n * 16); + for row in rows { + match row.data.get(key) { + None | Some(Value::Null) => b.append_null(), + Some(v) => b.append_value(cell_as_string(v)), + } + } + columns.push(Arc::new(b.finish())); + } + } + } + + RecordBatch::try_new(Arc::new(schema.clone()), columns) + .map_err(|e| CustodianError::Message(format!("arrow record batch: {e}"))) +} + +fn write_parquet_file(path: &Path, batch: &RecordBatch) -> Result<(), CustodianError> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let file = File::create(path)?; + let mut writer = ArrowWriter::try_new(file, batch.schema(), None) + .map_err(|e| CustodianError::Message(format!("parquet writer: {e}")))?; + writer + .write(batch) + .map_err(|e| CustodianError::Message(format!("parquet write: {e}")))?; + writer + .close() + .map_err(|e| CustodianError::Message(format!("parquet close: {e}")))?; + Ok(()) +} + +/// Heuristic attachment basename extraction (mirrors Desktop import heuristic). +pub fn collect_attachment_basenames(value: &Value) -> HashSet { + let mut names = HashSet::new(); + walk_attachment_refs(value, 0, &mut names); + names +} + +fn basename_only(s: &str) -> String { + let t = s.trim().replace('\\', "/"); + t.rsplit('/').next().unwrap_or(&t).to_string() +} + +fn string_looks_like_attachment_ref(s: &str) -> bool { + let t = s.trim(); + if t.is_empty() || t.len() > 512 { + return false; + } + let b = basename_only(t); + if b.is_empty() || b.contains("..") { + return false; + } + let lower = b.to_ascii_lowercase(); + const EXTS: &[&str] = &[ + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".heic", ".bmp", ".tif", ".tiff", ".pdf", ".mp3", + ".wav", ".m4a", ".aac", ".ogg", ".mp4", ".mov", ".webm", ".avi", ".mkv", ".svg", ".bin", + ".dat", ".zip", + ]; + if EXTS.iter().any(|e| lower.ends_with(e)) { + return true; + } + // UUID-like attachment ids without extension + if b.len() >= 32 + && b.chars() + .all(|c| c.is_ascii_hexdigit() || c == '-' || c == '_') + { + return true; + } + false +} + +fn walk_attachment_refs(v: &Value, depth: usize, out: &mut HashSet) { + if depth > 16 { + return; + } + match v { + Value::Null | Value::Bool(_) | Value::Number(_) => {} + Value::String(s) => { + if string_looks_like_attachment_ref(s) { + let b = basename_only(s); + if !b.is_empty() && !b.contains("..") { + out.insert(b); + } + } + } + Value::Array(arr) => { + for el in arr { + walk_attachment_refs(el, depth + 1, out); + } + } + Value::Object(map) => { + for key in ["attachmentId", "attachment_id", "filename"] { + if let Some(Value::String(s)) = map.get(key) { + let b = basename_only(s); + if !b.is_empty() && !b.contains("..") { + out.insert(b); + } + } + } + for (k, val) in map { + if k == "filename" || k == "attachmentId" || k == "attachment_id" { + continue; + } + walk_attachment_refs(val, depth + 1, out); + } + } + } +} + +fn prepare_export_dir(parent: &Path, overwrite: bool) -> Result { + let leaf = export_leaf_folder_name(); + let dest = parent.join(&leaf); + if dest.exists() { + if !overwrite { + return Err(CustodianError::Message(format!( + "export folder already exists: {}", + dest.display() + ))); + } + if dest.is_dir() { + fs::remove_dir_all(&dest)?; + } else { + fs::remove_file(&dest)?; + } + } + fs::create_dir_all(&dest)?; + Ok(dest) +} + +fn escape_for_double_quoted(path: &str) -> String { + path.replace('\\', "\\\\").replace('"', "\\\"") +} + +fn escape_for_r_string(path: &str) -> String { + path.replace('\\', "/").replace('"', "\\\"") +} + +fn escape_for_stata(path: &str) -> String { + path.replace('\\', "/") +} + +fn write_load_snippets( + export_dir: &Path, + parquet_files: &BTreeMap, +) -> Result<(), CustodianError> { + if parquet_files.is_empty() { + return Ok(()); + } + let snippets_dir = export_dir.join("snippets"); + fs::create_dir_all(&snippets_dir)?; + + let mut entries: Vec<(&String, &String)> = parquet_files.iter().collect(); + entries.sort_by(|a, b| a.0.cmp(b.0)); + + // Deduplicate identifiers if form types collide after sanitize. + let mut used: HashMap = HashMap::new(); + let mut named: Vec<(String, String)> = Vec::new(); + for (form_type, path) in &entries { + let base = sanitize_identifier(form_type); + let n = used.entry(base.clone()).or_insert(0); + *n += 1; + let ident = if *n == 1 { base } else { format!("{base}_{n}") }; + named.push((ident, (*path).clone())); + } + + // R + { + let mut body = + String::from("# ODE Desktop export — load Parquet with arrow\nlibrary(arrow)\n\n"); + for (ident, path) in &named { + body.push_str(&format!( + "{ident} <- read_parquet(\"{}\")\n", + escape_for_r_string(path) + )); + } + fs::write(snippets_dir.join("load_r.R"), body)?; + } + + // Python + { + let mut body = String::from( + "# ODE Desktop export — load Parquet with pandas\nimport pandas as pd\n\n", + ); + for (ident, path) in &named { + body.push_str(&format!( + "{ident} = pd.read_parquet(r\"{}\")\n", + path.replace('"', "") + )); + } + fs::write(snippets_dir.join("load_python.py"), body)?; + } + + // Stata 19+ native import parquet (one frame per form type) + { + let mut body = String::from( + "* ODE Desktop export — Stata 19+ import parquet (one frame per form type)\n\n", + ); + for (ident, path) in &named { + let p = escape_for_stata(path); + let quoted = if p.contains(char::is_whitespace) { + format!("\"{p}\"") + } else { + p + }; + body.push_str(&format!("capture frame drop {ident}\n")); + body.push_str(&format!("frame create {ident}\n")); + body.push_str(&format!( + "frame {ident}: import parquet using {quoted}, clear\n\n" + )); + } + fs::write(snippets_dir.join("load_stata.do"), body)?; + } + + // Julia + { + let mut body = String::from( + "# ODE Desktop export — load Parquet with Parquet2 + DataFrames\nusing Parquet2, DataFrames\n\n", + ); + for (ident, path) in &named { + body.push_str(&format!( + "{ident} = DataFrame(Parquet2.Dataset(\"{}\"))\n", + escape_for_double_quoted(path) + )); + } + fs::write(snippets_dir.join("load_julia.jl"), body)?; + } + + Ok(()) +} + +fn copy_referenced_attachments_with_progress( + workspace: &Path, + export_dir: &Path, + rows: &[ExportRow], + progress: &mut ExportProgressFn<'_>, +) -> Result<(usize, usize, PathBuf), CustodianError> { + let mut names: BTreeSet = BTreeSet::new(); + for row in rows { + let obj: serde_json::Map = row + .data + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + for n in collect_attachment_basenames(&Value::Object(obj)) { + names.insert(n); + } + } + let att_dir = export_dir.join("attachments"); + fs::create_dir_all(&att_dir)?; + let total = names.len().max(1); + let mut copied = 0usize; + let mut missing = 0usize; + for (i, name) in names.into_iter().enumerate() { + progress( + i, + total, + &format!("Copying attachments ({}/{})…", i + 1, total), + ); + match resolve_attachment_path(workspace, &name) { + Some(src) => { + let dest = att_dir.join(&name); + fs::copy(&src, &dest).map_err(CustodianError::Io)?; + copied += 1; + } + None => { + missing += 1; + } + } + } + progress(total, total, "Attachments copy finished"); + Ok((copied, missing, att_dir)) +} + +/// Write a Parquet export from already-loaded rows (DB lock should be released before calling). +pub fn write_parquet_export( + workspace: &Path, + req: &ExportParquetRequest, + rows: Vec, + progress: &mut ExportProgressFn<'_>, +) -> Result { + let parent = PathBuf::from(req.parent_dir.trim()); + if parent.as_os_str().is_empty() { + return Err(CustodianError::Message( + "parent directory is required".to_string(), + )); + } + if !parent.is_dir() { + return Err(CustodianError::Message(format!( + "parent directory does not exist: {}", + parent.display() + ))); + } + + progress(0, 1, "Preparing export folder…"); + let export_dir = prepare_export_dir(&parent, req.overwrite)?; + + let mut by_form: BTreeMap> = BTreeMap::new(); + for row in rows { + by_form.entry(row.form_type.clone()).or_default().push(row); + } + + let form_count = by_form.len().max(1); + let mut form_type_counts: BTreeMap = BTreeMap::new(); + let mut parquet_files: BTreeMap = BTreeMap::new(); + let mut total_rows = 0usize; + let mut used_names: HashMap = HashMap::new(); + + for (idx, (form_type, form_rows)) in by_form.iter().enumerate() { + if form_rows.is_empty() { + continue; + } + progress( + idx, + form_count, + &format!("Writing {form_type}.parquet ({}/{})…", idx + 1, form_count), + ); + let (schema, data_cols) = schema_for_rows(form_rows); + let batch = build_record_batch(form_rows, &schema, &data_cols)?; + let stem = sanitize_filename(form_type); + let count = used_names.entry(stem.clone()).or_insert(0); + *count += 1; + let file_stem = if *count == 1 { + stem + } else { + format!("{stem}_{}", *count) + }; + let path = export_dir.join(format!("{file_stem}.parquet")); + write_parquet_file(&path, &batch)?; + parquet_files.insert(form_type.clone(), path.to_string_lossy().to_string()); + form_type_counts.insert(form_type.clone(), form_rows.len()); + total_rows += form_rows.len(); + } + progress(form_count, form_count, "Parquet files written"); + + let workspace_attachments_path = workspace.join("attachments"); + let mut attachments_copied = 0usize; + let mut attachments_missing = 0usize; + let mut export_attachments_path = None; + + if req.include_attachments { + let flat: Vec = by_form.values().flatten().cloned().collect(); + let (copied, missing, att_dir) = + copy_referenced_attachments_with_progress(workspace, &export_dir, &flat, progress)?; + attachments_copied = copied; + attachments_missing = missing; + export_attachments_path = Some(att_dir.to_string_lossy().to_string()); + } + + progress(0, 1, "Writing load snippets…"); + write_load_snippets(&export_dir, &parquet_files)?; + + let manifest = ExportManifest { + exported_at: Utc::now().to_rfc3339(), + exporter_version: EXPORTER_VERSION.to_string(), + include_pending: req.include_pending, + include_attachments: req.include_attachments, + profile_label: req.profile_label.clone(), + workspace_attachments_path: workspace_attachments_path.to_string_lossy().to_string(), + export_attachments_path: export_attachments_path.clone(), + form_type_counts: form_type_counts.clone(), + total_rows, + attachments_copied, + attachments_missing, + }; + let manifest_path = export_dir.join("export_manifest.json"); + fs::write(&manifest_path, serde_json::to_string_pretty(&manifest)?)?; + + progress(1, 1, "Export complete"); + + let export_dir_out = export_dir + .canonicalize() + .unwrap_or_else(|_| export_dir.clone()); + + Ok(ExportParquetResult { + export_dir: export_dir_out.to_string_lossy().to_string(), + form_type_counts, + parquet_files, + total_rows, + attachments_copied, + attachments_missing, + include_pending: req.include_pending, + include_attachments: req.include_attachments, + workspace_attachments_path: workspace_attachments_path.to_string_lossy().to_string(), + export_attachments_path, + manifest_path: manifest_path.to_string_lossy().to_string(), + }) +} + +/// Preview destination path without writing (`{parent}/{YYYYMMDD}`). +pub fn preview_export_dir(parent_dir: &str) -> Result { + let parent = PathBuf::from(parent_dir.trim()); + if parent.as_os_str().is_empty() { + return Err(CustodianError::Message( + "parent directory is required".to_string(), + )); + } + Ok(parent + .join(export_leaf_folder_name()) + .to_string_lossy() + .to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::{Connection, params}; + use serde_json::json; + + fn setup_db() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + r#" + CREATE TABLE observations ( + id TEXT PRIMARY KEY, + payload TEXT NOT NULL, + form_type TEXT, + updated_at TEXT, + remote_updated_at TEXT, + dirty INTEGER NOT NULL DEFAULT 0, + sync_status TEXT NOT NULL DEFAULT 'clean', + conflict_payload TEXT, + last_saved_at TEXT NOT NULL, + last_pushed_at TEXT, + observation_extras TEXT + ); + "#, + ) + .unwrap(); + conn + } + + #[test] + fn sanitize_filename_replaces_unsafe_chars() { + assert_eq!(sanitize_filename("hh/person"), "hh_person"); + assert_eq!(sanitize_filename(""), "unknown"); + } + + #[test] + fn infer_col_kind_prefers_string_on_mixed() { + let a = json!(1); + let b = json!(true); + assert_eq!(infer_col_kind(&[&a, &b]), ColKind::String); + let c = json!(1.5); + let d = json!(2); + assert_eq!(infer_col_kind(&[&c, &d]), ColKind::Float64); + let e = json!(true); + let f = json!(false); + assert_eq!(infer_col_kind(&[&e, &f]), ColKind::Boolean); + } + + #[test] + fn flatten_and_write_parquet_roundtrip_shape() { + let conn = setup_db(); + conn.execute( + "INSERT INTO observations (id, payload, form_type, updated_at, dirty, sync_status, last_saved_at, observation_extras) + VALUES (?1, ?2, ?3, ?4, 0, 'clean', ?4, ?5)", + params![ + "obs-1", + json!({"age": 12, "name": "Ada", "meta": {"x": 1}}).to_string(), + "person", + "2026-01-01T00:00:00Z", + json!({"formVersion": "1", "createdAt": "2026-01-01T00:00:00Z", "author": "a"}).to_string(), + ], + ) + .unwrap(); + // pending row excluded by default + conn.execute( + "INSERT INTO observations (id, payload, form_type, updated_at, dirty, sync_status, last_saved_at) + VALUES ('obs-pending', '{}', 'person', '2026-01-02T00:00:00Z', 1, 'dirty', '2026-01-02T00:00:00Z')", + [], + ) + .unwrap(); + // conflict excluded + conn.execute( + "INSERT INTO observations (id, payload, form_type, updated_at, dirty, sync_status, last_saved_at) + VALUES ('obs-conflict', '{}', 'person', '2026-01-02T00:00:00Z', 1, 'conflict', '2026-01-02T00:00:00Z')", + [], + ) + .unwrap(); + + let base = std::env::temp_dir().join(format!( + "ode_export_test_{}_{}", + std::process::id(), + Utc::now().timestamp_millis() + )); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(&base).unwrap(); + let ws = base.join("workspace"); + fs::create_dir_all(ws.join("attachments")).unwrap(); + let parent = base.join("out"); + fs::create_dir_all(&parent).unwrap(); + + let rows = load_export_rows(&conn, false).unwrap(); + let result = write_parquet_export( + &ws, + &ExportParquetRequest { + parent_dir: parent.to_string_lossy().to_string(), + include_pending: false, + include_attachments: false, + overwrite: true, + profile_label: Some("test".into()), + }, + rows, + &mut |_done, _total, _msg| {}, + ) + .unwrap(); + + assert_eq!(result.total_rows, 1); + assert_eq!(result.form_type_counts.get("person"), Some(&1)); + assert!(result.parquet_files.contains_key("person")); + let parquet_path = PathBuf::from(result.parquet_files.get("person").unwrap()); + assert!(parquet_path.is_file()); + assert!(PathBuf::from(&result.manifest_path).is_file()); + assert!( + PathBuf::from(&result.export_dir) + .join("snippets") + .join("load_r.R") + .is_file() + ); + + let rows = load_export_rows(&conn, true).unwrap(); + assert_eq!(rows.len(), 2); // clean + pending + assert!(rows.iter().any(|r| r.pending)); + + let _ = fs::remove_dir_all(&base); + } + + #[test] + fn collect_attachment_basenames_from_objects() { + let v = json!({ + "photo": { "filename": "a.jpg", "attachmentId": "uuid-1" }, + "note": "hello", + "nested": [{ "filename": "b.png" }] + }); + let names = collect_attachment_basenames(&v); + assert!(names.contains("a.jpg")); + assert!(names.contains("uuid-1")); + assert!(names.contains("b.png")); + assert!(!names.contains("hello")); + } + + #[test] + fn sanitize_identifier_for_languages() { + assert_eq!(sanitize_identifier("censo_milda"), "censo_milda"); + assert_eq!(sanitize_identifier("hh-person"), "hh_person"); + assert_eq!(sanitize_identifier("2bad"), "f_2bad"); + } + + #[test] + fn schema_includes_pending_and_data_prefix() { + let rows = vec![ExportRow { + observation_id: "1".into(), + form_type: "f".into(), + form_version: "1".into(), + created_at: "t".into(), + updated_at: "t".into(), + synced_at: None, + deleted: false, + version: 0, + geolocation: None, + author: None, + device_id: None, + tags: None, + pending: false, + data: BTreeMap::from([("age".into(), json!(3))]), + }]; + let (schema, cols) = schema_for_rows(&rows); + let names: Vec<_> = schema.fields().iter().map(|f| f.name().clone()).collect(); + assert!(names.contains(&"pending".to_string())); + assert!(names.contains(&"data_age".to_string())); + assert_eq!(cols[0].1, ColKind::Float64); + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 15db0e4c4..12e2337c5 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -28,12 +28,13 @@ use zip::read::ZipArchive; use zip::write::SimpleFileOptions; use zip::{CompressionMethod, ZipWriter}; +mod data_export; mod observation_index; mod observation_query; mod sync_engine; #[derive(Debug, Error)] -enum CustodianError { +pub(crate) enum CustodianError { #[error("workspace is not configured")] WorkspaceMissing, #[error("path is outside workspace")] @@ -127,6 +128,15 @@ struct ServerProfile { custom_app_developer_mode: bool, #[serde(default)] custom_app_local_folder: Option, + /// Parent folder last chosen on the Export page (survives restart). + #[serde(default, skip_serializing_if = "Option::is_none")] + export_destination_parent: Option, + /// ISO timestamp of the last successful Parquet export for this profile. + #[serde(default, skip_serializing_if = "Option::is_none")] + last_export_at: Option, + /// Summary of the last successful export (folder, counts, parquet paths). + #[serde(default, skip_serializing_if = "Option::is_none")] + last_export: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -294,7 +304,7 @@ fn migrate_attachments_flat_to_synced_layout(workspace: &Path) -> Result<(), Cus } /// Local resolution: draft → outbound queue (`pending` before `synced`) → loose under `attachments/`. -fn resolve_attachment_path(workspace: &Path, basename: &str) -> Option { +pub(crate) fn resolve_attachment_path(workspace: &Path, basename: &str) -> Option { let root = attachments_root(workspace); let candidates = [ root.join(ATTACH_SUBDIR_DRAFT).join(basename), @@ -535,7 +545,7 @@ struct WorkspaceItem { /// Row columns hold `observation_id`, `form_type`, `data` (payload), and `updated_at`; the rest live here. #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] -struct ObservationExtras { +pub(crate) struct ObservationExtras { #[serde(default, skip_serializing_if = "Option::is_none")] form_version: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -838,6 +848,9 @@ fn default_app_config(data_dir: &Path) -> AppConfigFile { default_app_mode: DefaultAppMode::default(), custom_app_developer_mode: false, custom_app_local_folder: None, + export_destination_parent: None, + last_export_at: None, + last_export: None, }], } } @@ -860,6 +873,9 @@ fn migrate_legacy_workspace(workspace_path: &str, _data_dir: &Path) -> AppConfig default_app_mode: DefaultAppMode::default(), custom_app_developer_mode: false, custom_app_local_folder: None, + export_destination_parent: None, + last_export_at: None, + last_export: None, }], } } @@ -3619,6 +3635,53 @@ fn backup_workspace( .map_err(|e| e.to_string()) } +#[tauri::command] +fn preview_export_dir(parent_dir: String) -> Result { + data_export::preview_export_dir(&parent_dir).map_err(|e| e.to_string()) +} + +#[tauri::command] +async fn export_observations_parquet( + request: data_export::ExportParquetRequest, + app: tauri::AppHandle, + ctx: tauri::State<'_, AppCtxHandle>, +) -> Result { + let ctx = ctx.inner().clone(); + let app_for_block = app.clone(); + tauri::async_runtime::spawn_blocking(move || { + let emit = |phase: &str, message: &str, done: usize, total: usize| { + let _ = app_for_block.emit( + "export/progress", + serde_json::json!({ + "phase": phase, + "message": message, + "done": done, + "total": total, + }), + ); + }; + + emit("starting", "Reading observations…", 0, 1); + let ws = get_workspace_path(&ctx).map_err(|e| e.to_string())?; + + // Hold the SQLite lock only while loading rows — never nest open_db inside + // with_workspace_fs_exclusive (same non-reentrant mutex → deadlock). + let rows = { + let conn = open_db(&ctx).map_err(|e| e.to_string())?; + data_export::load_export_rows(&conn, request.include_pending) + .map_err(|e| e.to_string())? + }; + + let mut progress = |done: usize, total: usize, message: &str| { + emit("running", message, done, total.max(1)); + }; + data_export::write_parquet_export(&ws, &request, rows, &mut progress) + .map_err(|e| e.to_string()) + }) + .await + .map_err(|e| format!("export task failed: {e}"))? +} + /// Moves `sqlite` and `attachments` under `workspace/previous_generations//`, then recreates a fresh layout. #[tauri::command] fn archive_workspace_for_repository_generation( @@ -5415,6 +5478,8 @@ pub fn run() { archive_workspace_for_repository_generation, move_workspace, backup_workspace, + preview_export_dir, + export_observations_parquet, write_workspace_attachment, copy_workspace_attachment_from_path, expand_import_staging_paths, diff --git a/desktop/src/App.css b/desktop/src/App.css index 9e91b7be8..4cc9d9302 100644 --- a/desktop/src/App.css +++ b/desktop/src/App.css @@ -1975,11 +1975,12 @@ button.secondary.danger { position: fixed; right: var(--space-lg); bottom: var(--space-lg); - z-index: 100; + /* Above app chrome / sticky controls; below nested formplayer overlay (300). */ + z-index: 250; display: flex; flex-direction: column-reverse; gap: var(--space-sm); - max-width: 22rem; + max-width: min(26rem, calc(100vw - 2 * var(--space-lg))); pointer-events: none; } @@ -1996,6 +1997,18 @@ button.secondary.danger { font-size: var(--font-size-sm); } +.toast-rich { + max-width: 100%; +} + +.toast-body { + flex: 1; + min-width: 0; + display: flex; + flex-direction: column; + gap: var(--space-sm); +} + .toast-success { border-color: var(--color-success-border); background: var(--color-success-bg); @@ -2014,6 +2027,24 @@ button.secondary.danger { .toast-message { flex: 1; min-width: 0; + white-space: pre-wrap; + overflow-wrap: anywhere; +} + +.toast-detail-list { + margin: 0; + padding: 0.35rem 0 0.15rem 1.1rem; + max-height: 9.5rem; + overflow-y: auto; + overscroll-behavior: contain; + font-size: 0.78rem; + line-height: 1.4; + color: var(--color-text-muted); + border-top: 1px solid rgba(255, 255, 255, 0.08); +} + +.toast-detail-list li { + margin: 0.1rem 0; } .toast-dismiss { @@ -3044,3 +3075,122 @@ input:not([type]) { grid-template-rows: minmax(0, 1fr) minmax(0, 1fr); } } + +.export-progress-panel { + margin-top: var(--space-md); +} + +.export-progress-row { + display: flex; + align-items: center; + gap: var(--space-sm); + margin-bottom: var(--space-sm); +} + +.export-page-header { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: var(--space-md); + flex-wrap: wrap; +} + +.export-page-header h2 { + margin: 0; +} + +.export-last-at { + margin: 0; + font-size: 0.85rem; +} + +.export-dest-block { + margin-top: var(--space-lg); +} + +.export-dest-label { + display: block; + margin-bottom: var(--space-xs); + font-size: 0.9rem; + color: var(--color-text-muted); +} + +.export-dest-row { + display: flex; + gap: var(--space-sm); + align-items: center; +} + +.export-dest-row .field-block { + flex: 1; + min-width: 0; + margin-bottom: 0; + height: var(--control-height); + min-height: var(--control-height); + box-sizing: border-box; +} + +.export-dest-row button { + height: var(--control-height); + min-height: var(--control-height); + padding-top: 0; + padding-bottom: 0; + box-sizing: border-box; +} + +.export-snippets-panel { + margin-top: var(--space-lg); +} + +.export-snippets-panel h3 { + margin-top: 0; +} + +.export-snippet-tab-bar { + margin-bottom: var(--space-md); +} + +.export-snippet-copy { + display: inline-flex; + align-items: center; + justify-content: center; + width: 2rem; + height: 2rem; + min-width: 2rem; + min-height: 2rem; + padding: 0; + border: 1px solid transparent; + border-radius: 0.4rem; + background: transparent; + color: var(--color-text-muted); + box-shadow: none; +} + +.export-snippet-copy:hover:not(:disabled) { + background: #1f2941; + border-color: #455070; + color: var(--color-text); +} + +.export-snippet-copy:focus-visible { + outline: 2px solid var(--color-accent); + outline-offset: 2px; +} + +.export-snippet-copy .material-symbols-outlined { + font-size: 1.15rem; +} + +.export-snippet-code { + margin: 0; + padding: var(--space-md); + max-height: 22rem; + overflow: auto; + background: #ffffff; + color: #111111; + border: 1px solid var(--color-border); + border-radius: 6px; + font-size: 0.82rem; + line-height: 1.45; + white-space: pre; +} diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx index 5e9cf18ed..3ecf059ec 100644 --- a/desktop/src/App.tsx +++ b/desktop/src/App.tsx @@ -15,6 +15,7 @@ import { ObservationsPage } from './pages/ObservationsPage'; import { SyncPage } from './pages/SyncPage'; import { ProfilesPage } from './pages/ProfilesPage'; import { ImportPage } from './pages/ImportPage'; +import { ExportPage } from './pages/ExportPage'; import { FormPreviewPage } from './pages/FormPreviewPage'; import { DeveloperModePanel } from './components/DeveloperModePanel'; import { ObservationIndexPrompt } from './components/ObservationIndexPrompt'; @@ -25,6 +26,7 @@ import { useSynkServerStatus } from './hooks/useSynkServerStatus'; import { selectActiveProfileState, selectBundleActivity, + selectExportActivity, selectSyncActivity, useCustodianStore, } from './store/useCustodianStore'; @@ -44,6 +46,7 @@ const DATA_NAV = [ { to: '/data/profiles', label: 'Profiles', icon: 'badge' }, { to: '/data/observations', label: 'Observations', icon: 'manage_search' }, { to: '/data/import', label: 'Import', icon: 'input' }, + { to: '/data/export', label: 'Export', icon: 'output' }, { to: '/data/sync', label: 'Sync', icon: 'sync_alt' }, { to: '/data/about', label: 'About', icon: 'info' }, ]; @@ -223,6 +226,7 @@ function Shell() { const pushToast = useToastStore(s => s.pushToast); const syncActivity = useCustodianStore(selectSyncActivity); const bundleActivity = useCustodianStore(selectBundleActivity); + const exportActivity = useCustodianStore(selectExportActivity); const importActivity = useImportStagingStore(selectImportActivity); const setBundleActivity = useCustodianStore(s => s.setBundleActivity); const clearBundleActivity = useCustodianStore(s => s.clearBundleActivity); @@ -239,6 +243,7 @@ function Shell() { syncActivity?.statusText ?? bundleActivity?.statusText ?? importActivity?.statusText ?? + exportActivity?.statusText ?? null; const activityProgress = @@ -247,10 +252,18 @@ function Shell() { 100, Math.round((bundleActivity.done / bundleActivity.total) * 100), ) - : null; + : exportActivity && exportActivity.total > 0 + ? Math.min( + 100, + Math.round((exportActivity.done / exportActivity.total) * 100), + ) + : null; const activityPresent = - syncActivity !== null || bundleActivity !== null || importActivity !== null; + syncActivity !== null || + bundleActivity !== null || + importActivity !== null || + exportActivity !== null; const [activityBannerDismissed, setActivityBannerDismissed] = useState(false); useEffect(() => { @@ -280,166 +293,174 @@ function Shell() { (syncMessage.includes('\n') || syncMessage.length > 120); return ( -
+ <> - + + +
+ + Open Data Ensemble {year} +
+ -
- - {isWorkbench ? : null} -
- {showActivityBanner ? ( -
-
- - {activityText} - {activityProgress !== null ? ( -
+
+ + {isWorkbench ? : null} +
+ {showActivityBanner ? ( +
+
+ + {activityText} + {activityProgress !== null ? (
-
- ) : null} + className="activity-progress" + role="progressbar" + aria-valuenow={activityProgress} + aria-valuemin={0} + aria-valuemax={100}> +
+
+ ) : null} +
+
- -
- ) : null} - {showSyncMessageBanner ? ( -
-
{syncMessage}
- -
- ) : null} - - } /> - } - /> - } - /> - } - /> - } - /> - } - /> - } - /> - } - /> - } - /> - } - /> - } - /> - } - /> + ) : null} + {showSyncMessageBanner ? ( +
+
{syncMessage}
+ +
+ ) : null} + + } /> + } + /> + } + /> + } + /> + } + /> + } + /> + } + /> + } + /> + } + /> + } + /> + } + /> + } + /> + } + /> - } /> - } /> - } /> - } /> - } /> + } /> + } /> + } /> + } /> + } /> + } /> - } - /> - } - /> - } - /> - } - /> - -
-
-
+ } + /> + } + /> + } + /> + } + /> + + + + + ); } diff --git a/desktop/src/components/ToastHost.tsx b/desktop/src/components/ToastHost.tsx index 653af6b7e..5c302bbbd 100644 --- a/desktop/src/components/ToastHost.tsx +++ b/desktop/src/components/ToastHost.tsx @@ -11,8 +11,20 @@ export function ToastHost() { return (
{toasts.map(t => ( -
- {t.message} +
+
+ {t.message} + {t.detailLines && t.detailLines.length > 0 ? ( +
    + {t.detailLines.map(line => ( +
  • {line}
    • + ))} +
+ ) : null} +
+
+
+ +
+ + {lastResult?.exportDir ? ( + + ) : null} +
+ + {error ? ( +
+ {error} +
+ ) : null} +
+ +
+

Load in analysis tools

+

+ {lastResult?.parquetFiles && + Object.keys(lastResult.parquetFiles).length > 0 + ? 'Paths match the last export. Scripts are also written under snippets/ in that folder.' + : 'Replace [INSERT PATH TO PARQUET FILE] after exporting, or copy again once paths are known.'} +

+
+ {EXPORT_SNIPPET_LANGS.map(lang => ( + + ))} +
+ +
+
+ 
+          {snippetCode}
+
+
+ + ); +} diff --git a/desktop/src/store/useCustodianStore.ts b/desktop/src/store/useCustodianStore.ts index 7a31fcd58..9671f8647 100644 --- a/desktop/src/store/useCustodianStore.ts +++ b/desktop/src/store/useCustodianStore.ts @@ -217,6 +217,11 @@ interface CustodianState { done: number; total: number; } | null; + exportActivity: { + statusText: string; + done: number; + total: number; + } | null; /** Persisted Rust sync job awaiting resume (transient stall, auth, or cold start). */ syncPausedJob: SyncJobRowOut | null; /** Bumped after each successful `refresh_custom_app_dev_mirror` (Workbench embeds). */ @@ -230,6 +235,8 @@ interface CustodianState { setDevError: (message: string | null) => void; setBundleActivity: (activity: CustodianState['bundleActivity']) => void; clearBundleActivity: () => void; + setExportActivity: (activity: CustodianState['exportActivity']) => void; + clearExportActivity: () => void; dismissObservationIndexPrompt: () => void; createPendingObservationIndexes: () => Promise; refreshSettings: () => Promise; @@ -365,6 +372,7 @@ export const useCustodianStore = create((set, get) => ({ syncMessage: null, syncActivity: null, bundleActivity: null, + exportActivity: null, syncPausedJob: null, devMirrorGeneration: 0, devBusy: false, @@ -377,6 +385,8 @@ export const useCustodianStore = create((set, get) => ({ setBundleActivity: activity => set({ bundleActivity: activity }), clearBundleActivity: () => set({ bundleActivity: null }), + setExportActivity: activity => set({ exportActivity: activity }), + clearExportActivity: () => set({ exportActivity: null }), ensureActiveProfileAuth: async () => { if (selectAuthSessionForActiveProfile(get())?.token) { @@ -952,6 +962,10 @@ export function selectBundleActivity(state: CustodianState) { return state.bundleActivity; } +export function selectExportActivity(state: CustodianState) { + return state.exportActivity; +} + export function selectPausedSyncJob(state: CustodianState) { return state.syncPausedJob; } diff --git a/desktop/src/store/useExportPageStore.ts b/desktop/src/store/useExportPageStore.ts new file mode 100644 index 000000000..49da97448 --- /dev/null +++ b/desktop/src/store/useExportPageStore.ts @@ -0,0 +1,85 @@ +import { create } from 'zustand'; +import type { ExportParquetResult, ServerProfile } from '../types/domain'; +import type { ExportSnippetLang } from '../lib/exportLoadSnippets'; + +interface ExportPageState { + /** Profile whose export prefs are currently loaded into this store. */ + hydratedProfileId: string | null; + /** Parent folder chosen for export (`YYYYMMDD` leaf is created underneath). */ + destinationParent: string | null; + includePending: boolean; + includeAttachments: boolean; + snippetLang: ExportSnippetLang; + lastResult: ExportParquetResult | null; + /** ISO timestamp of last successful export (persisted on the profile). */ + lastExportAt: string | null; + error: string | null; + busy: boolean; + hydrateFromProfile: (profile: ServerProfile) => void; + clearHydration: () => void; + setDestinationParent: (path: string | null) => void; + setIncludePending: (v: boolean) => void; + setIncludeAttachments: (v: boolean) => void; + setSnippetLang: (lang: ExportSnippetLang) => void; + setLastResult: (result: ExportParquetResult | null) => void; + setLastExportAt: (iso: string | null) => void; + setError: (error: string | null) => void; + setBusy: (busy: boolean) => void; + recordSuccessfulExport: (result: ExportParquetResult) => string; +} + +function trimPath(path: string | null | undefined): string | null { + const t = path?.trim(); + return t ? t : null; +} + +export const useExportPageStore = create(set => ({ + hydratedProfileId: null, + destinationParent: null, + includePending: false, + includeAttachments: false, + snippetLang: 'r', + lastResult: null, + lastExportAt: null, + error: null, + busy: false, + + hydrateFromProfile: profile => + set({ + hydratedProfileId: profile.id, + destinationParent: trimPath(profile.exportDestinationParent), + lastExportAt: trimPath(profile.lastExportAt), + lastResult: profile.lastExport ?? null, + error: null, + busy: false, + }), + clearHydration: () => + set({ + hydratedProfileId: null, + destinationParent: null, + lastResult: null, + lastExportAt: null, + error: null, + busy: false, + }), + setDestinationParent: path => + set({ + destinationParent: trimPath(path), + }), + setIncludePending: includePending => set({ includePending }), + setIncludeAttachments: includeAttachments => set({ includeAttachments }), + setSnippetLang: snippetLang => set({ snippetLang }), + setLastResult: lastResult => set({ lastResult }), + setLastExportAt: lastExportAt => set({ lastExportAt }), + setError: error => set({ error }), + setBusy: busy => set({ busy }), + recordSuccessfulExport: result => { + const lastExportAt = new Date().toISOString(); + set({ + lastResult: result, + lastExportAt, + error: null, + }); + return lastExportAt; + }, +})); diff --git a/desktop/src/store/useToastStore.ts b/desktop/src/store/useToastStore.ts index 3cdfb952d..804ffe75a 100644 --- a/desktop/src/store/useToastStore.ts +++ b/desktop/src/store/useToastStore.ts @@ -6,35 +6,52 @@ export interface ToastItem { id: string; message: string; variant: ToastVariant; + /** Optional secondary lines (e.g. per–form-type counts); shown in a scrollable list. */ + detailLines?: string[]; } const MAX_TOASTS = 3; let nextId = 0; -function autoDismissMs(variant: ToastVariant): number { +function autoDismissMs(variant: ToastVariant, hasDetails: boolean): number { + if (hasDetails) { + return 14000; + } if (variant === 'error' || variant === 'warn') { return 8000; } return 5000; } +export type PushToastInput = { + message: string; + variant?: ToastVariant; + detailLines?: string[]; +}; + interface ToastState { toasts: ToastItem[]; - pushToast: (input: { message: string; variant?: ToastVariant }) => void; + pushToast: (input: PushToastInput) => void; dismissToast: (id: string) => void; } export const useToastStore = create((set, get) => ({ toasts: [], - pushToast: ({ message, variant = 'info' }) => { + pushToast: ({ message, variant = 'info', detailLines }) => { const id = `toast-${++nextId}`; - const item: ToastItem = { id, message, variant }; + const lines = detailLines?.filter(l => l.trim().length > 0); + const item: ToastItem = { + id, + message, + variant, + ...(lines && lines.length > 0 ? { detailLines: lines } : {}), + }; set(state => ({ toasts: [item, ...state.toasts].slice(0, MAX_TOASTS), })); - const ms = autoDismissMs(variant); + const ms = autoDismissMs(variant, Boolean(item.detailLines?.length)); window.setTimeout(() => { if (get().toasts.some(t => t.id === id)) { get().dismissToast(id); diff --git a/desktop/src/types/domain.ts b/desktop/src/types/domain.ts index 4f29a5eaa..488dbef3a 100644 --- a/desktop/src/types/domain.ts +++ b/desktop/src/types/domain.ts @@ -230,6 +230,12 @@ export interface ServerProfile { customAppDeveloperMode?: boolean | null; /** Absolute path to a folder containing `index.html` (e.g. custom app `dist/`). */ customAppLocalFolder?: string | null; + /** Parent folder last chosen on the Export page (survives restart). */ + exportDestinationParent?: string | null; + /** ISO timestamp of the last successful Parquet export for this profile. */ + lastExportAt?: string | null; + /** Summary of the last successful export (folder, counts, parquet paths). */ + lastExport?: ExportParquetResult | null; } /** Result of mirroring a local custom app folder into the profile workspace. */ @@ -435,3 +441,27 @@ export interface WorkspaceDomainItem { path: string; kind: 'directory' | 'file'; } + +/** Result of local Parquet export (`export_observations_parquet`). */ +export interface ExportParquetResult { + exportDir: string; + formTypeCounts: Record; + /** Absolute `.parquet` paths keyed by form type. */ + parquetFiles: Record; + totalRows: number; + attachmentsCopied: number; + attachmentsMissing: number; + includePending: boolean; + includeAttachments: boolean; + workspaceAttachmentsPath: string; + exportAttachmentsPath?: string | null; + manifestPath: string; +} + +export interface ExportParquetRequest { + parentDir: string; + includePending?: boolean; + includeAttachments?: boolean; + overwrite?: boolean; + profileLabel?: string | null; +}