diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 3e3319a..ab6c3e8 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -462,6 +462,28 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "chrono-tz" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -2100,6 +2122,7 @@ dependencies = [ "base64 0.22.1", "bytes", "chrono", + "chrono-tz", "clap", "diesel", "diesel_migrations", @@ -2117,6 +2140,7 @@ dependencies = [ "percent-encoding", "quick-xml", "rand 0.9.2", + "regex", "reqwest", "rust-s3", "serde", @@ -2124,6 +2148,7 @@ dependencies = [ "serde_bytes", "serde_cbor_2", "serde_json", + "serde_yaml", "sha2", "tempfile", "thiserror 2.0.17", @@ -2162,6 +2187,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "parse-zoneinfo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] + [[package]] name = "password-hash" version = "0.5.0" @@ -2230,6 +2264,44 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -3001,6 +3073,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1" version = "0.10.6" @@ -3075,6 +3160,12 @@ dependencies = [ "time", ] +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "slab" version = "0.4.11" @@ -3555,6 +3646,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index dddccc8..f4ce6ce 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -45,6 +45,7 @@ quick-xml = "0.38" futures-util = "0.3" url = "2.5" once_cell = "1.19" +regex = "1.11" utoipa = { version = "4.2", default-features = false, features = ["chrono", "uuid", "preserve_order"] } clap = { version = "4.5", features = ["derive"] } @@ -63,10 +64,18 @@ serde_cbor_2 = "0.13" rand = "0.9" hyper = "1.2" http-body-util = "0.1" +chrono-tz = "0.8" [dev-dependencies] once_cell = "1.19" webauthn-rs-core = "0.5" +serde_yaml = "0.9" + +[build-dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_yaml = "0.9" +serde_json = "1.0" +regex = "1.11" [[bin]] name = "backend" diff --git a/backend/build.rs b/backend/build.rs new file mode 100644 index 0000000..e040bf0 --- /dev/null +++ b/backend/build.rs @@ -0,0 +1,123 @@ +use std::env; +use std::fs; +use std::path::PathBuf; + +use regex::escape; +use serde::Deserialize; + +#[derive(Deserialize)] +struct CaseSuite { + cases: Vec, +} + +#[derive(Deserialize)] +struct CaseName { + name: String, +} + +#[derive(Deserialize)] +struct MonthSuite { + months: Vec, +} + +#[derive(Deserialize)] +struct MonthDefinition { + name: String, + month: u32, + #[serde(default)] + locales: Vec, +} + +fn sanitize(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + for ch in name.chars() { + if ch.is_ascii_alphanumeric() { + out.push(ch); + } else { + out.push('_'); + } + } + if out.is_empty() { + "case".to_string() + } else if out.chars().next().unwrap().is_ascii_digit() { + format!("_{}", out) + } else { + out + } +} + +fn quote(value: &str) -> String { + serde_json::to_string(value).expect("string literal") +} + +fn generate_tests() -> Result> { + let yaml_path = PathBuf::from("tests/data/issued_at_cases.yaml"); + let contents = fs::read_to_string(&yaml_path)?; + let suite: CaseSuite = serde_yaml::from_str(&contents)?; + + let mut output = + String::from("#[cfg(test)]\npub mod issued_at_generated_tests {\n use super::*;\n"); + + for case in suite.cases { + let ident = sanitize(&case.name); + output.push_str(&format!( + " #[test]\n fn {}() {{\n run_named_case(\"{}\");\n }}\n", + ident, case.name + )); + } + + output.push_str("}\n"); + Ok(output) +} + +fn generate_months() -> Result> { + let yaml_path = PathBuf::from("resources/issued_at_months.yaml"); + let contents = fs::read_to_string(&yaml_path)?; + let suite: MonthSuite = serde_yaml::from_str(&contents)?; + + let mut pattern_parts = Vec::with_capacity(suite.months.len()); + let mut entries = String::new(); + for entry in &suite.months { + pattern_parts.push(escape(&entry.name)); + let locales_literal = if entry.locales.is_empty() { + "&[]".to_string() + } else { + let joined = entry + .locales + .iter() + .map(|loc| quote(loc)) + .collect::>() + .join(", "); + format!("&[{}]", joined) + }; + entries.push_str(&format!( + " MonthVariant {{ name: {}, month: {}, locales: {} }},\n", + quote(&entry.name), + entry.month, + locales_literal + )); + } + + let pattern_literal = quote(&pattern_parts.join("|")); + let output = format!( + "pub(super) static MONTH_VARIANTS: &[MonthVariant] = &[\n{entries}];\n\n", + entries = entries + ) + &format!( + "pub(super) const MONTH_PATTERN: &str = {};\n", + pattern_literal + ); + Ok(output) +} + +fn main() -> Result<(), Box> { + println!("cargo:rerun-if-changed=tests/data/issued_at_cases.yaml"); + println!("cargo:rerun-if-changed=resources/issued_at_months.yaml"); + + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + fs::write( + out_dir.join("issued_at_generated_tests.rs"), + generate_tests()?, + )?; + fs::write(out_dir.join("issued_at_months.rs"), generate_months()?)?; + Ok(()) +} diff --git a/backend/resources/issued_at_months.yaml b/backend/resources/issued_at_months.yaml new file mode 100644 index 0000000..28c727d --- /dev/null +++ b/backend/resources/issued_at_months.yaml @@ -0,0 +1,648 @@ +months: + - name: "january" + month: 1 + locales: + - "en" + - name: "jan" + month: 1 + locales: + - "en" + - name: "janvier" + month: 1 + locales: + - "fr" + - name: "janv" + month: 1 + locales: + - "fr" + - name: "januar" + month: 1 + locales: + - "de" + - name: "janu\u00e1r" + month: 1 + locales: + - "hu" + - name: "leden" + month: 1 + locales: + - "cs" + - name: "sije\u010danj" + month: 1 + locales: + - "hr" + - name: "sijecanj" + month: 1 + locales: + - "hr" + - name: "stycze\u0144" + month: 1 + locales: + - "pl" + - name: "styczen" + month: 1 + locales: + - "pl" + - name: "ocak" + month: 1 + locales: + - "tr" + - name: "february" + month: 2 + locales: + - "en" + - name: "feb" + month: 2 + locales: + - "en" + - name: "f\u00e9vrier" + month: 2 + locales: + - "fr" + - name: "fevrier" + month: 2 + locales: + - "fr" + - name: "f\u00e9vr" + month: 2 + locales: + - "fr" + - name: "fevr" + month: 2 + locales: + - "fr" + - name: "februar" + month: 2 + locales: + - "de" + - name: "\u00fanor" + month: 2 + locales: + - "cs" + - name: "unor" + month: 2 + locales: + - "cs" + - name: "\u00fanora" + month: 2 + locales: + - "cs" + - name: "unora" + month: 2 + locales: + - "cs" + - name: "velja\u010da" + month: 2 + locales: + - "hr" + - name: "veljaca" + month: 2 + locales: + - "hr" + - name: "velja\u010de" + month: 2 + locales: + - "hr" + - name: "veljace" + month: 2 + locales: + - "hr" + - name: "luty" + month: 2 + locales: + - "pl" + - name: "\u015fubat" + month: 2 + locales: + - "tr" + - name: "subat" + month: 2 + locales: + - "tr" + - name: "march" + month: 3 + locales: + - "en" + - name: "mar" + month: 3 + locales: + - "en" + - name: "m\u00e4rz" + month: 3 + locales: + - "de" + - name: "maerz" + month: 3 + locales: + - "de" + - name: "mars" + month: 3 + locales: + - "fr" + - name: "m\u00e4r" + month: 3 + locales: + - "de" + - name: "marz" + month: 3 + locales: + - "de" + - name: "b\u0159ezen" + month: 3 + locales: + - "cs" + - name: "brezen" + month: 3 + locales: + - "cs" + - name: "b\u0159ezna" + month: 3 + locales: + - "cs" + - name: "brezna" + month: 3 + locales: + - "cs" + - name: "o\u017eujak" + month: 3 + locales: + - "hr" + - name: "ozujak" + month: 3 + locales: + - "hr" + - name: "o\u017eujka" + month: 3 + locales: + - "hr" + - name: "ozujka" + month: 3 + locales: + - "hr" + - name: "marzec" + month: 3 + locales: + - "pl" + - name: "mart" + month: 3 + locales: + - "sr" + - "bs" + - name: "m\u00e1rcius" + month: 3 + locales: + - "hu" + - name: "martie" + month: 3 + locales: + - "ro" + - name: "april" + month: 4 + locales: + - "en" + - name: "apr" + month: 4 + locales: + - "en" + - name: "avril" + month: 4 + locales: + - "fr" + - name: "abril" + month: 4 + locales: + - "es" + - "pt" + - name: "duben" + month: 4 + locales: + - "cs" + - name: "travanj" + month: 4 + locales: + - "hr" + - name: "nisan" + month: 4 + locales: + - "tr" + - name: "kwiecie\u0144" + month: 4 + locales: + - "pl" + - name: "kwiecien" + month: 4 + locales: + - "pl" + - name: "aprile" + month: 4 + locales: + - "it" + - name: "may" + month: 5 + locales: + - "en" + - name: "mai" + month: 5 + locales: + - "fr" + - "de" + - name: "mayo" + month: 5 + locales: + - "es" + - name: "kv\u011bten" + month: 5 + locales: + - "cs" + - name: "kveten" + month: 5 + locales: + - "cs" + - name: "kv\u011btna" + month: 5 + locales: + - "cs" + - name: "kvetna" + month: 5 + locales: + - "cs" + - name: "svibanj" + month: 5 + locales: + - "hr" + - name: "maj" + month: 5 + locales: + - "pl" + - "bs" + - "sr" + - "hr" + - name: "may\u0131s" + month: 5 + locales: + - "tr" + - name: "mayis" + month: 5 + locales: + - "tr" + - name: "maggio" + month: 5 + locales: + - "it" + - name: "m\u00e1j" + month: 5 + locales: + - "hu" + - "sk" + - name: "june" + month: 6 + locales: + - "en" + - name: "jun" + month: 6 + locales: + - "en" + - "de" + - name: "juin" + month: 6 + locales: + - "fr" + - name: "junio" + month: 6 + locales: + - "es" + - name: "juni" + month: 6 + locales: + - "de" + - name: "\u010derven" + month: 6 + locales: + - "cs" + - name: "cerven" + month: 6 + locales: + - "cs" + - name: "lipanj" + month: 6 + locales: + - "hr" + - name: "haziran" + month: 6 + locales: + - "tr" + - name: "czerwiec" + month: 6 + locales: + - "pl" + - name: "j\u00fanius" + month: 6 + locales: + - "hu" + - name: "giugno" + month: 6 + locales: + - "it" + - name: "july" + month: 7 + locales: + - "en" + - name: "jul" + month: 7 + locales: + - "en" + - "de" + - name: "juillet" + month: 7 + locales: + - "fr" + - name: "julio" + month: 7 + locales: + - "es" + - name: "temmuz" + month: 7 + locales: + - "tr" + - name: "\u010dervenec" + month: 7 + locales: + - "cs" + - name: "cervenec" + month: 7 + locales: + - "cs" + - name: "srpanj" + month: 7 + locales: + - "hr" + - name: "lipiec" + month: 7 + locales: + - "pl" + - name: "luglio" + month: 7 + locales: + - "it" + - name: "j\u00falius" + month: 7 + locales: + - "hu" + - name: "august" + month: 8 + locales: + - "en" + - name: "aug" + month: 8 + locales: + - "en" + - "de" + - name: "ao\u00fbt" + month: 8 + locales: + - "fr" + - name: "aout" + month: 8 + locales: + - "fr" + - name: "agosto" + month: 8 + locales: + - "es" + - "pt" + - "it" + - name: "a\u011fustos" + month: 8 + locales: + - "tr" + - name: "agustos" + month: 8 + locales: + - "tr" + - name: "kolovoz" + month: 8 + locales: + - "hr" + - name: "srpen" + month: 8 + locales: + - "cs" + - name: "sierpie\u0144" + month: 8 + locales: + - "pl" + - name: "sierpien" + month: 8 + locales: + - "pl" + - name: "augustus" + month: 8 + locales: + - "nl" + - name: "agost" + month: 8 + locales: + - "it" + - "ca" + - name: "september" + month: 9 + locales: + - "en" + - name: "sept" + month: 9 + locales: + - "en" + - "fr" + - name: "sep" + month: 9 + locales: + - "en" + - name: "septembre" + month: 9 + locales: + - "fr" + - name: "septiembre" + month: 9 + locales: + - "es" + - name: "eyl\u00fcl" + month: 9 + locales: + - "tr" + - name: "eylul" + month: 9 + locales: + - "tr" + - name: "z\u00e1\u0159\u00ed" + month: 9 + locales: + - "cs" + - name: "zari" + month: 9 + locales: + - "cs" + - name: "rujan" + month: 9 + locales: + - "hr" + - name: "wrzesie\u0144" + month: 9 + locales: + - "pl" + - name: "wrzesien" + month: 9 + locales: + - "pl" + - name: "septembrie" + month: 9 + locales: + - "ro" + - name: "settembre" + month: 9 + locales: + - "it" + - name: "october" + month: 10 + locales: + - "en" + - name: "oct" + month: 10 + locales: + - "en" + - name: "oktober" + month: 10 + locales: + - "de" + - name: "okt\u00f3ber" + month: 10 + locales: + - "hu" + - name: "octobre" + month: 10 + locales: + - "fr" + - name: "octubre" + month: 10 + locales: + - "es" + - name: "\u0159\u00edjen" + month: 10 + locales: + - "cs" + - name: "rijen" + month: 10 + locales: + - "hr" + - name: "pa\u017adziernik" + month: 10 + locales: + - "pl" + - name: "pazdziernik" + month: 10 + locales: + - "pl" + - name: "ekim" + month: 10 + locales: + - "tr" + - name: "octombrie" + month: 10 + locales: + - "ro" + - name: "november" + month: 11 + locales: + - "en" + - name: "nov" + month: 11 + locales: + - "en" + - name: "novembre" + month: 11 + locales: + - "fr" + - name: "noviembre" + month: 11 + locales: + - "es" + - name: "studeni" + month: 11 + locales: + - "hr" + - name: "kas\u0131m" + month: 11 + locales: + - "tr" + - name: "kasim" + month: 11 + locales: + - "tr" + - name: "listopad" + month: 11 + locales: + - "pl" + - "cs" + - name: "novembro" + month: 11 + locales: + - "pt" + - name: "listopadu" + month: 11 + locales: + - "cs" + - name: "noiembrie" + month: 11 + locales: + - "ro" + - name: "december" + month: 12 + locales: + - "en" + - name: "dec" + month: 12 + locales: + - "en" + - name: "dezember" + month: 12 + locales: + - "de" + - name: "d\u00e9cembre" + month: 12 + locales: + - "fr" + - name: "decembre" + month: 12 + locales: + - "fr" + - name: "prosinec" + month: 12 + locales: + - "cs" + - name: "prosinac" + month: 12 + locales: + - "hr" + - name: "grudzie\u0144" + month: 12 + locales: + - "pl" + - name: "grudzien" + month: 12 + locales: + - "pl" + - name: "grudnia" + month: 12 + locales: + - "pl" + - name: "aral\u0131k" + month: 12 + locales: + - "tr" + - name: "aralik" + month: 12 + locales: + - "tr" + - name: "decembrie" + month: 12 + locales: + - "ro" diff --git a/backend/src/config.rs b/backend/src/config.rs index d095a96..6fc2e77 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -1,6 +1,7 @@ use anyhow::{Context, Result}; use url::Url; +use serde::de::Deserializer; use serde::Deserialize; use serde_aux::field_attributes::deserialize_bool_from_anything; @@ -60,6 +61,16 @@ pub struct AppConfig { pub worker_max_document_bytes: u64, #[serde(default = "default_upload_body_limit_bytes")] pub upload_body_limit_bytes: u64, + #[serde(default = "default_service_timezone")] + pub service_timezone: String, + #[serde(default = "default_issued_at_date_order")] + pub issued_at_date_order: String, + #[serde(default)] + pub issued_at_filename_date_order: Option, + #[serde(default, deserialize_with = "deserialize_string_list")] + pub issued_at_date_parser_locales: Vec, + #[serde(default, deserialize_with = "deserialize_string_list")] + pub issued_at_ignore_dates: Vec, #[serde(default)] pub webauthn_rp_id: Option, #[serde(default)] @@ -183,6 +194,52 @@ fn default_upload_body_limit_bytes() -> u64 { 128 * 1024 * 1024 } +fn default_service_timezone() -> String { + "UTC".to_string() +} + +fn default_issued_at_date_order() -> String { + "DMY".to_string() +} + +fn deserialize_string_list<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum Helper { + List(Vec), + Single(String), + } + + let helper = Option::::deserialize(deserializer)?; + let mut values = Vec::new(); + + if let Some(helper) = helper { + match helper { + Helper::List(list) => { + for entry in list { + let trimmed = entry.trim(); + if !trimmed.is_empty() { + values.push(trimmed.to_string()); + } + } + } + Helper::Single(value) => { + for part in value.split(',') { + let trimmed = part.trim(); + if !trimmed.is_empty() { + values.push(trimmed.to_string()); + } + } + } + } + } + + Ok(values) +} + fn default_webauthn_rp_name() -> String { "Papercrate".to_string() } diff --git a/backend/src/issued_at.rs b/backend/src/issued_at.rs new file mode 100644 index 0000000..f70487e --- /dev/null +++ b/backend/src/issued_at.rs @@ -0,0 +1,167 @@ +use std::collections::HashSet; + +use chrono::{DateTime, Datelike, NaiveDate, TimeZone, Utc}; +use chrono_tz::Tz; +use once_cell::sync::Lazy; +use tracing::warn; + +use crate::config::AppConfig; + +#[derive(Clone, Copy, Debug)] +pub enum DateOrder { + Dmy, + Mdy, + Ymd, +} + +impl DateOrder { + pub fn parse(value: &str) -> Self { + Self::try_parse(value).unwrap_or(DateOrder::Dmy) + } + + pub fn try_parse(value: &str) -> Option { + match value.trim().to_ascii_uppercase().as_str() { + "YMD" => Some(DateOrder::Ymd), + "MDY" => Some(DateOrder::Mdy), + "DMY" => Some(DateOrder::Dmy), + _ => None, + } + } +} + +static MIN_ISSUED_AT_DATE: Lazy = + Lazy::new(|| NaiveDate::from_ymd_opt(1901, 1, 1).expect("valid minimum issued_at date")); + +#[derive(Clone, Debug)] +pub struct IssuedAtSettings { + pub timezone: Tz, + pub date_order: DateOrder, + pub filename_date_order: Option, + pub locales: HashSet, + pub ignore_dates: HashSet, + pub min_date: NaiveDate, +} + +impl IssuedAtSettings { + pub fn from_config(config: &AppConfig) -> Self { + let timezone = config.service_timezone.parse::().unwrap_or_else(|_| { + warn!( + timezone = %config.service_timezone, + "invalid service timezone configured; falling back to UTC" + ); + chrono_tz::UTC + }); + + let date_order = DateOrder::parse(&config.issued_at_date_order); + let filename_date_order = + config + .issued_at_filename_date_order + .as_deref() + .and_then(|value| { + DateOrder::try_parse(value).or_else(|| { + warn!(value, "invalid issued_at filename date order; ignoring"); + None + }) + }); + + let locales = config + .issued_at_date_parser_locales + .iter() + .filter_map(|value| { + let trimmed = value.trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_ascii_lowercase()) + } + }) + .collect::>(); + + // Ignore dates are evaluated after normalizing candidate timestamps to + // the configured service timezone, so administrators should provide + // local calendar dates rather than UTC midnights. + let ignore_dates = config + .issued_at_ignore_dates + .iter() + .filter_map(|value| { + let trimmed = value.trim(); + if trimmed.is_empty() { + return None; + } + match NaiveDate::parse_from_str(trimmed, "%Y-%m-%d") { + Ok(date) => Some(date), + Err(err) => { + warn!(value = trimmed, error = %err, "invalid issued_at ignore date"); + None + } + } + }) + .collect(); + + Self { + timezone, + date_order, + filename_date_order, + locales, + ignore_dates, + min_date: *MIN_ISSUED_AT_DATE, + } + } + + /// Returns the immutable (lowercase) locale allowlist supplied via config. + pub fn locales(&self) -> &HashSet { + &self.locales + } + + /// Returns the immutable set of local-calendar dates that should be ignored. + pub fn ignore_dates(&self) -> &HashSet { + &self.ignore_dates + } + + /// Returns the configured service timezone (copy type). + pub fn timezone(&self) -> Tz { + self.timezone + } + + pub fn date_order(&self) -> DateOrder { + self.date_order + } + + pub fn filename_date_order(&self) -> Option { + self.filename_date_order + } + + pub fn normalize_naive( + &self, + date: NaiveDate, + now_utc: chrono::DateTime, + ) -> Option> { + if !self.is_valid_with_now(date, now_utc) { + return None; + } + self.timezone + .with_ymd_and_hms(date.year(), date.month(), date.day(), 0, 0, 0) + .earliest() + .map(|dt| dt.with_timezone(&Utc)) + } + + pub fn normalize_datetime( + &self, + dt: chrono::DateTime, + now_utc: chrono::DateTime, + ) -> Option> { + let local_date = dt.with_timezone(&self.timezone).date_naive(); + self.normalize_naive(local_date, now_utc) + } + + fn is_valid_with_now(&self, date: NaiveDate, now_utc: chrono::DateTime) -> bool { + if date < self.min_date { + return false; + } + let now_local = now_utc.with_timezone(&self.timezone).date_naive(); + if date > now_local { + return false; + } + !self.ignore_dates.contains(&date) + } +} diff --git a/backend/src/lib.rs b/backend/src/lib.rs index 10ea765..5162182 100644 --- a/backend/src/lib.rs +++ b/backend/src/lib.rs @@ -4,6 +4,7 @@ pub mod db; pub mod documents; pub mod error; pub mod http; +pub mod issued_at; pub mod jobs; pub mod models; pub mod openapi; diff --git a/backend/src/state.rs b/backend/src/state.rs index 2526d74..484fcbe 100644 --- a/backend/src/state.rs +++ b/backend/src/state.rs @@ -11,6 +11,7 @@ use crate::{ config::AppConfig, db::PgPool, error::{AppError, AppResult}, + issued_at::IssuedAtSettings, storage::{ObjectStorage, TenantStorage}, tenants::{apply_tenant_guc, clear_tenant_context, clear_user_guc, TenantService}, }; @@ -25,6 +26,7 @@ pub struct AppState { pub jwt: JwtService, pub tenants: TenantService, pub passkeys: Option, + issued_at: Arc, } impl AppState { @@ -47,6 +49,7 @@ impl AppState { storage: Arc, jwt: JwtService, ) -> Self { + let issued_at = Arc::new(IssuedAtSettings::from_config(&config)); let config = Arc::new(config); let tenants = TenantService::new(pool.clone()); @@ -65,6 +68,7 @@ impl AppState { jwt, tenants, passkeys, + issued_at, } } @@ -96,4 +100,8 @@ impl AppState { AppError::internal("tenant storage error") }) } + + pub fn issued_at_settings(&self) -> Arc { + self.issued_at.clone() + } } diff --git a/backend/src/test_support/mod.rs b/backend/src/test_support/mod.rs index 8b60d3e..a0ce6e6 100644 --- a/backend/src/test_support/mod.rs +++ b/backend/src/test_support/mod.rs @@ -176,6 +176,11 @@ impl TestApp { quickwit_index: None, worker_max_document_bytes: 200 * 1024 * 1024, upload_body_limit_bytes: 128 * 1024 * 1024, + service_timezone: "UTC".to_string(), + issued_at_date_order: "DMY".to_string(), + issued_at_filename_date_order: None, + issued_at_date_parser_locales: Vec::new(), + issued_at_ignore_dates: Vec::new(), webauthn_rp_id: Some("localhost".to_string()), webauthn_origin: Some("http://localhost".to_string()), webauthn_rp_name: "Papercrate".to_string(), diff --git a/backend/src/workers/analyze.rs b/backend/src/workers/analyze.rs index d997c9a..344cd87 100644 --- a/backend/src/workers/analyze.rs +++ b/backend/src/workers/analyze.rs @@ -11,6 +11,7 @@ use crate::{ use super::{ index::IndexDocumentTask, + issued_at::DetermineIssuedAtTask, job_execution_from_task_error, ocr::{GenerateOcrTask, OCR_TEXT_ASSET_TYPE}, taskflow::{ @@ -115,6 +116,8 @@ impl TaskPlanner for AnalyzePlanner { } } + tasks.push(Box::new(DetermineIssuedAtTask::new())); + if should_index { tasks.push(Box::new(IndexDocumentTask::new())); } diff --git a/backend/src/workers/issued_at.rs b/backend/src/workers/issued_at.rs new file mode 100644 index 0000000..216597f --- /dev/null +++ b/backend/src/workers/issued_at.rs @@ -0,0 +1,697 @@ +use std::time::Duration; + +use async_trait::async_trait; +use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, TimeZone, Utc}; +use diesel::prelude::*; +use once_cell::sync::Lazy; +use regex::Match; +use regex::Regex; +use serde_json::Value; +use tracing::{info, warn}; +use uuid::Uuid; + +use crate::issued_at::{DateOrder, IssuedAtSettings}; +use crate::schema::documents::dsl as documents_dsl; +use crate::workers::ocr::OCR_TEXT_ASSET_TYPE; +use crate::workers::taskflow::document::DocumentVersionTaskContext; +use crate::workers::taskflow::{Task, TaskContext, TaskError, TaskResult}; + +#[path = "issued_at_months.rs"] +mod issued_at_months; + +const MAX_FILENAME_CHARS: usize = 256; +const MAX_TEXT_CHARS: usize = 50_000; +const DATE_SEP_PATTERN: &str = r"[\s._/\-]+"; + +static YMD_RE: Lazy = + Lazy::new(|| Regex::new(r"(?u)\b(\d{4})[./-](\d{1,2})[./-](\d{1,2})\b").expect("ymd regex")); + +static NUMERIC_RE: Lazy = Lazy::new(|| { + Regex::new(r"(?u)\b(\d{1,2})[./-](\d{1,2})[./-](\d{2,4})\b").expect("numeric regex") +}); + +static DAY_MONTH_RE: Lazy = Lazy::new(|| { + Regex::new(&format!( + r"(?u)\b(\d{{1,2}})(?:st|nd|rd|th)?{SEP}({MONTH_PATTERN}){SEP}(\d{{2,4}})\b", + SEP = DATE_SEP_PATTERN, + MONTH_PATTERN = month_pattern() + )) + .expect("day month regex") +}); + +static MONTH_DAY_RE: Lazy = Lazy::new(|| { + Regex::new(&format!( + r"(?u)\b({MONTH_PATTERN}){SEP}(\d{{1,2}})(?:st|nd|rd|th)?(?:,)?{SEP}(\d{{2,4}})\b", + SEP = DATE_SEP_PATTERN, + MONTH_PATTERN = month_pattern() + )) + .expect("month day regex") +}); + +static MONTH_YEAR_RE: Lazy = Lazy::new(|| { + Regex::new(&format!( + r"(?u)\b({MONTH_PATTERN})[\s._-]*(\d{{4}})\b", + MONTH_PATTERN = month_pattern() + )) + .expect("month year regex") +}); + +fn month_pattern() -> &'static str { + issued_at_months::pattern() +} + +pub struct DetermineIssuedAtTask; + +impl DetermineIssuedAtTask { + pub fn new() -> Self { + Self + } +} + +#[async_trait] +impl Task for DetermineIssuedAtTask { + fn name(&self) -> &'static str { + "determine-issued-at" + } + + async fn execute(&self, ctx: &mut DocumentVersionTaskContext) -> TaskResult<()> { + let document = ctx.document().await?.clone(); + if document.issued_at.is_some() { + return Ok(()); + } + + let version = ctx.version().await?.clone(); + let settings = ctx.state().issued_at_settings(); + let now_utc = Utc::now(); + + let parser_hint = parser_supplied_date(&document.metadata, &version.metadata, &settings) + .and_then(|dt| settings.normalize_datetime(dt, now_utc)); + + let filename_candidate = settings.filename_date_order().and_then(|order| { + let normalized = normalize_content(&document.original_name, MAX_FILENAME_CHARS); + find_date_in_text(&normalized, order, &settings, now_utc) + }); + + let text_candidate = load_document_text(ctx).await?.and_then(|text| { + let normalized = normalize_content(&text, MAX_TEXT_CHARS); + find_date_in_text(&normalized, settings.date_order(), &settings, now_utc) + }); + + if let Some((final_date, source)) = parser_hint + .map(|dt| (dt, IssuedAtSource::Parser)) + .or_else(|| filename_candidate.map(|dt| (dt, IssuedAtSource::Filename))) + .or_else(|| text_candidate.map(|dt| (dt, IssuedAtSource::Text))) + { + persist_issued_at(ctx, document.id, final_date.naive_utc()).await?; + info!( + job_id = %ctx.job_id(), + document_id = %document.id, + issued_at = %final_date, + source = source.as_ref(), + "issued_at determined" + ); + } else { + info!( + job_id = %ctx.job_id(), + document_id = %document.id, + "no issued_at signals discovered; leaving unset" + ); + } + + Ok(()) + } +} + +async fn persist_issued_at( + ctx: &DocumentVersionTaskContext, + document_id: Uuid, + issued_at: NaiveDateTime, +) -> TaskResult<()> { + let tenant_id = ctx.tenant_id(); + let state = ctx.state().clone(); + let result = tokio::task::spawn_blocking(move || -> Result<(), String> { + let mut conn = state + .db_for_tenant(tenant_id) + .map_err(|err| format!("failed to scope connection: {err:?}"))?; + diesel::update( + documents_dsl::documents + .filter(documents_dsl::tenant_id.eq(tenant_id)) + .filter(documents_dsl::id.eq(document_id)), + ) + .set(documents_dsl::issued_at.eq(issued_at)) + .execute(&mut conn) + .map_err(|err| format!("failed to update issued_at: {err}"))?; + Ok(()) + }) + .await + .map_err(|err| { + TaskError::retry( + Duration::from_secs(30), + format!("issued_at task panicked: {err}"), + ) + })?; + result.map_err(|err| TaskError::retry(Duration::from_secs(30), err))?; + Ok(()) +} + +fn parser_supplied_date( + document_meta: &Value, + version_meta: &Value, + settings: &IssuedAtSettings, +) -> Option> { + metadata_datetime(document_meta) + .or_else(|| metadata_datetime(version_meta)) + .and_then(|raw| parse_hint_datetime(raw, settings)) +} + +fn metadata_datetime(value: &Value) -> Option<&str> { + match value { + Value::String(s) => Some(s.as_str()), + Value::Object(map) => { + for key in [ + "issued_at_override", + "issued_at", + "source_date", + "created_at", + ] { + if let Some(Value::String(s)) = map.get(key) { + return Some(s.as_str()); + } + } + if let Some(Value::Object(parser)) = map.get("parser") { + if let Some(Value::String(s)) = parser.get("issued_at") { + return Some(s.as_str()); + } + } + None + } + _ => None, + } +} + +fn parse_hint_datetime(raw: &str, settings: &IssuedAtSettings) -> Option> { + if let Ok(dt) = DateTime::parse_from_rfc3339(raw) { + return Some(dt.with_timezone(&Utc)); + } + + if let Ok(date) = NaiveDate::parse_from_str(raw, "%Y-%m-%d") { + return settings + .timezone() + .with_ymd_and_hms(date.year(), date.month(), date.day(), 0, 0, 0) + .single() + .map(|dt| dt.with_timezone(&Utc)); + } + + None +} + +fn normalize_content(input: &str, limit: usize) -> String { + let truncated: String = input.chars().take(limit).collect(); + let mut normalized = String::with_capacity(truncated.len()); + for ch in truncated.chars() { + if ch.is_control() { + normalized.push(' '); + } else { + normalized.extend(ch.to_lowercase()); + } + } + normalized +} + +fn find_date_in_text( + text: &str, + order: DateOrder, + settings: &IssuedAtSettings, + now_utc: DateTime, +) -> Option> { + collect_matches_with_spans(text, order, settings, now_utc) + .into_iter() + .map(|(_, dt)| dt) + .next() +} + +fn collect_matches_with_spans( + text: &str, + order: DateOrder, + settings: &IssuedAtSettings, + now_utc: DateTime, +) -> Vec<(usize, DateTime)> { + let mut matches: Vec<(usize, usize, DateTime)> = Vec::new(); + let mut push_date = |span: Option, date: NaiveDate| { + if let Some(dt) = settings.normalize_naive(date, now_utc) { + if let Some(span) = span { + let start = span.start(); + let end = span.end(); + if let Some(existing) = + matches + .iter_mut() + .find(|(existing_start, existing_end, _)| { + *existing_start != usize::MAX + && start < *existing_end + && *existing_start < end + }) + { + let existing_len = existing.1.saturating_sub(existing.0); + let new_len = end.saturating_sub(start); + if new_len > existing_len { + *existing = (start, end, dt); + } + return; + } + matches.push((start, end, dt)); + } else { + matches.push((usize::MAX, usize::MAX, dt)); + } + } + }; + + for caps in YMD_RE.captures_iter(text) { + let (Some(year_match), Some(month_match), Some(day_match)) = + (caps.get(1), caps.get(2), caps.get(3)) + else { + continue; + }; + let year = match year_match.as_str().parse::() { + Ok(value) => value, + Err(_) => continue, + }; + let month = match month_match.as_str().parse::() { + Ok(value) => value, + Err(_) => continue, + }; + let day = match day_match.as_str().parse::() { + Ok(value) => value, + Err(_) => continue, + }; + if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) { + push_date(caps.get(0), date); + } + } + + for caps in NUMERIC_RE.captures_iter(text) { + let (Some(first_match), Some(second_match), Some(year_match)) = + (caps.get(1), caps.get(2), caps.get(3)) + else { + continue; + }; + let first = match first_match.as_str().parse::() { + Ok(value) => value, + Err(_) => continue, + }; + let second = match second_match.as_str().parse::() { + Ok(value) => value, + Err(_) => continue, + }; + let year_raw = year_match.as_str(); + let mut year = match year_raw.parse::() { + Ok(value) => value, + Err(_) => continue, + }; + if year_raw.len() == 2 { + year += if year >= 70 { 1900 } else { 2000 }; + } + // NUMERIC_RE always captures a day-first form (dd[sep]mm[sep]yy(yy)); + // YMD layouts are handled earlier by YMD_RE, so YMD here is treated the + // same as DMY to avoid mis-parsing strings like 01-07-2024. + let (day, month) = match order { + DateOrder::Dmy | DateOrder::Ymd => (first, second), + DateOrder::Mdy => (second, first), + }; + if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) { + push_date(caps.get(0), date); + } + } + + for caps in DAY_MONTH_RE.captures_iter(text) { + let (Some(day_match), Some(month_match), Some(year_match)) = + (caps.get(1), caps.get(2), caps.get(3)) + else { + continue; + }; + let day_str = day_match + .as_str() + .trim_matches(|c: char| !c.is_ascii_digit()); + let day = match day_str.parse::() { + Ok(value) => value, + Err(_) => continue, + }; + let Some(month) = month_name_to_number(month_match.as_str(), settings) else { + continue; + }; + if year_match.as_str().len() < 3 { + continue; + } + let Some(year) = normalize_year(year_match.as_str()) else { + continue; + }; + if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) { + push_date(caps.get(0), date); + } + } + + for caps in MONTH_DAY_RE.captures_iter(text) { + let (Some(month_match), Some(day_match), Some(year_match)) = + (caps.get(1), caps.get(2), caps.get(3)) + else { + continue; + }; + let Some(month) = month_name_to_number(month_match.as_str(), settings) else { + continue; + }; + let day_str = day_match + .as_str() + .trim_matches(|c: char| !c.is_ascii_digit()); + let day = match day_str.parse::() { + Ok(value) => value, + Err(_) => continue, + }; + let Some(year) = normalize_year(year_match.as_str()) else { + continue; + }; + if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) { + push_date(caps.get(0), date); + } + } + + for caps in MONTH_YEAR_RE.captures_iter(text) { + let (Some(month_match), Some(year_match)) = (caps.get(1), caps.get(2)) else { + continue; + }; + let Some(month) = month_name_to_number(month_match.as_str(), settings) else { + continue; + }; + let year = match year_match.as_str().parse::() { + Ok(value) => value, + Err(_) => continue, + }; + if let Some(date) = NaiveDate::from_ymd_opt(year, month, 1) { + push_date(caps.get(0), date); + } + } + + matches.sort_by_key(|(start, _, _)| *start); + matches + .into_iter() + .map(|(start, _, dt)| (start, dt)) + .collect() +} + +fn normalize_year(raw: &str) -> Option { + if raw.len() == 2 { + let mut year = raw.parse::().ok()?; + year += if year >= 70 { 1900 } else { 2000 }; + Some(year) + } else { + raw.parse::().ok() + } +} + +fn month_name_to_number(value: &str, settings: &IssuedAtSettings) -> Option { + let normalized = value.trim(); + let variant = issued_at_months::variants() + .iter() + .find(|entry| entry.name == normalized)?; + if settings.locales().is_empty() || variant.locales.is_empty() { + return Some(variant.month); + } + if variant + .locales + .iter() + .any(|locale| settings.locales().contains(*locale)) + { + Some(variant.month) + } else { + None + } +} + +async fn load_document_text(ctx: &mut DocumentVersionTaskContext) -> TaskResult> { + let object_key = { + let assets = ctx.assets().await?; + assets + .get(OCR_TEXT_ASSET_TYPE) + .and_then(|asset| asset.objects.first()) + .map(|object| object.s3_key.clone()) + }; + + let Some(key) = object_key else { + return Ok(None); + }; + + match ctx.storage().get_object(&key).await { + Ok(bytes) => match String::from_utf8(bytes) { + Ok(mut text) => { + if text.len() > MAX_TEXT_CHARS { + text.truncate(MAX_TEXT_CHARS); + } + Ok(Some(text)) + } + Err(err) => { + warn!(error = %err, "ocr text asset not valid utf-8"); + Ok(None) + } + }, + Err(err) => { + warn!(error = %err, "failed to download ocr text for issued_at extractor"); + Ok(None) + } + } +} + +#[derive(Copy, Clone)] +enum IssuedAtSource { + Parser, + Filename, + Text, +} + +impl IssuedAtSource { + fn as_ref(&self) -> &'static str { + match self { + IssuedAtSource::Parser => "parser", + IssuedAtSource::Filename => "filename", + IssuedAtSource::Text => "text", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::AppConfig; + use serde::Deserialize; + use serde_yaml::Value as YamlValue; + + use once_cell::sync::Lazy; + + #[derive(Clone, Deserialize)] + struct CaseSuite { + cases: Vec, + } + + #[derive(Clone, Deserialize)] + struct CaseDefinition { + name: String, + parser: String, + #[serde(default)] + filename: Option, + #[serde(default)] + content: Option, + #[serde(default)] + settings: CaseSettings, + expected: ExpectedCase, + } + + #[derive(Clone, Default, Deserialize)] + struct CaseSettings { + #[serde(rename = "DATE_PARSER_LANGUAGES", default)] + date_parser_languages: Vec, + #[serde(rename = "FILENAME_DATE_ORDER")] + filename_date_order: Option, + #[serde(rename = "DATE_ORDER")] + date_order: Option, + #[serde(rename = "IGNORE_DATES", default)] + ignore_dates: Vec, + } + + #[derive(Clone, Deserialize)] + struct ExpectedCase { + mode: ExpectedMode, + #[serde(default)] + value: Option, + } + + #[derive(Clone, Deserialize, PartialEq)] + #[serde(rename_all = "lowercase")] + enum ExpectedMode { + None, + Single, + Multiple, + } + + static CASES: Lazy = Lazy::new(|| { + let raw = include_str!("../../tests/data/issued_at_cases.yaml"); + serde_yaml::from_str(raw).expect("failed to parse issued_at cases") + }); + + pub(crate) fn run_named_case(name: &str) { + let case = CASES + .cases + .iter() + .find(|case| case.name == name) + .unwrap_or_else(|| panic!("case '{}' not found", name)) + .clone(); + run_case(case); + } + + fn run_case(case: CaseDefinition) { + let mut config = base_config(); + if let Some(order) = case.settings.date_order { + config.issued_at_date_order = order; + } + config.issued_at_filename_date_order = case.settings.filename_date_order; + config.issued_at_date_parser_locales = case.settings.date_parser_languages; + config.issued_at_ignore_dates = case.settings.ignore_dates; + + let settings = IssuedAtSettings::from_config(&config); + let now_utc = Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).single().unwrap(); + let mut matches = Vec::new(); + + if let Some(filename) = &case.filename { + if let Some(order) = settings.filename_date_order() { + let normalized = normalize_content(filename, MAX_FILENAME_CHARS); + matches.extend( + collect_matches_with_spans(&normalized, order, &settings, now_utc) + .into_iter() + .map(|(_, dt)| dt.date_naive()), + ); + } + } + + if let Some(content) = &case.content { + let normalized = normalize_content(content, MAX_TEXT_CHARS); + matches.extend( + collect_matches_with_spans(&normalized, settings.date_order(), &settings, now_utc) + .into_iter() + .map(|(_, dt)| dt.date_naive()), + ); + } + + let actual: Vec = matches + .into_iter() + .map(|date| date.format("%Y-%m-%d").to_string()) + .collect(); + + match case.parser.as_str() { + "parse_date" => match case.expected.mode { + ExpectedMode::None => assert!( + actual.is_empty(), + "case '{}' expected no matches, got {:?}", + case.name, + actual + ), + ExpectedMode::Single => { + let expected = case.expected.single_value(); + assert!( + expected.is_some(), + "case '{}' is missing expected single value", + case.name + ); + assert_eq!( + actual.first(), + expected.as_ref(), + "case '{}' single mismatch", + case.name + ); + } + ExpectedMode::Multiple => panic!( + "case '{}' declares parse_date but expects multiple results", + case.name + ), + }, + "parse_date_generator" => { + let expected = case.expected.multiple_values().unwrap_or_default(); + assert_eq!(expected, actual, "case '{}' multiple mismatch", case.name); + } + other => panic!("unsupported parser '{}' in case {}", other, case.name), + } + } + + impl ExpectedCase { + fn single_value(&self) -> Option { + match self.value.as_ref()? { + YamlValue::String(value) => Some(value.clone()), + other => Some(other.as_str()?.to_string()), + } + } + + fn multiple_values(&self) -> Option> { + let list = match self.value.as_ref()? { + YamlValue::Sequence(seq) => seq, + _ => return None, + }; + Some( + list.iter() + .filter_map(|value| value.as_str().map(|s| s.to_string())) + .collect(), + ) + } + } + + fn base_config() -> AppConfig { + AppConfig { + database_url: "postgres://test".to_string(), + database_max_pool_size: 5, + server_host: "127.0.0.1".to_string(), + server_port: 0, + webdav_host: "127.0.0.1".to_string(), + webdav_port: 0, + jwt_secret: "secret".to_string(), + jwt_issuer: "issuer".to_string(), + jwt_audience: "audience".to_string(), + jwt_expiry_minutes: 60, + download_token_audience: "download".to_string(), + download_token_expiry_minutes: 60, + refresh_token_expiry_days: 30, + refresh_cookie_secure: false, + refresh_cookie_domain: None, + cors_allowed_origin: None, + proxy_downloads: false, + aws_endpoint_url: None, + aws_access_key_id: None, + aws_secret_access_key: None, + aws_region: "us-east-1".to_string(), + s3_bucket: "bucket".to_string(), + quickwit_endpoint: None, + quickwit_index: None, + worker_max_document_bytes: 100 * 1024 * 1024, + upload_body_limit_bytes: 64 * 1024 * 1024, + service_timezone: "UTC".to_string(), + issued_at_date_order: "DMY".to_string(), + issued_at_filename_date_order: None, + issued_at_date_parser_locales: Vec::new(), + issued_at_ignore_dates: Vec::new(), + webauthn_rp_id: Some("localhost".to_string()), + webauthn_origin: Some("http://localhost".to_string()), + webauthn_rp_name: "Papercrate".to_string(), + } + } + + include!(concat!(env!("OUT_DIR"), "/issued_at_generated_tests.rs")); + + #[test] + fn month_name_lookup_handles_turkish_variants() { + let settings = IssuedAtSettings::from_config(&base_config()); + assert_eq!(month_name_to_number("şubat", &settings), Some(2)); + assert_eq!(month_name_to_number("subat", &settings), Some(2)); + } + + #[test] + fn locale_filter_limits_month_names() { + let mut config = base_config(); + config.issued_at_date_parser_locales = vec!["tr".into()]; + let settings = IssuedAtSettings::from_config(&config); + assert_eq!(month_name_to_number("january", &settings), None); + assert_eq!(month_name_to_number("şubat", &settings), Some(2)); + } +} diff --git a/backend/src/workers/issued_at_months.rs b/backend/src/workers/issued_at_months.rs new file mode 100644 index 0000000..dc648a3 --- /dev/null +++ b/backend/src/workers/issued_at_months.rs @@ -0,0 +1,15 @@ +pub(super) struct MonthVariant { + pub name: &'static str, + pub month: u32, + pub locales: &'static [&'static str], +} + +include!(concat!(env!("OUT_DIR"), "/issued_at_months.rs")); + +pub(super) fn pattern() -> &'static str { + MONTH_PATTERN +} + +pub(super) fn variants() -> &'static [MonthVariant] { + MONTH_VARIANTS +} diff --git a/backend/src/workers/mod.rs b/backend/src/workers/mod.rs index 333046e..650667c 100644 --- a/backend/src/workers/mod.rs +++ b/backend/src/workers/mod.rs @@ -16,6 +16,7 @@ use taskflow::TaskError; pub mod analyze; pub mod common; pub mod index; +pub mod issued_at; pub mod ocr; pub mod purge; pub mod taskflow; diff --git a/backend/tests/data/issued_at_cases.yaml b/backend/tests/data/issued_at_cases.yaml new file mode 100644 index 0000000..8e58004 --- /dev/null +++ b/backend/tests/data/issued_at_cases.yaml @@ -0,0 +1,582 @@ +cases: + - name: date_format_1 + parser: parse_date + filename: null + content: "lorem ipsum 130218 lorem ipsum" + settings: {} + expected: + mode: none + + - name: date_format_2 + parser: parse_date + filename: null + content: "lorem ipsum 2018 lorem ipsum" + settings: {} + expected: + mode: none + + - name: date_format_3 + parser: parse_date + filename: null + content: "lorem ipsum 20180213 lorem ipsum" + settings: {} + expected: + mode: none + + - name: date_format_4 + parser: parse_date + filename: null + content: "lorem ipsum 13.02.2018 lorem ipsum" + settings: {} + expected: + mode: single + value: 2018-02-13 + + - name: date_format_5 + parser: parse_date + filename: null + content: "lorem ipsum 130218, 2018, 20180213 and lorem 13.02.2018 lorem ipsum" + settings: {} + expected: + mode: single + value: 2018-02-13 + + - name: date_format_6 + parser: parse_date + filename: null + content: | + lorem ipsum + Wohnort + 3100 + IBAN + AT87 4534 + 1234 + 1234 5678 + BIC + lorem ipsum + settings: {} + expected: + mode: none + + - name: date_format_7 + parser: parse_date + filename: null + content: | + lorem ipsum + März 2019 + lorem ipsum + settings: + DATE_PARSER_LANGUAGES: + - de + expected: + mode: single + value: 2019-03-01 + + - name: date_format_8 + parser: parse_date + filename: null + content: | + lorem ipsum + Wohnort + 3100 + IBAN + AT87 4534 + 1234 + 1234 5678 + BIC + lorem ipsum + März 2020 + settings: + DATE_PARSER_LANGUAGES: + - de + expected: + mode: single + value: 2020-03-01 + + - name: date_format_9 + parser: parse_date + filename: null + content: | + lorem ipsum + 27. Nullmonth 2020 + März 2020 + lorem ipsum + settings: + DATE_PARSER_LANGUAGES: + - de + expected: + mode: single + value: 2020-03-01 + + - name: date_format_10 + parser: parse_date + filename: null + content: "Customer Number Currency 22-MAR-2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-22 + + - name: date_format_11 + parser: parse_date + filename: null + content: "Customer Number Currency 22 MAR 2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-22 + + - name: date_format_12 + parser: parse_date + filename: null + content: "Customer Number Currency 22/MAR/2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-22 + + - name: date_format_13 + parser: parse_date + filename: null + content: "Customer Number Currency 22.MAR.2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-22 + + - name: date_format_14 + parser: parse_date + filename: null + content: "Customer Number Currency 22.MAR 2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-22 + + - name: date_format_15 + parser: parse_date + filename: null + content: "Customer Number Currency 22.MAR.22 Credit Card 1934829304" + settings: {} + expected: + mode: none + + - name: date_format_16 + parser: parse_date + filename: null + content: "Customer Number Currency 22.MAR,22 Credit Card 1934829304" + settings: {} + expected: + mode: none + + - name: date_format_17 + parser: parse_date + filename: null + content: "Customer Number Currency 22,MAR,2022 Credit Card 1934829304" + settings: {} + expected: + mode: none + + - name: date_format_18 + parser: parse_date + filename: null + content: "Customer Number Currency 22 MAR,2022 Credit Card 1934829304" + settings: {} + expected: + mode: none + + - name: date_format_19 + parser: parse_date + filename: null + content: "Customer Number Currency 21st MAR 2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-21 + + - name: date_format_20 + parser: parse_date + filename: null + content: "Customer Number Currency 22nd March 2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-22 + + - name: date_format_21 + parser: parse_date + filename: null + content: "Customer Number Currency 2nd MAR 2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-02 + + - name: date_format_22 + parser: parse_date + filename: null + content: "Customer Number Currency 23rd MAR 2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-23 + + - name: date_format_23 + parser: parse_date + filename: null + content: "Customer Number Currency 24th MAR 2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-24 + + - name: date_format_24 + parser: parse_date + filename: null + content: "Customer Number Currency 21-MAR-2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-21 + + - name: date_format_25 + parser: parse_date + filename: null + content: "Customer Number Currency 25TH MAR 2022 Credit Card 1934829304" + settings: {} + expected: + mode: single + value: 2022-03-25 + + - name: date_format_26 + parser: parse_date + filename: null + content: "CHASE 0 September 25, 2019 JPMorgan Chase Bank, NA. P0 Box 182051" + settings: {} + expected: + mode: single + value: 2019-09-25 + + - name: numeric_mdy_slash + parser: parse_date + filename: null + content: "03/17/2008" + settings: + DATE_ORDER: MDY + expected: + mode: single + value: 2008-03-17 + + - name: crazy_date_past + parser: parse_date + filename: null + content: "01-07-0590 00:00:00" + settings: {} + expected: + mode: none + + - name: crazy_date_future + parser: parse_date + filename: null + content: "01-07-2350 00:00:00" + settings: {} + expected: + mode: none + + - name: crazy_date_with_spaces + parser: parse_date + filename: null + content: "20 408000l 2475" + settings: {} + expected: + mode: none + + - name: utf_month_names_decembre + parser: parse_date + filename: null + content: "13 décembre 2023" + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 2023-12-13 + + - name: utf_month_names_aout + parser: parse_date + filename: null + content: "13 août 2022" + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 2022-08-13 + + - name: utf_month_names_marz + parser: parse_date + filename: null + content: "11 März 2020" + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 2020-03-11 + + - name: utf_month_names_ozujka + parser: parse_date + filename: null + content: "17. ožujka 2018." + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 2018-03-17 + + - name: utf_month_names_veljace + parser: parse_date + filename: null + content: "1. veljače 2016." + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 2016-02-01 + + - name: utf_month_names_unora + parser: parse_date + filename: null + content: "15. února 1985" + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 1985-02-15 + + - name: utf_month_names_zari + parser: parse_date + filename: null + content: "30. září 2011" + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 2011-09-30 + + - name: utf_month_names_kvetna + parser: parse_date + filename: null + content: "28. května 1990" + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 1990-05-28 + + - name: utf_month_names_grudzien + parser: parse_date + filename: null + content: "1. grudzień 1997" + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 1997-12-01 + + - name: utf_month_names_subat + parser: parse_date + filename: null + content: "17 Şubat 2024" + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 2024-02-17 + + - name: utf_month_names_agustos + parser: parse_date + filename: null + content: "30 Ağustos 2012" + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 2012-08-30 + + - name: utf_month_names_eylul + parser: parse_date + filename: null + content: "17 Eylül 2000" + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + expected: + mode: single + value: 2000-09-17 + + - name: utf_month_names_oktober + parser: parse_date + filename: null + content: "5. október 1992" + settings: + DATE_PARSER_LANGUAGES: + - fr + - de + - hr + - cs + - pl + - tr + - hu + expected: + mode: single + value: 1992-10-05 + + - name: multiple_dates + parser: parse_date_generator + filename: null + content: | + This text has multiple dates. + For example 02.02.2018, 22 July 2022 and December 2021. + But not 24-12-9999 because it's in the future... + settings: {} + expected: + mode: multiple + value: + - 2018-02-02 + - 2022-07-22 + - 2021-12-01 + + - name: filename_date_parse_valid_ymd + parser: parse_date + filename: /tmp/Scan-2022-04-01.pdf + content: "No date in here" + settings: + FILENAME_DATE_ORDER: YMD + expected: + mode: single + value: 2022-04-01 + + - name: filename_date_parse_valid_dmy + parser: parse_date + filename: /tmp/Scan-10.01.2021.pdf + content: "No date in here" + settings: + FILENAME_DATE_ORDER: DMY + expected: + mode: single + value: 2021-01-10 + + - name: filename_date_parse_invalid + parser: parse_date + filename: "/tmp/20 408000l 2475 - test.pdf" + content: "No date in here" + settings: + FILENAME_DATE_ORDER: YMD + expected: + mode: none + + - name: filename_date_ignored_use_content + parser: parse_date + filename: /tmp/Scan-2022-04-01.pdf + content: "The matching date is 24.03.2022" + settings: + FILENAME_DATE_ORDER: YMD + IGNORE_DATES: + - 2022-04-01 + expected: + mode: single + value: 2022-03-24 + + - name: ignored_dates_default_order + parser: parse_date + filename: null + content: "lorem ipsum 110319, 20200117 and lorem 13.02.2018 lorem ipsum" + settings: + IGNORE_DATES: + - 2019-11-03 + - 2020-01-17 + expected: + mode: single + value: 2018-02-13 + + - name: ignored_dates_order_ymd + parser: parse_date + filename: null + content: "lorem ipsum 190311, 20200117 and lorem 13.02.2018 lorem ipsum" + settings: + FILENAME_DATE_ORDER: YMD + IGNORE_DATES: + - 2019-11-03 + - 2020-01-17 + expected: + mode: single + value: 2018-02-13