nils/papercrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better date parsing

Browse Source
This commit is contained in:
nils
2025-11-11 01:36:17 +01:00
parent 8788cd3e25
commit 4ce4bd1a4b
14 changed files with 2413 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Cargo.lock
Generated
+97
View File
@@ -462,6 +462,28 @@ dependencies = [
"windows-link 0.2.1",
]
[[package]]
name = "chrono-tz"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e"
dependencies = [
"chrono",
"chrono-tz-build",
"phf",
]
[[package]]
name = "chrono-tz-build"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f"
dependencies = [
"parse-zoneinfo",
"phf",
"phf_codegen",
]
[[package]]
name = "clang-sys"
version = "1.8.1"
@@ -2100,6 +2122,7 @@ dependencies = [
"base64 0.22.1",
"bytes",
"chrono",
"chrono-tz",
"clap",
"diesel",
"diesel_migrations",
@@ -2117,6 +2140,7 @@ dependencies = [
"percent-encoding",
"quick-xml",
"rand 0.9.2",
"regex",
"reqwest",
"rust-s3",
"serde",
@@ -2124,6 +2148,7 @@ dependencies = [
"serde_bytes",
"serde_cbor_2",
"serde_json",
"serde_yaml",
"sha2",
"tempfile",
"thiserror 2.0.17",
@@ -2162,6 +2187,15 @@ dependencies = [
"windows-link 0.2.1",
]
[[package]]
name = "parse-zoneinfo"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24"
dependencies = [
"regex",
]
[[package]]
name = "password-hash"
version = "0.5.0"
@@ -2230,6 +2264,44 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "phf"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
dependencies = [
"phf_shared",
]
[[package]]
name = "phf_codegen"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
dependencies = [
"phf_generator",
"phf_shared",
]
[[package]]
name = "phf_generator"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
"phf_shared",
"rand 0.8.5",
]
[[package]]
name = "phf_shared"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
dependencies = [
"siphasher",
]
[[package]]
name = "pin-project-lite"
version = "0.2.16"
@@ -3001,6 +3073,19 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_yaml"
version = "0.9.34+deprecated"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
dependencies = [
"indexmap",
"itoa",
"ryu",
"serde",
"unsafe-libyaml",
]
[[package]]
name = "sha1"
version = "0.10.6"
@@ -3075,6 +3160,12 @@ dependencies = [
"time",
]
[[package]]
name = "siphasher"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
[[package]]
name = "slab"
version = "0.4.11"
@@ -3555,6 +3646,12 @@ version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
[[package]]
name = "unsafe-libyaml"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
[[package]]
name = "untrusted"
version = "0.9.0"
backend/Cargo.toml
+9
View File
@@ -45,6 +45,7 @@ quick-xml = "0.38"
futures-util = "0.3"
url = "2.5"
once_cell = "1.19"
regex = "1.11"
utoipa = { version = "4.2", default-features = false, features = ["chrono", "uuid", "preserve_order"] }
clap = { version = "4.5", features = ["derive"] }
@@ -63,10 +64,18 @@ serde_cbor_2 = "0.13"
rand = "0.9"
hyper = "1.2"
http-body-util = "0.1"
chrono-tz = "0.8"
[dev-dependencies]
once_cell = "1.19"
webauthn-rs-core = "0.5"
serde_yaml = "0.9"
[build-dependencies]
serde = { version = "1.0", features = ["derive"] }
serde_yaml = "0.9"
serde_json = "1.0"
regex = "1.11"
[[bin]]
name = "backend"
backend/build.rs
+123
View File
@@ -0,0 +1,123 @@
use std::env;
use std::fs;
use std::path::PathBuf;
use regex::escape;
use serde::Deserialize;
#[derive(Deserialize)]
struct CaseSuite {
cases: Vec<CaseName>,
}
#[derive(Deserialize)]
struct CaseName {
name: String,
}
#[derive(Deserialize)]
struct MonthSuite {
months: Vec<MonthDefinition>,
}
#[derive(Deserialize)]
struct MonthDefinition {
name: String,
month: u32,
#[serde(default)]
locales: Vec<String>,
}
fn sanitize(name: &str) -> String {
let mut out = String::with_capacity(name.len());
for ch in name.chars() {
if ch.is_ascii_alphanumeric() {
out.push(ch);
} else {
out.push('_');
}
}
if out.is_empty() {
"case".to_string()
} else if out.chars().next().unwrap().is_ascii_digit() {
format!("_{}", out)
} else {
out
}
}
fn quote(value: &str) -> String {
serde_json::to_string(value).expect("string literal")
}
fn generate_tests() -> Result<String, Box<dyn std::error::Error>> {
let yaml_path = PathBuf::from("tests/data/issued_at_cases.yaml");
let contents = fs::read_to_string(&yaml_path)?;
let suite: CaseSuite = serde_yaml::from_str(&contents)?;
let mut output =
String::from("#[cfg(test)]\npub mod issued_at_generated_tests {\n use super::*;\n");
for case in suite.cases {
let ident = sanitize(&case.name);
output.push_str(&format!(
" #[test]\n fn {}() {{\n run_named_case(\"{}\");\n }}\n",
ident, case.name
));
}
output.push_str("}\n");
Ok(output)
}
fn generate_months() -> Result<String, Box<dyn std::error::Error>> {
let yaml_path = PathBuf::from("resources/issued_at_months.yaml");
let contents = fs::read_to_string(&yaml_path)?;
let suite: MonthSuite = serde_yaml::from_str(&contents)?;
let mut pattern_parts = Vec::with_capacity(suite.months.len());
let mut entries = String::new();
for entry in &suite.months {
pattern_parts.push(escape(&entry.name));
let locales_literal = if entry.locales.is_empty() {
"&[]".to_string()
} else {
let joined = entry
.locales
.iter()
.map(|loc| quote(loc))
.collect::<Vec<_>>()
.join(", ");
format!("&[{}]", joined)
};
entries.push_str(&format!(
" MonthVariant {{ name: {}, month: {}, locales: {} }},\n",
quote(&entry.name),
entry.month,
locales_literal
));
}
let pattern_literal = quote(&pattern_parts.join("|"));
let output = format!(
"pub(super) static MONTH_VARIANTS: &[MonthVariant] = &[\n{entries}];\n\n",
entries = entries
) + &format!(
"pub(super) const MONTH_PATTERN: &str = {};\n",
pattern_literal
);
Ok(output)
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("cargo:rerun-if-changed=tests/data/issued_at_cases.yaml");
println!("cargo:rerun-if-changed=resources/issued_at_months.yaml");
let out_dir = PathBuf::from(env::var("OUT_DIR")?);
fs::write(
out_dir.join("issued_at_generated_tests.rs"),
generate_tests()?,
)?;
fs::write(out_dir.join("issued_at_months.rs"), generate_months()?)?;
Ok(())
}
backend/resources/issued_at_months.yaml
+648
View File
@@ -0,0 +1,648 @@
months:
- name: "january"
month: 1
locales:
- "en"
- name: "jan"
month: 1
locales:
- "en"
- name: "janvier"
month: 1
locales:
- "fr"
- name: "janv"
month: 1
locales:
- "fr"
- name: "januar"
month: 1
locales:
- "de"
- name: "janu\u00e1r"
month: 1
locales:
- "hu"
- name: "leden"
month: 1
locales:
- "cs"
- name: "sije\u010danj"
month: 1
locales:
- "hr"
- name: "sijecanj"
month: 1
locales:
- "hr"
- name: "stycze\u0144"
month: 1
locales:
- "pl"
- name: "styczen"
month: 1
locales:
- "pl"
- name: "ocak"
month: 1
locales:
- "tr"
- name: "february"
month: 2
locales:
- "en"
- name: "feb"
month: 2
locales:
- "en"
- name: "f\u00e9vrier"
month: 2
locales:
- "fr"
- name: "fevrier"
month: 2
locales:
- "fr"
- name: "f\u00e9vr"
month: 2
locales:
- "fr"
- name: "fevr"
month: 2
locales:
- "fr"
- name: "februar"
month: 2
locales:
- "de"
- name: "\u00fanor"
month: 2
locales:
- "cs"
- name: "unor"
month: 2
locales:
- "cs"
- name: "\u00fanora"
month: 2
locales:
- "cs"
- name: "unora"
month: 2
locales:
- "cs"
- name: "velja\u010da"
month: 2
locales:
- "hr"
- name: "veljaca"
month: 2
locales:
- "hr"
- name: "velja\u010de"
month: 2
locales:
- "hr"
- name: "veljace"
month: 2
locales:
- "hr"
- name: "luty"
month: 2
locales:
- "pl"
- name: "\u015fubat"
month: 2
locales:
- "tr"
- name: "subat"
month: 2
locales:
- "tr"
- name: "march"
month: 3
locales:
- "en"
- name: "mar"
month: 3
locales:
- "en"
- name: "m\u00e4rz"
month: 3
locales:
- "de"
- name: "maerz"
month: 3
locales:
- "de"
- name: "mars"
month: 3
locales:
- "fr"
- name: "m\u00e4r"
month: 3
locales:
- "de"
- name: "marz"
month: 3
locales:
- "de"
- name: "b\u0159ezen"
month: 3
locales:
- "cs"
- name: "brezen"
month: 3
locales:
- "cs"
- name: "b\u0159ezna"
month: 3
locales:
- "cs"
- name: "brezna"
month: 3
locales:
- "cs"
- name: "o\u017eujak"
month: 3
locales:
- "hr"
- name: "ozujak"
month: 3
locales:
- "hr"
- name: "o\u017eujka"
month: 3
locales:
- "hr"
- name: "ozujka"
month: 3
locales:
- "hr"
- name: "marzec"
month: 3
locales:
- "pl"
- name: "mart"
month: 3
locales:
- "sr"
- "bs"
- name: "m\u00e1rcius"
month: 3
locales:
- "hu"
- name: "martie"
month: 3
locales:
- "ro"
- name: "april"
month: 4
locales:
- "en"
- name: "apr"
month: 4
locales:
- "en"
- name: "avril"
month: 4
locales:
- "fr"
- name: "abril"
month: 4
locales:
- "es"
- "pt"
- name: "duben"
month: 4
locales:
- "cs"
- name: "travanj"
month: 4
locales:
- "hr"
- name: "nisan"
month: 4
locales:
- "tr"
- name: "kwiecie\u0144"
month: 4
locales:
- "pl"
- name: "kwiecien"
month: 4
locales:
- "pl"
- name: "aprile"
month: 4
locales:
- "it"
- name: "may"
month: 5
locales:
- "en"
- name: "mai"
month: 5
locales:
- "fr"
- "de"
- name: "mayo"
month: 5
locales:
- "es"
- name: "kv\u011bten"
month: 5
locales:
- "cs"
- name: "kveten"
month: 5
locales:
- "cs"
- name: "kv\u011btna"
month: 5
locales:
- "cs"
- name: "kvetna"
month: 5
locales:
- "cs"
- name: "svibanj"
month: 5
locales:
- "hr"
- name: "maj"
month: 5
locales:
- "pl"
- "bs"
- "sr"
- "hr"
- name: "may\u0131s"
month: 5
locales:
- "tr"
- name: "mayis"
month: 5
locales:
- "tr"
- name: "maggio"
month: 5
locales:
- "it"
- name: "m\u00e1j"
month: 5
locales:
- "hu"
- "sk"
- name: "june"
month: 6
locales:
- "en"
- name: "jun"
month: 6
locales:
- "en"
- "de"
- name: "juin"
month: 6
locales:
- "fr"
- name: "junio"
month: 6
locales:
- "es"
- name: "juni"
month: 6
locales:
- "de"
- name: "\u010derven"
month: 6
locales:
- "cs"
- name: "cerven"
month: 6
locales:
- "cs"
- name: "lipanj"
month: 6
locales:
- "hr"
- name: "haziran"
month: 6
locales:
- "tr"
- name: "czerwiec"
month: 6
locales:
- "pl"
- name: "j\u00fanius"
month: 6
locales:
- "hu"
- name: "giugno"
month: 6
locales:
- "it"
- name: "july"
month: 7
locales:
- "en"
- name: "jul"
month: 7
locales:
- "en"
- "de"
- name: "juillet"
month: 7
locales:
- "fr"
- name: "julio"
month: 7
locales:
- "es"
- name: "temmuz"
month: 7
locales:
- "tr"
- name: "\u010dervenec"
month: 7
locales:
- "cs"
- name: "cervenec"
month: 7
locales:
- "cs"
- name: "srpanj"
month: 7
locales:
- "hr"
- name: "lipiec"
month: 7
locales:
- "pl"
- name: "luglio"
month: 7
locales:
- "it"
- name: "j\u00falius"
month: 7
locales:
- "hu"
- name: "august"
month: 8
locales:
- "en"
- name: "aug"
month: 8
locales:
- "en"
- "de"
- name: "ao\u00fbt"
month: 8
locales:
- "fr"
- name: "aout"
month: 8
locales:
- "fr"
- name: "agosto"
month: 8
locales:
- "es"
- "pt"
- "it"
- name: "a\u011fustos"
month: 8
locales:
- "tr"
- name: "agustos"
month: 8
locales:
- "tr"
- name: "kolovoz"
month: 8
locales:
- "hr"
- name: "srpen"
month: 8
locales:
- "cs"
- name: "sierpie\u0144"
month: 8
locales:
- "pl"
- name: "sierpien"
month: 8
locales:
- "pl"
- name: "augustus"
month: 8
locales:
- "nl"
- name: "agost"
month: 8
locales:
- "it"
- "ca"
- name: "september"
month: 9
locales:
- "en"
- name: "sept"
month: 9
locales:
- "en"
- "fr"
- name: "sep"
month: 9
locales:
- "en"
- name: "septembre"
month: 9
locales:
- "fr"
- name: "septiembre"
month: 9
locales:
- "es"
- name: "eyl\u00fcl"
month: 9
locales:
- "tr"
- name: "eylul"
month: 9
locales:
- "tr"
- name: "z\u00e1\u0159\u00ed"
month: 9
locales:
- "cs"
- name: "zari"
month: 9
locales:
- "cs"
- name: "rujan"
month: 9
locales:
- "hr"
- name: "wrzesie\u0144"
month: 9
locales:
- "pl"
- name: "wrzesien"
month: 9
locales:
- "pl"
- name: "septembrie"
month: 9
locales:
- "ro"
- name: "settembre"
month: 9
locales:
- "it"
- name: "october"
month: 10
locales:
- "en"
- name: "oct"
month: 10
locales:
- "en"
- name: "oktober"
month: 10
locales:
- "de"
- name: "okt\u00f3ber"
month: 10
locales:
- "hu"
- name: "octobre"
month: 10
locales:
- "fr"
- name: "octubre"
month: 10
locales:
- "es"
- name: "\u0159\u00edjen"
month: 10
locales:
- "cs"
- name: "rijen"
month: 10
locales:
- "hr"
- name: "pa\u017adziernik"
month: 10
locales:
- "pl"
- name: "pazdziernik"
month: 10
locales:
- "pl"
- name: "ekim"
month: 10
locales:
- "tr"
- name: "octombrie"
month: 10
locales:
- "ro"
- name: "november"
month: 11
locales:
- "en"
- name: "nov"
month: 11
locales:
- "en"
- name: "novembre"
month: 11
locales:
- "fr"
- name: "noviembre"
month: 11
locales:
- "es"
- name: "studeni"
month: 11
locales:
- "hr"
- name: "kas\u0131m"
month: 11
locales:
- "tr"
- name: "kasim"
month: 11
locales:
- "tr"
- name: "listopad"
month: 11
locales:
- "pl"
- "cs"
- name: "novembro"
month: 11
locales:
- "pt"
- name: "listopadu"
month: 11
locales:
- "cs"
- name: "noiembrie"
month: 11
locales:
- "ro"
- name: "december"
month: 12
locales:
- "en"
- name: "dec"
month: 12
locales:
- "en"
- name: "dezember"
month: 12
locales:
- "de"
- name: "d\u00e9cembre"
month: 12
locales:
- "fr"
- name: "decembre"
month: 12
locales:
- "fr"
- name: "prosinec"
month: 12
locales:
- "cs"
- name: "prosinac"
month: 12
locales:
- "hr"
- name: "grudzie\u0144"
month: 12
locales:
- "pl"
- name: "grudzien"
month: 12
locales:
- "pl"
- name: "grudnia"
month: 12
locales:
- "pl"
- name: "aral\u0131k"
month: 12
locales:
- "tr"
- name: "aralik"
month: 12
locales:
- "tr"
- name: "decembrie"
month: 12
locales:
- "ro"
backend/src/config.rs
+57
View File
@@ -1,6 +1,7 @@
use anyhow::{Context, Result};
use url::Url;
use serde::de::Deserializer;
use serde::Deserialize;
use serde_aux::field_attributes::deserialize_bool_from_anything;
@@ -60,6 +61,16 @@ pub struct AppConfig {
pub worker_max_document_bytes: u64,
#[serde(default = "default_upload_body_limit_bytes")]
pub upload_body_limit_bytes: u64,
#[serde(default = "default_service_timezone")]
pub service_timezone: String,
#[serde(default = "default_issued_at_date_order")]
pub issued_at_date_order: String,
#[serde(default)]
pub issued_at_filename_date_order: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_list")]
pub issued_at_date_parser_locales: Vec<String>,
#[serde(default, deserialize_with = "deserialize_string_list")]
pub issued_at_ignore_dates: Vec<String>,
#[serde(default)]
pub webauthn_rp_id: Option<String>,
#[serde(default)]
@@ -183,6 +194,52 @@ fn default_upload_body_limit_bytes() -> u64 {
128 * 1024 * 1024
}
fn default_service_timezone() -> String {
"UTC".to_string()
}
fn default_issued_at_date_order() -> String {
"DMY".to_string()
}
fn deserialize_string_list<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum Helper {
List(Vec<String>),
Single(String),
}
let helper = Option::<Helper>::deserialize(deserializer)?;
let mut values = Vec::new();
if let Some(helper) = helper {
match helper {
Helper::List(list) => {
for entry in list {
let trimmed = entry.trim();
if !trimmed.is_empty() {
values.push(trimmed.to_string());
}
}
}
Helper::Single(value) => {
for part in value.split(',') {
let trimmed = part.trim();
if !trimmed.is_empty() {
values.push(trimmed.to_string());
}
}
}
}
}
Ok(values)
}
fn default_webauthn_rp_name() -> String {
"Papercrate".to_string()
}
backend/src/issued_at.rs
+167
View File
@@ -0,0 +1,167 @@
use std::collections::HashSet;
use chrono::{DateTime, Datelike, NaiveDate, TimeZone, Utc};
use chrono_tz::Tz;
use once_cell::sync::Lazy;
use tracing::warn;
use crate::config::AppConfig;
#[derive(Clone, Copy, Debug)]
pub enum DateOrder {
Dmy,
Mdy,
Ymd,
}
impl DateOrder {
pub fn parse(value: &str) -> Self {
Self::try_parse(value).unwrap_or(DateOrder::Dmy)
}
pub fn try_parse(value: &str) -> Option<Self> {
match value.trim().to_ascii_uppercase().as_str() {
"YMD" => Some(DateOrder::Ymd),
"MDY" => Some(DateOrder::Mdy),
"DMY" => Some(DateOrder::Dmy),
_ => None,
}
}
}
static MIN_ISSUED_AT_DATE: Lazy<NaiveDate> =
Lazy::new(|| NaiveDate::from_ymd_opt(1901, 1, 1).expect("valid minimum issued_at date"));
#[derive(Clone, Debug)]
pub struct IssuedAtSettings {
pub timezone: Tz,
pub date_order: DateOrder,
pub filename_date_order: Option<DateOrder>,
pub locales: HashSet<String>,
pub ignore_dates: HashSet<NaiveDate>,
pub min_date: NaiveDate,
}
impl IssuedAtSettings {
pub fn from_config(config: &AppConfig) -> Self {
let timezone = config.service_timezone.parse::<Tz>().unwrap_or_else(|_| {
warn!(
timezone = %config.service_timezone,
"invalid service timezone configured; falling back to UTC"
);
chrono_tz::UTC
});
let date_order = DateOrder::parse(&config.issued_at_date_order);
let filename_date_order =
config
.issued_at_filename_date_order
.as_deref()
.and_then(|value| {
DateOrder::try_parse(value).or_else(|| {
warn!(value, "invalid issued_at filename date order; ignoring");
None
})
});
let locales = config
.issued_at_date_parser_locales
.iter()
.filter_map(|value| {
let trimmed = value.trim();
if trimmed.is_empty() {
None
} else {
Some(trimmed.to_ascii_lowercase())
}
})
.collect::<HashSet<_>>();
// Ignore dates are evaluated after normalizing candidate timestamps to
// the configured service timezone, so administrators should provide
// local calendar dates rather than UTC midnights.
let ignore_dates = config
.issued_at_ignore_dates
.iter()
.filter_map(|value| {
let trimmed = value.trim();
if trimmed.is_empty() {
return None;
}
match NaiveDate::parse_from_str(trimmed, "%Y-%m-%d") {
Ok(date) => Some(date),
Err(err) => {
warn!(value = trimmed, error = %err, "invalid issued_at ignore date");
None
}
}
})
.collect();
Self {
timezone,
date_order,
filename_date_order,
locales,
ignore_dates,
min_date: *MIN_ISSUED_AT_DATE,
}
}
/// Returns the immutable (lowercase) locale allowlist supplied via config.
pub fn locales(&self) -> &HashSet<String> {
&self.locales
}
/// Returns the immutable set of local-calendar dates that should be ignored.
pub fn ignore_dates(&self) -> &HashSet<NaiveDate> {
&self.ignore_dates
}
/// Returns the configured service timezone (copy type).
pub fn timezone(&self) -> Tz {
self.timezone
}
pub fn date_order(&self) -> DateOrder {
self.date_order
}
pub fn filename_date_order(&self) -> Option<DateOrder> {
self.filename_date_order
}
pub fn normalize_naive(
&self,
date: NaiveDate,
now_utc: chrono::DateTime<Utc>,
) -> Option<DateTime<Utc>> {
if !self.is_valid_with_now(date, now_utc) {
return None;
}
self.timezone
.with_ymd_and_hms(date.year(), date.month(), date.day(), 0, 0, 0)
.earliest()
.map(|dt| dt.with_timezone(&Utc))
}
pub fn normalize_datetime(
&self,
dt: chrono::DateTime<Utc>,
now_utc: chrono::DateTime<Utc>,
) -> Option<DateTime<Utc>> {
let local_date = dt.with_timezone(&self.timezone).date_naive();
self.normalize_naive(local_date, now_utc)
}
fn is_valid_with_now(&self, date: NaiveDate, now_utc: chrono::DateTime<Utc>) -> bool {
if date < self.min_date {
return false;
}
let now_local = now_utc.with_timezone(&self.timezone).date_naive();
if date > now_local {
return false;
}
!self.ignore_dates.contains(&date)
}
}
backend/src/lib.rs
+1
View File
@@ -4,6 +4,7 @@ pub mod db;
pub mod documents;
pub mod error;
pub mod http;
pub mod issued_at;
pub mod jobs;
pub mod models;
pub mod openapi;
backend/src/state.rs
+8
View File
@@ -11,6 +11,7 @@ use crate::{
config::AppConfig,
db::PgPool,
error::{AppError, AppResult},
issued_at::IssuedAtSettings,
storage::{ObjectStorage, TenantStorage},
tenants::{apply_tenant_guc, clear_tenant_context, clear_user_guc, TenantService},
};
@@ -25,6 +26,7 @@ pub struct AppState {
pub jwt: JwtService,
pub tenants: TenantService,
pub passkeys: Option<PasskeyService>,
issued_at: Arc<IssuedAtSettings>,
}
impl AppState {
@@ -47,6 +49,7 @@ impl AppState {
storage: Arc<dyn ObjectStorage>,
jwt: JwtService,
) -> Self {
let issued_at = Arc::new(IssuedAtSettings::from_config(&config));
let config = Arc::new(config);
let tenants = TenantService::new(pool.clone());
@@ -65,6 +68,7 @@ impl AppState {
jwt,
tenants,
passkeys,
issued_at,
}
}
@@ -96,4 +100,8 @@ impl AppState {
AppError::internal("tenant storage error")
})
}
pub fn issued_at_settings(&self) -> Arc<IssuedAtSettings> {
self.issued_at.clone()
}
}
backend/src/test_support/mod.rs
+5
View File
@@ -176,6 +176,11 @@ impl TestApp {
quickwit_index: None,
worker_max_document_bytes: 200 * 1024 * 1024,
upload_body_limit_bytes: 128 * 1024 * 1024,
service_timezone: "UTC".to_string(),
issued_at_date_order: "DMY".to_string(),
issued_at_filename_date_order: None,
issued_at_date_parser_locales: Vec::new(),
issued_at_ignore_dates: Vec::new(),
webauthn_rp_id: Some("localhost".to_string()),
webauthn_origin: Some("http://localhost".to_string()),
webauthn_rp_name: "Papercrate".to_string(),
backend/src/workers/analyze.rs
+3
View File
@@ -11,6 +11,7 @@ use crate::{
use super::{
index::IndexDocumentTask,
issued_at::DetermineIssuedAtTask,
job_execution_from_task_error,
ocr::{GenerateOcrTask, OCR_TEXT_ASSET_TYPE},
taskflow::{
@@ -115,6 +116,8 @@ impl TaskPlanner<DocumentVersionTaskContext> for AnalyzePlanner {
}
}
tasks.push(Box::new(DetermineIssuedAtTask::new()));
if should_index {
tasks.push(Box::new(IndexDocumentTask::new()));
}
backend/src/workers/issued_at.rs
+697
View File
@@ -0,0 +1,697 @@
use std::time::Duration;
use async_trait::async_trait;
use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, TimeZone, Utc};
use diesel::prelude::*;
use once_cell::sync::Lazy;
use regex::Match;
use regex::Regex;
use serde_json::Value;
use tracing::{info, warn};
use uuid::Uuid;
use crate::issued_at::{DateOrder, IssuedAtSettings};
use crate::schema::documents::dsl as documents_dsl;
use crate::workers::ocr::OCR_TEXT_ASSET_TYPE;
use crate::workers::taskflow::document::DocumentVersionTaskContext;
use crate::workers::taskflow::{Task, TaskContext, TaskError, TaskResult};
#[path = "issued_at_months.rs"]
mod issued_at_months;
const MAX_FILENAME_CHARS: usize = 256;
const MAX_TEXT_CHARS: usize = 50_000;
const DATE_SEP_PATTERN: &str = r"[\s._/\-]+";
static YMD_RE: Lazy<Regex> =
Lazy::new(|| Regex::new(r"(?u)\b(\d{4})[./-](\d{1,2})[./-](\d{1,2})\b").expect("ymd regex"));
static NUMERIC_RE: Lazy<Regex> = Lazy::new(|| {
Regex::new(r"(?u)\b(\d{1,2})[./-](\d{1,2})[./-](\d{2,4})\b").expect("numeric regex")
});
static DAY_MONTH_RE: Lazy<Regex> = Lazy::new(|| {
Regex::new(&format!(
r"(?u)\b(\d{{1,2}})(?:st|nd|rd|th)?{SEP}({MONTH_PATTERN}){SEP}(\d{{2,4}})\b",
SEP = DATE_SEP_PATTERN,
MONTH_PATTERN = month_pattern()
))
.expect("day month regex")
});
static MONTH_DAY_RE: Lazy<Regex> = Lazy::new(|| {
Regex::new(&format!(
r"(?u)\b({MONTH_PATTERN}){SEP}(\d{{1,2}})(?:st|nd|rd|th)?(?:,)?{SEP}(\d{{2,4}})\b",
SEP = DATE_SEP_PATTERN,
MONTH_PATTERN = month_pattern()
))
.expect("month day regex")
});
static MONTH_YEAR_RE: Lazy<Regex> = Lazy::new(|| {
Regex::new(&format!(
r"(?u)\b({MONTH_PATTERN})[\s._-]*(\d{{4}})\b",
MONTH_PATTERN = month_pattern()
))
.expect("month year regex")
});
fn month_pattern() -> &'static str {
issued_at_months::pattern()
}
pub struct DetermineIssuedAtTask;
impl DetermineIssuedAtTask {
pub fn new() -> Self {
Self
}
}
#[async_trait]
impl Task<DocumentVersionTaskContext> for DetermineIssuedAtTask {
fn name(&self) -> &'static str {
"determine-issued-at"
}
async fn execute(&self, ctx: &mut DocumentVersionTaskContext) -> TaskResult<()> {
let document = ctx.document().await?.clone();
if document.issued_at.is_some() {
return Ok(());
}
let version = ctx.version().await?.clone();
let settings = ctx.state().issued_at_settings();
let now_utc = Utc::now();
let parser_hint = parser_supplied_date(&document.metadata, &version.metadata, &settings)
.and_then(|dt| settings.normalize_datetime(dt, now_utc));
let filename_candidate = settings.filename_date_order().and_then(|order| {
let normalized = normalize_content(&document.original_name, MAX_FILENAME_CHARS);
find_date_in_text(&normalized, order, &settings, now_utc)
});
let text_candidate = load_document_text(ctx).await?.and_then(|text| {
let normalized = normalize_content(&text, MAX_TEXT_CHARS);
find_date_in_text(&normalized, settings.date_order(), &settings, now_utc)
});
if let Some((final_date, source)) = parser_hint
.map(|dt| (dt, IssuedAtSource::Parser))
.or_else(|| filename_candidate.map(|dt| (dt, IssuedAtSource::Filename)))
.or_else(|| text_candidate.map(|dt| (dt, IssuedAtSource::Text)))
{
persist_issued_at(ctx, document.id, final_date.naive_utc()).await?;
info!(
job_id = %ctx.job_id(),
document_id = %document.id,
issued_at = %final_date,
source = source.as_ref(),
"issued_at determined"
);
} else {
info!(
job_id = %ctx.job_id(),
document_id = %document.id,
"no issued_at signals discovered; leaving unset"
);
}
Ok(())
}
}
async fn persist_issued_at(
ctx: &DocumentVersionTaskContext,
document_id: Uuid,
issued_at: NaiveDateTime,
) -> TaskResult<()> {
let tenant_id = ctx.tenant_id();
let state = ctx.state().clone();
let result = tokio::task::spawn_blocking(move || -> Result<(), String> {
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("failed to scope connection: {err:?}"))?;
diesel::update(
documents_dsl::documents
.filter(documents_dsl::tenant_id.eq(tenant_id))
.filter(documents_dsl::id.eq(document_id)),
)
.set(documents_dsl::issued_at.eq(issued_at))
.execute(&mut conn)
.map_err(|err| format!("failed to update issued_at: {err}"))?;
Ok(())
})
.await
.map_err(|err| {
TaskError::retry(
Duration::from_secs(30),
format!("issued_at task panicked: {err}"),
)
})?;
result.map_err(|err| TaskError::retry(Duration::from_secs(30), err))?;
Ok(())
}
fn parser_supplied_date(
document_meta: &Value,
version_meta: &Value,
settings: &IssuedAtSettings,
) -> Option<DateTime<Utc>> {
metadata_datetime(document_meta)
.or_else(|| metadata_datetime(version_meta))
.and_then(|raw| parse_hint_datetime(raw, settings))
}
fn metadata_datetime(value: &Value) -> Option<&str> {
match value {
Value::String(s) => Some(s.as_str()),
Value::Object(map) => {
for key in [
"issued_at_override",
"issued_at",
"source_date",
"created_at",
] {
if let Some(Value::String(s)) = map.get(key) {
return Some(s.as_str());
}
}
if let Some(Value::Object(parser)) = map.get("parser") {
if let Some(Value::String(s)) = parser.get("issued_at") {
return Some(s.as_str());
}
}
None
}
_ => None,
}
}
fn parse_hint_datetime(raw: &str, settings: &IssuedAtSettings) -> Option<DateTime<Utc>> {
if let Ok(dt) = DateTime::parse_from_rfc3339(raw) {
return Some(dt.with_timezone(&Utc));
}
if let Ok(date) = NaiveDate::parse_from_str(raw, "%Y-%m-%d") {
return settings
.timezone()
.with_ymd_and_hms(date.year(), date.month(), date.day(), 0, 0, 0)
.single()
.map(|dt| dt.with_timezone(&Utc));
}
None
}
fn normalize_content(input: &str, limit: usize) -> String {
let truncated: String = input.chars().take(limit).collect();
let mut normalized = String::with_capacity(truncated.len());
for ch in truncated.chars() {
if ch.is_control() {
normalized.push(' ');
} else {
normalized.extend(ch.to_lowercase());
}
}
normalized
}
fn find_date_in_text(
text: &str,
order: DateOrder,
settings: &IssuedAtSettings,
now_utc: DateTime<Utc>,
) -> Option<DateTime<Utc>> {
collect_matches_with_spans(text, order, settings, now_utc)
.into_iter()
.map(|(_, dt)| dt)
.next()
}
fn collect_matches_with_spans(
text: &str,
order: DateOrder,
settings: &IssuedAtSettings,
now_utc: DateTime<Utc>,
) -> Vec<(usize, DateTime<Utc>)> {
let mut matches: Vec<(usize, usize, DateTime<Utc>)> = Vec::new();
let mut push_date = |span: Option<Match>, date: NaiveDate| {
if let Some(dt) = settings.normalize_naive(date, now_utc) {
if let Some(span) = span {
let start = span.start();
let end = span.end();
if let Some(existing) =
matches
.iter_mut()
.find(|(existing_start, existing_end, _)| {
*existing_start != usize::MAX
&& start < *existing_end
&& *existing_start < end
})
{
let existing_len = existing.1.saturating_sub(existing.0);
let new_len = end.saturating_sub(start);
if new_len > existing_len {
*existing = (start, end, dt);
}
return;
}
matches.push((start, end, dt));
} else {
matches.push((usize::MAX, usize::MAX, dt));
}
}
};
for caps in YMD_RE.captures_iter(text) {
let (Some(year_match), Some(month_match), Some(day_match)) =
(caps.get(1), caps.get(2), caps.get(3))
else {
continue;
};
let year = match year_match.as_str().parse::<i32>() {
Ok(value) => value,
Err(_) => continue,
};
let month = match month_match.as_str().parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
let day = match day_match.as_str().parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
push_date(caps.get(0), date);
}
}
for caps in NUMERIC_RE.captures_iter(text) {
let (Some(first_match), Some(second_match), Some(year_match)) =
(caps.get(1), caps.get(2), caps.get(3))
else {
continue;
};
let first = match first_match.as_str().parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
let second = match second_match.as_str().parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
let year_raw = year_match.as_str();
let mut year = match year_raw.parse::<i32>() {
Ok(value) => value,
Err(_) => continue,
};
if year_raw.len() == 2 {
year += if year >= 70 { 1900 } else { 2000 };
}
// NUMERIC_RE always captures a day-first form (dd[sep]mm[sep]yy(yy));
// YMD layouts are handled earlier by YMD_RE, so YMD here is treated the
// same as DMY to avoid mis-parsing strings like 01-07-2024.
let (day, month) = match order {
DateOrder::Dmy | DateOrder::Ymd => (first, second),
DateOrder::Mdy => (second, first),
};
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
push_date(caps.get(0), date);
}
}
for caps in DAY_MONTH_RE.captures_iter(text) {
let (Some(day_match), Some(month_match), Some(year_match)) =
(caps.get(1), caps.get(2), caps.get(3))
else {
continue;
};
let day_str = day_match
.as_str()
.trim_matches(|c: char| !c.is_ascii_digit());
let day = match day_str.parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
let Some(month) = month_name_to_number(month_match.as_str(), settings) else {
continue;
};
if year_match.as_str().len() < 3 {
continue;
}
let Some(year) = normalize_year(year_match.as_str()) else {
continue;
};
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
push_date(caps.get(0), date);
}
}
for caps in MONTH_DAY_RE.captures_iter(text) {
let (Some(month_match), Some(day_match), Some(year_match)) =
(caps.get(1), caps.get(2), caps.get(3))
else {
continue;
};
let Some(month) = month_name_to_number(month_match.as_str(), settings) else {
continue;
};
let day_str = day_match
.as_str()
.trim_matches(|c: char| !c.is_ascii_digit());
let day = match day_str.parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
let Some(year) = normalize_year(year_match.as_str()) else {
continue;
};
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
push_date(caps.get(0), date);
}
}
for caps in MONTH_YEAR_RE.captures_iter(text) {
let (Some(month_match), Some(year_match)) = (caps.get(1), caps.get(2)) else {
continue;
};
let Some(month) = month_name_to_number(month_match.as_str(), settings) else {
continue;
};
let year = match year_match.as_str().parse::<i32>() {
Ok(value) => value,
Err(_) => continue,
};
if let Some(date) = NaiveDate::from_ymd_opt(year, month, 1) {
push_date(caps.get(0), date);
}
}
matches.sort_by_key(|(start, _, _)| *start);
matches
.into_iter()
.map(|(start, _, dt)| (start, dt))
.collect()
}
fn normalize_year(raw: &str) -> Option<i32> {
if raw.len() == 2 {
let mut year = raw.parse::<i32>().ok()?;
year += if year >= 70 { 1900 } else { 2000 };
Some(year)
} else {
raw.parse::<i32>().ok()
}
}
fn month_name_to_number(value: &str, settings: &IssuedAtSettings) -> Option<u32> {
let normalized = value.trim();
let variant = issued_at_months::variants()
.iter()
.find(|entry| entry.name == normalized)?;
if settings.locales().is_empty() || variant.locales.is_empty() {
return Some(variant.month);
}
if variant
.locales
.iter()
.any(|locale| settings.locales().contains(*locale))
{
Some(variant.month)
} else {
None
}
}
async fn load_document_text(ctx: &mut DocumentVersionTaskContext) -> TaskResult<Option<String>> {
let object_key = {
let assets = ctx.assets().await?;
assets
.get(OCR_TEXT_ASSET_TYPE)
.and_then(|asset| asset.objects.first())
.map(|object| object.s3_key.clone())
};
let Some(key) = object_key else {
return Ok(None);
};
match ctx.storage().get_object(&key).await {
Ok(bytes) => match String::from_utf8(bytes) {
Ok(mut text) => {
if text.len() > MAX_TEXT_CHARS {
text.truncate(MAX_TEXT_CHARS);
}
Ok(Some(text))
}
Err(err) => {
warn!(error = %err, "ocr text asset not valid utf-8");
Ok(None)
}
},
Err(err) => {
warn!(error = %err, "failed to download ocr text for issued_at extractor");
Ok(None)
}
}
}
#[derive(Copy, Clone)]
enum IssuedAtSource {
Parser,
Filename,
Text,
}
impl IssuedAtSource {
fn as_ref(&self) -> &'static str {
match self {
IssuedAtSource::Parser => "parser",
IssuedAtSource::Filename => "filename",
IssuedAtSource::Text => "text",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::AppConfig;
use serde::Deserialize;
use serde_yaml::Value as YamlValue;
use once_cell::sync::Lazy;
#[derive(Clone, Deserialize)]
struct CaseSuite {
cases: Vec<CaseDefinition>,
}
#[derive(Clone, Deserialize)]
struct CaseDefinition {
name: String,
parser: String,
#[serde(default)]
filename: Option<String>,
#[serde(default)]
content: Option<String>,
#[serde(default)]
settings: CaseSettings,
expected: ExpectedCase,
}
#[derive(Clone, Default, Deserialize)]
struct CaseSettings {
#[serde(rename = "DATE_PARSER_LANGUAGES", default)]
date_parser_languages: Vec<String>,
#[serde(rename = "FILENAME_DATE_ORDER")]
filename_date_order: Option<String>,
#[serde(rename = "DATE_ORDER")]
date_order: Option<String>,
#[serde(rename = "IGNORE_DATES", default)]
ignore_dates: Vec<String>,
}
#[derive(Clone, Deserialize)]
struct ExpectedCase {
mode: ExpectedMode,
#[serde(default)]
value: Option<YamlValue>,
}
#[derive(Clone, Deserialize, PartialEq)]
#[serde(rename_all = "lowercase")]
enum ExpectedMode {
None,
Single,
Multiple,
}
static CASES: Lazy<CaseSuite> = Lazy::new(|| {
let raw = include_str!("../../tests/data/issued_at_cases.yaml");
serde_yaml::from_str(raw).expect("failed to parse issued_at cases")
});
pub(crate) fn run_named_case(name: &str) {
let case = CASES
.cases
.iter()
.find(|case| case.name == name)
.unwrap_or_else(|| panic!("case '{}' not found", name))
.clone();
run_case(case);
}
fn run_case(case: CaseDefinition) {
let mut config = base_config();
if let Some(order) = case.settings.date_order {
config.issued_at_date_order = order;
}
config.issued_at_filename_date_order = case.settings.filename_date_order;
config.issued_at_date_parser_locales = case.settings.date_parser_languages;
config.issued_at_ignore_dates = case.settings.ignore_dates;
let settings = IssuedAtSettings::from_config(&config);
let now_utc = Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).single().unwrap();
let mut matches = Vec::new();
if let Some(filename) = &case.filename {
if let Some(order) = settings.filename_date_order() {
let normalized = normalize_content(filename, MAX_FILENAME_CHARS);
matches.extend(
collect_matches_with_spans(&normalized, order, &settings, now_utc)
.into_iter()
.map(|(_, dt)| dt.date_naive()),
);
}
}
if let Some(content) = &case.content {
let normalized = normalize_content(content, MAX_TEXT_CHARS);
matches.extend(
collect_matches_with_spans(&normalized, settings.date_order(), &settings, now_utc)
.into_iter()
.map(|(_, dt)| dt.date_naive()),
);
}
let actual: Vec<String> = matches
.into_iter()
.map(|date| date.format("%Y-%m-%d").to_string())
.collect();
match case.parser.as_str() {
"parse_date" => match case.expected.mode {
ExpectedMode::None => assert!(
actual.is_empty(),
"case '{}' expected no matches, got {:?}",
case.name,
actual
),
ExpectedMode::Single => {
let expected = case.expected.single_value();
assert!(
expected.is_some(),
"case '{}' is missing expected single value",
case.name
);
assert_eq!(
actual.first(),
expected.as_ref(),
"case '{}' single mismatch",
case.name
);
}
ExpectedMode::Multiple => panic!(
"case '{}' declares parse_date but expects multiple results",
case.name
),
},
"parse_date_generator" => {
let expected = case.expected.multiple_values().unwrap_or_default();
assert_eq!(expected, actual, "case '{}' multiple mismatch", case.name);
}
other => panic!("unsupported parser '{}' in case {}", other, case.name),
}
}
impl ExpectedCase {
fn single_value(&self) -> Option<String> {
match self.value.as_ref()? {
YamlValue::String(value) => Some(value.clone()),
other => Some(other.as_str()?.to_string()),
}
}
fn multiple_values(&self) -> Option<Vec<String>> {
let list = match self.value.as_ref()? {
YamlValue::Sequence(seq) => seq,
_ => return None,
};
Some(
list.iter()
.filter_map(|value| value.as_str().map(|s| s.to_string()))
.collect(),
)
}
}
fn base_config() -> AppConfig {
AppConfig {
database_url: "postgres://test".to_string(),
database_max_pool_size: 5,
server_host: "127.0.0.1".to_string(),
server_port: 0,
webdav_host: "127.0.0.1".to_string(),
webdav_port: 0,
jwt_secret: "secret".to_string(),
jwt_issuer: "issuer".to_string(),
jwt_audience: "audience".to_string(),
jwt_expiry_minutes: 60,
download_token_audience: "download".to_string(),
download_token_expiry_minutes: 60,
refresh_token_expiry_days: 30,
refresh_cookie_secure: false,
refresh_cookie_domain: None,
cors_allowed_origin: None,
proxy_downloads: false,
aws_endpoint_url: None,
aws_access_key_id: None,
aws_secret_access_key: None,
aws_region: "us-east-1".to_string(),
s3_bucket: "bucket".to_string(),
quickwit_endpoint: None,
quickwit_index: None,
worker_max_document_bytes: 100 * 1024 * 1024,
upload_body_limit_bytes: 64 * 1024 * 1024,
service_timezone: "UTC".to_string(),
issued_at_date_order: "DMY".to_string(),
issued_at_filename_date_order: None,
issued_at_date_parser_locales: Vec::new(),
issued_at_ignore_dates: Vec::new(),
webauthn_rp_id: Some("localhost".to_string()),
webauthn_origin: Some("http://localhost".to_string()),
webauthn_rp_name: "Papercrate".to_string(),
}
}
include!(concat!(env!("OUT_DIR"), "/issued_at_generated_tests.rs"));
#[test]
fn month_name_lookup_handles_turkish_variants() {
let settings = IssuedAtSettings::from_config(&base_config());
assert_eq!(month_name_to_number("şubat", &settings), Some(2));
assert_eq!(month_name_to_number("subat", &settings), Some(2));
}
#[test]
fn locale_filter_limits_month_names() {
let mut config = base_config();
config.issued_at_date_parser_locales = vec!["tr".into()];
let settings = IssuedAtSettings::from_config(&config);
assert_eq!(month_name_to_number("january", &settings), None);
assert_eq!(month_name_to_number("şubat", &settings), Some(2));
}
}
backend/src/workers/issued_at_months.rs
+15
View File
@@ -0,0 +1,15 @@
pub(super) struct MonthVariant {
pub name: &'static str,
pub month: u32,
pub locales: &'static [&'static str],
}
include!(concat!(env!("OUT_DIR"), "/issued_at_months.rs"));
pub(super) fn pattern() -> &'static str {
MONTH_PATTERN
}
pub(super) fn variants() -> &'static [MonthVariant] {
MONTH_VARIANTS
}
backend/src/workers/mod.rs
+1
View File
@@ -16,6 +16,7 @@ use taskflow::TaskError;
pub mod analyze;
pub mod common;
pub mod index;
pub mod issued_at;
pub mod ocr;
pub mod purge;
pub mod taskflow;
backend/tests/data/issued_at_cases.yaml
+582
View File
@@ -0,0 +1,582 @@
cases:
- name: date_format_1
parser: parse_date
filename: null
content: "lorem ipsum 130218 lorem ipsum"
settings: {}
expected:
mode: none
- name: date_format_2
parser: parse_date
filename: null
content: "lorem ipsum 2018 lorem ipsum"
settings: {}
expected:
mode: none
- name: date_format_3
parser: parse_date
filename: null
content: "lorem ipsum 20180213 lorem ipsum"
settings: {}
expected:
mode: none
- name: date_format_4
parser: parse_date
filename: null
content: "lorem ipsum 13.02.2018 lorem ipsum"
settings: {}
expected:
mode: single
value: 2018-02-13
- name: date_format_5
parser: parse_date
filename: null
content: "lorem ipsum 130218, 2018, 20180213 and lorem 13.02.2018 lorem ipsum"
settings: {}
expected:
mode: single
value: 2018-02-13
- name: date_format_6
parser: parse_date
filename: null
content: |
lorem ipsum
Wohnort
3100
IBAN
AT87 4534
1234
1234 5678
BIC
lorem ipsum
settings: {}
expected:
mode: none
- name: date_format_7
parser: parse_date
filename: null
content: |
lorem ipsum
März 2019
lorem ipsum
settings:
DATE_PARSER_LANGUAGES:
- de
expected:
mode: single
value: 2019-03-01
- name: date_format_8
parser: parse_date
filename: null
content: |
lorem ipsum
Wohnort
3100
IBAN
AT87 4534
1234
1234 5678
BIC
lorem ipsum
März 2020
settings:
DATE_PARSER_LANGUAGES:
- de
expected:
mode: single
value: 2020-03-01
- name: date_format_9
parser: parse_date
filename: null
content: |
lorem ipsum
27. Nullmonth 2020
März 2020
lorem ipsum
settings:
DATE_PARSER_LANGUAGES:
- de
expected:
mode: single
value: 2020-03-01
- name: date_format_10
parser: parse_date
filename: null
content: "Customer Number Currency 22-MAR-2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_11
parser: parse_date
filename: null
content: "Customer Number Currency 22 MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_12
parser: parse_date
filename: null
content: "Customer Number Currency 22/MAR/2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_13
parser: parse_date
filename: null
content: "Customer Number Currency 22.MAR.2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_14
parser: parse_date
filename: null
content: "Customer Number Currency 22.MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_15
parser: parse_date
filename: null
content: "Customer Number Currency 22.MAR.22 Credit Card 1934829304"
settings: {}
expected:
mode: none
- name: date_format_16
parser: parse_date
filename: null
content: "Customer Number Currency 22.MAR,22 Credit Card 1934829304"
settings: {}
expected:
mode: none
- name: date_format_17
parser: parse_date
filename: null
content: "Customer Number Currency 22,MAR,2022 Credit Card 1934829304"
settings: {}
expected:
mode: none
- name: date_format_18
parser: parse_date
filename: null
content: "Customer Number Currency 22 MAR,2022 Credit Card 1934829304"
settings: {}
expected:
mode: none
- name: date_format_19
parser: parse_date
filename: null
content: "Customer Number Currency 21st MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-21
- name: date_format_20
parser: parse_date
filename: null
content: "Customer Number Currency 22nd March 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_21
parser: parse_date
filename: null
content: "Customer Number Currency 2nd MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-02
- name: date_format_22
parser: parse_date
filename: null
content: "Customer Number Currency 23rd MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-23
- name: date_format_23
parser: parse_date
filename: null
content: "Customer Number Currency 24th MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-24
- name: date_format_24
parser: parse_date
filename: null
content: "Customer Number Currency 21-MAR-2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-21
- name: date_format_25
parser: parse_date
filename: null
content: "Customer Number Currency 25TH MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-25
- name: date_format_26
parser: parse_date
filename: null
content: "CHASE 0 September 25, 2019 JPMorgan Chase Bank, NA. P0 Box 182051"
settings: {}
expected:
mode: single
value: 2019-09-25
- name: numeric_mdy_slash
parser: parse_date
filename: null
content: "03/17/2008"
settings:
DATE_ORDER: MDY
expected:
mode: single
value: 2008-03-17
- name: crazy_date_past
parser: parse_date
filename: null
content: "01-07-0590 00:00:00"
settings: {}
expected:
mode: none
- name: crazy_date_future
parser: parse_date
filename: null
content: "01-07-2350 00:00:00"
settings: {}
expected:
mode: none
- name: crazy_date_with_spaces
parser: parse_date
filename: null
content: "20 408000l 2475"
settings: {}
expected:
mode: none
- name: utf_month_names_decembre
parser: parse_date
filename: null
content: "13 décembre 2023"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2023-12-13
- name: utf_month_names_aout
parser: parse_date
filename: null
content: "13 août 2022"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2022-08-13
- name: utf_month_names_marz
parser: parse_date
filename: null
content: "11 März 2020"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2020-03-11
- name: utf_month_names_ozujka
parser: parse_date
filename: null
content: "17. ožujka 2018."
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2018-03-17
- name: utf_month_names_veljace
parser: parse_date
filename: null
content: "1. veljače 2016."
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2016-02-01
- name: utf_month_names_unora
parser: parse_date
filename: null
content: "15. února 1985"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 1985-02-15
- name: utf_month_names_zari
parser: parse_date
filename: null
content: "30. září 2011"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2011-09-30
- name: utf_month_names_kvetna
parser: parse_date
filename: null
content: "28. května 1990"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 1990-05-28
- name: utf_month_names_grudzien
parser: parse_date
filename: null
content: "1. grudzień 1997"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 1997-12-01
- name: utf_month_names_subat
parser: parse_date
filename: null
content: "17 Şubat 2024"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2024-02-17
- name: utf_month_names_agustos
parser: parse_date
filename: null
content: "30 Ağustos 2012"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2012-08-30
- name: utf_month_names_eylul
parser: parse_date
filename: null
content: "17 Eylül 2000"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2000-09-17
- name: utf_month_names_oktober
parser: parse_date
filename: null
content: "5. október 1992"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
- hu
expected:
mode: single
value: 1992-10-05
- name: multiple_dates
parser: parse_date_generator
filename: null
content: |
This text has multiple dates.
For example 02.02.2018, 22 July 2022 and December 2021.
But not 24-12-9999 because it's in the future...
settings: {}
expected:
mode: multiple
value:
- 2018-02-02
- 2022-07-22
- 2021-12-01
- name: filename_date_parse_valid_ymd
parser: parse_date
filename: /tmp/Scan-2022-04-01.pdf
content: "No date in here"
settings:
FILENAME_DATE_ORDER: YMD
expected:
mode: single
value: 2022-04-01
- name: filename_date_parse_valid_dmy
parser: parse_date
filename: /tmp/Scan-10.01.2021.pdf
content: "No date in here"
settings:
FILENAME_DATE_ORDER: DMY
expected:
mode: single
value: 2021-01-10
- name: filename_date_parse_invalid
parser: parse_date
filename: "/tmp/20 408000l 2475 - test.pdf"
content: "No date in here"
settings:
FILENAME_DATE_ORDER: YMD
expected:
mode: none
- name: filename_date_ignored_use_content
parser: parse_date
filename: /tmp/Scan-2022-04-01.pdf
content: "The matching date is 24.03.2022"
settings:
FILENAME_DATE_ORDER: YMD
IGNORE_DATES:
- 2022-04-01
expected:
mode: single
value: 2022-03-24
- name: ignored_dates_default_order
parser: parse_date
filename: null
content: "lorem ipsum 110319, 20200117 and lorem 13.02.2018 lorem ipsum"
settings:
IGNORE_DATES:
- 2019-11-03
- 2020-01-17
expected:
mode: single
value: 2018-02-13
- name: ignored_dates_order_ymd
parser: parse_date
filename: null
content: "lorem ipsum 190311, 20200117 and lorem 13.02.2018 lorem ipsum"
settings:
FILENAME_DATE_ORDER: YMD
IGNORE_DATES:
- 2019-11-03
- 2020-01-17
expected:
mode: single
value: 2018-02-13