Generate compact classic NPC mapping from ProjectEQ data

src/bin/generate_peq_npc_index.rs

@@ -0,0 +1,420 @@
+//! Generate Mudmouth's compact NPC gender lookup from a ProjectEQ SQL dump.
+//!
+//! This is deliberately a build-time tool. It reads only the four tables used
+//! by the lookup and does not add a database dependency to the application.
+
+use anyhow::{Context, Result, bail};
+use clap::Parser;
+use serde::Serialize;
+use std::{
+    collections::{BTreeMap, BTreeSet, HashMap},
+    fs::{self, File},
+    io::{BufRead, BufReader},
+    path::{Path, PathBuf},
+    process::{Command, Stdio},
+};
+
+#[allow(dead_code)]
+#[path = "../events.rs"]
+mod events;
+
+#[derive(Debug, Parser)]
+#[command(about = "Generate a minified NPC gender index from a PEQ SQL dump")]
+struct Cli {
+    /// PEQ create_tables_content.sql file, or a .zip containing that file.
+    #[arg(long)]
+    dump: PathBuf,
+
+    /// Destination for the generated minified JSON index.
+    #[arg(long)]
+    output: PathBuf,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
+#[serde(rename_all = "lowercase")]
+enum Gender {
+    Masculine,
+    Feminine,
+    Neuter,
+}
+
+impl Gender {
+    fn from_database(value: &str) -> Result<Self> {
+        match value {
+            "0" => Ok(Self::Masculine),
+            "1" => Ok(Self::Feminine),
+            "2" => Ok(Self::Neuter),
+            _ => bail!("invalid npc_types.gender {value:?}; expected 0, 1, or 2"),
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+struct PeqRows {
+    zones: HashMap<String, BTreeSet<String>>,
+    spawn_entries: HashMap<u32, BTreeSet<u32>>,
+    spawns: Vec<(String, u32)>,
+    npcs: HashMap<u32, (String, Gender)>,
+    seen_tables: BTreeSet<Table>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+enum Table {
+    NpcTypes,
+    SpawnEntry,
+    Spawn2,
+    Zone,
+}
+
+impl Table {
+    fn from_insert(line: &str) -> Option<Self> {
+        [
+            ("INSERT INTO `npc_types` VALUES", Self::NpcTypes),
+            ("INSERT INTO `spawnentry` VALUES", Self::SpawnEntry),
+            ("INSERT INTO `spawn2` VALUES", Self::Spawn2),
+            ("INSERT INTO `zone` VALUES", Self::Zone),
+        ]
+        .into_iter()
+        .find_map(|(prefix, table)| line.starts_with(prefix).then_some(table))
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct GeneratedIndex {
+    schema_version: u8,
+    source: String,
+    stats: Statistics,
+    entries: Vec<IndexEntry>,
+}
+
+#[derive(Debug, Default, Serialize, PartialEq, Eq)]
+struct Statistics {
+    pairs: usize,
+    masculine_only: usize,
+    feminine_only: usize,
+    neuter_only: usize,
+    conflicting: usize,
+    accepted: usize,
+}
+
+#[derive(Debug, Serialize, PartialEq, Eq)]
+struct IndexEntry(String, String, Gender);
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+    let rows = read_dump(&cli.dump)?;
+    let index = generate_index(&rows, source_identity(&cli.dump)?);
+    let bytes = serde_json::to_vec(&index).context("could not serialize PEQ index")?;
+    fs::write(&cli.output, &bytes)
+        .with_context(|| format!("could not write {}", cli.output.display()))?;
+
+    println!("source: {}", index.source);
+    println!(
+        "pairs: {} (masculine: {}, feminine: {}, neuter: {}, conflicting: {}, accepted: {})",
+        index.stats.pairs,
+        index.stats.masculine_only,
+        index.stats.feminine_only,
+        index.stats.neuter_only,
+        index.stats.conflicting,
+        index.stats.accepted,
+    );
+    println!("output: {} bytes ({})", bytes.len(), cli.output.display());
+    Ok(())
+}
+
+fn source_identity(path: &Path) -> Result<String> {
+    path.file_name()
+        .and_then(|name| name.to_str())
+        .map(str::to_owned)
+        .context("dump path must have a UTF-8 file name")
+}
+
+fn read_dump(path: &Path) -> Result<PeqRows> {
+    if path.extension().is_some_and(|extension| extension == "zip") {
+        let mut child = Command::new("unzip")
+            .args([
+                "-p",
+                &path.to_string_lossy(),
+                "peq-dump/create_tables_content.sql",
+            ])
+            .stdout(Stdio::piped())
+            .spawn()
+            .context("could not run unzip; install it or provide the extracted SQL dump")?;
+        let stdout = child
+            .stdout
+            .take()
+            .context("unzip did not provide stdout")?;
+        let rows = parse_sql(BufReader::new(stdout));
+        let status = child.wait().context("could not wait for unzip")?;
+        if !status.success() {
+            bail!("unzip failed while reading {}", path.display());
+        }
+        rows
+    } else {
+        let file =
+            File::open(path).with_context(|| format!("could not read {}", path.display()))?;
+        parse_sql(BufReader::new(file))
+    }
+}
+
+fn parse_sql<R: BufRead>(reader: R) -> Result<PeqRows> {
+    let mut rows = PeqRows::default();
+    let mut current_table = None;
+
+    for (line_number, line) in reader.lines().enumerate() {
+        let line = line.with_context(|| format!("could not read SQL line {}", line_number + 1))?;
+        if let Some(table) = Table::from_insert(&line) {
+            current_table = Some(table);
+            rows.seen_tables.insert(table);
+        }
+
+        if let Some(table) = current_table {
+            for fields in sql_tuples(&line).with_context(|| {
+                format!("malformed {table:?} row at SQL line {}", line_number + 1)
+            })? {
+                add_row(&mut rows, table, &fields).with_context(|| {
+                    format!("invalid {table:?} row at SQL line {}", line_number + 1)
+                })?;
+            }
+            if line.trim_end().ends_with(';') {
+                current_table = None;
+            }
+        }
+    }
+
+    for table in [
+        Table::NpcTypes,
+        Table::SpawnEntry,
+        Table::Spawn2,
+        Table::Zone,
+    ] {
+        if !rows.seen_tables.contains(&table) {
+            bail!("dump is missing INSERT rows for {table:?}");
+        }
+    }
+    Ok(rows)
+}
+
+fn add_row(rows: &mut PeqRows, table: Table, fields: &[String]) -> Result<()> {
+    match table {
+        Table::NpcTypes => {
+            let id = field_u32(fields, 0, "id")?;
+            let name = field_string(fields, 1, "name")?;
+            let gender = Gender::from_database(field(fields, 9, "gender")?)?;
+            if rows.npcs.insert(id, (name, gender)).is_some() {
+                bail!("duplicate npc_types.id {id}");
+            }
+        }
+        Table::SpawnEntry => {
+            let group = field_u32(fields, 0, "spawngroupID")?;
+            let npc = field_u32(fields, 1, "npcID")?;
+            rows.spawn_entries.entry(group).or_default().insert(npc);
+        }
+        Table::Spawn2 => {
+            let group = field_u32(fields, 1, "spawngroupID")?;
+            let zone = field_string(fields, 2, "zone")?;
+            if !zone.is_empty() {
+                rows.spawns.push((zone, group));
+            }
+        }
+        Table::Zone => {
+            let short_name = field_string(fields, 3, "short_name")?;
+            let long_name = field_string(fields, 4, "long_name")?;
+            if !short_name.is_empty() {
+                rows.zones.entry(short_name).or_default().insert(long_name);
+            }
+        }
+    }
+    Ok(())
+}
+
+fn field<'a>(fields: &'a [String], index: usize, name: &str) -> Result<&'a str> {
+    fields
+        .get(index)
+        .map(String::as_str)
+        .with_context(|| format!("missing {name} column"))
+}
+
+fn field_u32(fields: &[String], index: usize, name: &str) -> Result<u32> {
+    field(fields, index, name)?
+        .parse()
+        .with_context(|| format!("{name} is not an unsigned integer"))
+}
+
+fn field_string(fields: &[String], index: usize, name: &str) -> Result<String> {
+    let value = field(fields, index, name)?;
+    if value == "NULL" {
+        bail!("{name} may not be NULL");
+    }
+    Ok(value.to_owned())
+}
+
+fn generate_index(rows: &PeqRows, source: String) -> GeneratedIndex {
+    let mut candidates = BTreeMap::<(String, String), BTreeSet<Gender>>::new();
+    for (short_zone, group) in &rows.spawns {
+        let Some(long_zones) = rows.zones.get(short_zone) else {
+            continue;
+        };
+        let Some(npc_ids) = rows.spawn_entries.get(group) else {
+            continue;
+        };
+        for npc_id in npc_ids {
+            let Some((name, gender)) = rows.npcs.get(npc_id) else {
+                continue;
+            };
+            if name.is_empty() {
+                continue;
+            }
+            let normalized_name = events::normalize_npc_name(name);
+            if normalized_name.is_empty() {
+                continue;
+            }
+            for long_zone in long_zones {
+                candidates
+                    .entry((long_zone.clone(), normalized_name.clone()))
+                    .or_default()
+                    .insert(*gender);
+            }
+        }
+    }
+
+    let mut stats = Statistics {
+        pairs: candidates.len(),
+        ..Default::default()
+    };
+    let mut entries = Vec::new();
+    for ((zone, name), genders) in candidates {
+        match genders
+            .iter()
+            .next()
+            .copied()
+            .filter(|_| genders.len() == 1)
+        {
+            Some(Gender::Masculine) => {
+                stats.masculine_only += 1;
+                stats.accepted += 1;
+                entries.push(IndexEntry(zone, name, Gender::Masculine));
+            }
+            Some(Gender::Feminine) => {
+                stats.feminine_only += 1;
+                stats.accepted += 1;
+                entries.push(IndexEntry(zone, name, Gender::Feminine));
+            }
+            Some(Gender::Neuter) => stats.neuter_only += 1,
+            None => stats.conflicting += 1,
+        }
+    }
+
+    GeneratedIndex {
+        schema_version: 1,
+        source,
+        stats,
+        entries,
+    }
+}
+
+/// Parses the SQL tuple syntax emitted by `mariadb-dump`.
+fn sql_tuples(line: &str) -> Result<Vec<Vec<String>>> {
+    let mut tuples = Vec::new();
+    let mut fields = Vec::new();
+    let mut field = String::new();
+    let mut in_tuple = false;
+    let mut in_string = false;
+    let mut escaped = false;
+
+    for character in line.chars() {
+        if !in_tuple {
+            if character == '(' {
+                in_tuple = true;
+                fields.clear();
+                field.clear();
+            }
+            continue;
+        }
+
+        if escaped {
+            field.push(match character {
+                '0' => '\0',
+                'b' => '\u{0008}',
+                'n' => '\n',
+                'r' => '\r',
+                't' => '\t',
+                'Z' => '\u{001a}',
+                other => other,
+            });
+            escaped = false;
+        } else if in_string && character == '\\' {
+            escaped = true;
+        } else if character == '\'' {
+            in_string = !in_string;
+        } else if !in_string && character == ',' {
+            fields.push(std::mem::take(&mut field));
+        } else if !in_string && character == ')' {
+            fields.push(std::mem::take(&mut field));
+            tuples.push(std::mem::take(&mut fields));
+            in_tuple = false;
+        } else {
+            field.push(character);
+        }
+    }
+
+    if in_tuple || in_string || escaped {
+        bail!("unterminated SQL tuple");
+    }
+    Ok(tuples)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const FIXTURE: &str = include_str!("../../tests/fixtures/peq-small.sql");
+
+    #[test]
+    fn groups_short_zones_and_filters_ambiguous_gender() {
+        let rows = parse_sql(BufReader::new(FIXTURE.as_bytes())).unwrap();
+        let index = generate_index(&rows, "fixture.sql".into());
+
+        assert_eq!(
+            index.entries,
+            vec![
+                IndexEntry(
+                    "Shared Zone".into(),
+                    "captain_bob".into(),
+                    Gender::Masculine
+                ),
+                IndexEntry("Shared Zone".into(), "lady_ada".into(), Gender::Feminine),
+            ]
+        );
+        assert_eq!(
+            index.stats,
+            Statistics {
+                pairs: 4,
+                masculine_only: 1,
+                feminine_only: 1,
+                neuter_only: 1,
+                conflicting: 1,
+                accepted: 2,
+            }
+        );
+    }
+
+    #[test]
+    fn output_is_minified_and_deterministic() {
+        let rows = parse_sql(BufReader::new(FIXTURE.as_bytes())).unwrap();
+        let first = serde_json::to_vec(&generate_index(&rows, "fixture.sql".into())).unwrap();
+        let second = serde_json::to_vec(&generate_index(&rows, "fixture.sql".into())).unwrap();
+
+        assert_eq!(first, second);
+        assert!(!first.contains(&b'\n'));
+    }
+
+    #[test]
+    fn rejects_malformed_rows_and_unknown_gender_values() {
+        let malformed = "INSERT INTO `npc_types` VALUES\n(1,'too short');\n";
+        assert!(parse_sql(BufReader::new(malformed.as_bytes())).is_err());
+
+        let invalid_gender = "INSERT INTO `npc_types` VALUES\n(1,'Bad','','',0,0,0,0,0,9);\n";
+        assert!(parse_sql(BufReader::new(invalid_gender.as_bytes())).is_err());
+    }
+}

src/events.rs

@@ -75,7 +75,11 @@ fn parse_npc_says(message: &str) -> Option<(String, String)> {
     Some((speaker.to_string(), text.to_string()))
 }
 
-pub fn speaker_key(name: &str) -> String {
+/// Normalizes an EverQuest NPC name for use in stable lookup identities.
+///
+/// Non-ASCII and punctuation characters are separators, and repeated
+/// separators collapse to a single underscore.
+pub fn normalize_npc_name(name: &str) -> String {
     let normalized = name
         .chars()
         .flat_map(char::to_lowercase)
@@ -88,12 +92,15 @@ pub fn speaker_key(name: &str) -> String {
         })
         .collect::<String>();
     let normalized = normalized.trim_matches('_');
-    let compact = normalized
+    normalized
         .split('_')
         .filter(|part| !part.is_empty())
         .collect::<Vec<_>>()
-        .join("_");
-    format!("npc:{compact}")
+        .join("_")
+}
+
+pub fn speaker_key(name: &str) -> String {
+    format!("npc:{}", normalize_npc_name(name))
 }
 
 #[cfg(test)]
@@ -140,4 +147,19 @@ mod tests {
         assert_eq!(speaker_key("A lizardman healer"), "npc:a_lizardman_healer");
         assert_eq!(speaker_key("Zok  Caropni!"), "npc:zok_caropni");
     }
+
+    #[test]
+    fn normalizes_npc_names_for_shared_lookup() {
+        let fixtures = [
+            ("Zok Caropni!", "zok_caropni"),
+            ("a---lizardman__healer", "a_lizardman_healer"),
+            ("MiXeD CaSe NPC", "mixed_case_npc"),
+            ("F\u{00e9}lix the C\u{00e1}t", "f_lix_the_c_t"),
+        ];
+
+        for (name, expected) in fixtures {
+            assert_eq!(normalize_npc_name(name), expected, "{name}");
+            assert_eq!(speaker_key(name), format!("npc:{expected}"), "{name}");
+        }
+    }
 }