mudmouth/src/bin/generate_peq_npc_index.rs

//! Generate Mudmouth's compact NPC gender lookup from a ProjectEQ SQL dump.
//!
//! This is deliberately a build-time tool. It reads only the four tables used
//! by the lookup and does not add a database dependency to the application.

use anyhow::{Context, Result, bail};
use clap::Parser;
use serde::Serialize;
use std::{
    collections::{BTreeMap, BTreeSet, HashMap},
    fs::{self, File},
    io::{BufRead, BufReader},
    path::{Path, PathBuf},
    process::{Command, Stdio},
};

#[allow(dead_code)]
#[path = "../events.rs"]
mod events;

#[derive(Debug, Parser)]
#[command(about = "Generate a minified NPC gender index from a PEQ SQL dump")]
struct Cli {
    /// PEQ create_tables_content.sql file, or a .zip containing that file.
    #[arg(long)]
    dump: PathBuf,

    /// Destination for the generated minified JSON index.
    #[arg(long)]
    output: PathBuf,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
#[serde(rename_all = "lowercase")]
enum Gender {
    Masculine,
    Feminine,
    Neuter,
}

impl Gender {
    fn from_database(value: &str) -> Result<Self> {
        match value {
            "0" => Ok(Self::Masculine),
            "1" => Ok(Self::Feminine),
            "2" => Ok(Self::Neuter),
            _ => bail!("invalid npc_types.gender {value:?}; expected 0, 1, or 2"),
        }
    }
}

#[derive(Debug, Default)]
struct PeqRows {
    zones: HashMap<String, BTreeSet<String>>,
    spawn_entries: HashMap<u32, BTreeSet<u32>>,
    spawns: Vec<(String, u32)>,
    npcs: HashMap<u32, (String, Gender)>,
    seen_tables: BTreeSet<Table>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
enum Table {
    NpcTypes,
    SpawnEntry,
    Spawn2,
    Zone,
}

impl Table {
    fn from_insert(line: &str) -> Option<Self> {
        [
            ("INSERT INTO `npc_types` VALUES", Self::NpcTypes),
            ("INSERT INTO `spawnentry` VALUES", Self::SpawnEntry),
            ("INSERT INTO `spawn2` VALUES", Self::Spawn2),
            ("INSERT INTO `zone` VALUES", Self::Zone),
        ]
        .into_iter()
        .find_map(|(prefix, table)| line.starts_with(prefix).then_some(table))
    }
}

#[derive(Debug, Serialize)]
struct GeneratedIndex {
    schema_version: u8,
    source: String,
    stats: Statistics,
    entries: Vec<IndexEntry>,
}

#[derive(Debug, Default, Serialize, PartialEq, Eq)]
struct Statistics {
    pairs: usize,
    masculine_only: usize,
    feminine_only: usize,
    neuter_only: usize,
    conflicting: usize,
    accepted: usize,
}

#[derive(Debug, Serialize, PartialEq, Eq)]
struct IndexEntry(String, String, Gender);

fn main() -> Result<()> {
    let cli = Cli::parse();
    let rows = read_dump(&cli.dump)?;
    let index = generate_index(&rows, source_identity(&cli.dump)?);
    let bytes = serde_json::to_vec(&index).context("could not serialize PEQ index")?;
    fs::write(&cli.output, &bytes)
        .with_context(|| format!("could not write {}", cli.output.display()))?;

    println!("source: {}", index.source);
    println!(
        "pairs: {} (masculine: {}, feminine: {}, neuter: {}, conflicting: {}, accepted: {})",
        index.stats.pairs,
        index.stats.masculine_only,
        index.stats.feminine_only,
        index.stats.neuter_only,
        index.stats.conflicting,
        index.stats.accepted,
    );
    println!("output: {} bytes ({})", bytes.len(), cli.output.display());
    Ok(())
}

fn source_identity(path: &Path) -> Result<String> {
    path.file_name()
        .and_then(|name| name.to_str())
        .map(str::to_owned)
        .context("dump path must have a UTF-8 file name")
}

fn read_dump(path: &Path) -> Result<PeqRows> {
    if path.extension().is_some_and(|extension| extension == "zip") {
        let mut child = Command::new("unzip")
            .args([
                "-p",
                &path.to_string_lossy(),
                "peq-dump/create_tables_content.sql",
            ])
            .stdout(Stdio::piped())
            .spawn()
            .context("could not run unzip; install it or provide the extracted SQL dump")?;
        let stdout = child
            .stdout
            .take()
            .context("unzip did not provide stdout")?;
        let rows = parse_sql(BufReader::new(stdout));
        let status = child.wait().context("could not wait for unzip")?;
        if !status.success() {
            bail!("unzip failed while reading {}", path.display());
        }
        rows
    } else {
        let file =
            File::open(path).with_context(|| format!("could not read {}", path.display()))?;
        parse_sql(BufReader::new(file))
    }
}

fn parse_sql<R: BufRead>(reader: R) -> Result<PeqRows> {
    let mut rows = PeqRows::default();
    let mut current_table = None;

    for (line_number, line) in reader.lines().enumerate() {
        let line = line.with_context(|| format!("could not read SQL line {}", line_number + 1))?;
        if let Some(table) = Table::from_insert(&line) {
            current_table = Some(table);
            rows.seen_tables.insert(table);
        }

        if let Some(table) = current_table {
            for fields in sql_tuples(&line).with_context(|| {
                format!("malformed {table:?} row at SQL line {}", line_number + 1)
            })? {
                add_row(&mut rows, table, &fields).with_context(|| {
                    format!("invalid {table:?} row at SQL line {}", line_number + 1)
                })?;
            }
            if line.trim_end().ends_with(';') {
                current_table = None;
            }
        }
    }

    for table in [
        Table::NpcTypes,
        Table::SpawnEntry,
        Table::Spawn2,
        Table::Zone,
    ] {
        if !rows.seen_tables.contains(&table) {
            bail!("dump is missing INSERT rows for {table:?}");
        }
    }
    Ok(rows)
}

fn add_row(rows: &mut PeqRows, table: Table, fields: &[String]) -> Result<()> {
    match table {
        Table::NpcTypes => {
            let id = field_u32(fields, 0, "id")?;
            let name = field_string(fields, 1, "name")?;
            let gender = Gender::from_database(field(fields, 9, "gender")?)?;
            if rows.npcs.insert(id, (name, gender)).is_some() {
                bail!("duplicate npc_types.id {id}");
            }
        }
        Table::SpawnEntry => {
            let group = field_u32(fields, 0, "spawngroupID")?;
            let npc = field_u32(fields, 1, "npcID")?;
            rows.spawn_entries.entry(group).or_default().insert(npc);
        }
        Table::Spawn2 => {
            let group = field_u32(fields, 1, "spawngroupID")?;
            let zone = field_string(fields, 2, "zone")?;
            if !zone.is_empty() {
                rows.spawns.push((zone, group));
            }
        }
        Table::Zone => {
            let short_name = field_string(fields, 3, "short_name")?;
            let long_name = field_string(fields, 4, "long_name")?;
            if !short_name.is_empty() {
                rows.zones.entry(short_name).or_default().insert(long_name);
            }
        }
    }
    Ok(())
}

fn field<'a>(fields: &'a [String], index: usize, name: &str) -> Result<&'a str> {
    fields
        .get(index)
        .map(String::as_str)
        .with_context(|| format!("missing {name} column"))
}

fn field_u32(fields: &[String], index: usize, name: &str) -> Result<u32> {
    field(fields, index, name)?
        .parse()
        .with_context(|| format!("{name} is not an unsigned integer"))
}

fn field_string(fields: &[String], index: usize, name: &str) -> Result<String> {
    let value = field(fields, index, name)?;
    if value == "NULL" {
        bail!("{name} may not be NULL");
    }
    Ok(value.to_owned())
}

fn generate_index(rows: &PeqRows, source: String) -> GeneratedIndex {
    let mut candidates = BTreeMap::<(String, String), BTreeSet<Gender>>::new();
    for (short_zone, group) in &rows.spawns {
        let Some(long_zones) = rows.zones.get(short_zone) else {
            continue;
        };
        let Some(npc_ids) = rows.spawn_entries.get(group) else {
            continue;
        };
        for npc_id in npc_ids {
            let Some((name, gender)) = rows.npcs.get(npc_id) else {
                continue;
            };
            if name.is_empty() {
                continue;
            }
            let normalized_name = events::normalize_npc_name(name);
            if normalized_name.is_empty() {
                continue;
            }
            for long_zone in long_zones {
                candidates
                    .entry((long_zone.clone(), normalized_name.clone()))
                    .or_default()
                    .insert(*gender);
            }
        }
    }

    let mut stats = Statistics {
        pairs: candidates.len(),
        ..Default::default()
    };
    let mut entries = Vec::new();
    for ((zone, name), genders) in candidates {
        match genders
            .iter()
            .next()
            .copied()
            .filter(|_| genders.len() == 1)
        {
            Some(Gender::Masculine) => {
                stats.masculine_only += 1;
                stats.accepted += 1;
                entries.push(IndexEntry(zone, name, Gender::Masculine));
            }
            Some(Gender::Feminine) => {
                stats.feminine_only += 1;
                stats.accepted += 1;
                entries.push(IndexEntry(zone, name, Gender::Feminine));
            }
            Some(Gender::Neuter) => stats.neuter_only += 1,
            None => stats.conflicting += 1,
        }
    }

    GeneratedIndex {
        schema_version: 1,
        source,
        stats,
        entries,
    }
}

/// Parses the SQL tuple syntax emitted by `mariadb-dump`.
fn sql_tuples(line: &str) -> Result<Vec<Vec<String>>> {
    let mut tuples = Vec::new();
    let mut fields = Vec::new();
    let mut field = String::new();
    let mut in_tuple = false;
    let mut in_string = false;
    let mut escaped = false;

    for character in line.chars() {
        if !in_tuple {
            if character == '(' {
                in_tuple = true;
                fields.clear();
                field.clear();
            }
            continue;
        }

        if escaped {
            field.push(match character {
                '0' => '\0',
                'b' => '\u{0008}',
                'n' => '\n',
                'r' => '\r',
                't' => '\t',
                'Z' => '\u{001a}',
                other => other,
            });
            escaped = false;
        } else if in_string && character == '\\' {
            escaped = true;
        } else if character == '\'' {
            in_string = !in_string;
        } else if !in_string && character == ',' {
            fields.push(std::mem::take(&mut field));
        } else if !in_string && character == ')' {
            fields.push(std::mem::take(&mut field));
            tuples.push(std::mem::take(&mut fields));
            in_tuple = false;
        } else {
            field.push(character);
        }
    }

    if in_tuple || in_string || escaped {
        bail!("unterminated SQL tuple");
    }
    Ok(tuples)
}

#[cfg(test)]
mod tests {
    use super::*;

    const FIXTURE: &str = include_str!("../../tests/fixtures/peq-small.sql");

    #[test]
    fn groups_short_zones_and_filters_ambiguous_gender() {
        let rows = parse_sql(BufReader::new(FIXTURE.as_bytes())).unwrap();
        let index = generate_index(&rows, "fixture.sql".into());

        assert_eq!(
            index.entries,
            vec![
                IndexEntry(
                    "Shared Zone".into(),
                    "captain_bob".into(),
                    Gender::Masculine
                ),
                IndexEntry("Shared Zone".into(), "lady_ada".into(), Gender::Feminine),
            ]
        );
        assert_eq!(
            index.stats,
            Statistics {
                pairs: 4,
                masculine_only: 1,
                feminine_only: 1,
                neuter_only: 1,
                conflicting: 1,
                accepted: 2,
            }
        );
    }

    #[test]
    fn output_is_minified_and_deterministic() {
        let rows = parse_sql(BufReader::new(FIXTURE.as_bytes())).unwrap();
        let first = serde_json::to_vec(&generate_index(&rows, "fixture.sql".into())).unwrap();
        let second = serde_json::to_vec(&generate_index(&rows, "fixture.sql".into())).unwrap();

        assert_eq!(first, second);
        assert!(!first.contains(&b'\n'));
    }

    #[test]
    fn rejects_malformed_rows_and_unknown_gender_values() {
        let malformed = "INSERT INTO `npc_types` VALUES\n(1,'too short');\n";
        assert!(parse_sql(BufReader::new(malformed.as_bytes())).is_err());

        let invalid_gender = "INSERT INTO `npc_types` VALUES\n(1,'Bad','','',0,0,0,0,0,9);\n";
        assert!(parse_sql(BufReader::new(invalid_gender.as_bytes())).is_err());
    }
}