diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..12577f8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.venv +.DS_Store/ +__pycache__/ +*.pyc +*.egg-info/ +dist/ + +.env \ No newline at end of file diff --git a/EverQuestLegends/README.md b/EverQuestLegends/README.md deleted file mode 100644 index 38c7f29..0000000 --- a/EverQuestLegends/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# EverQuest Legends configs - -Run `bash eql/update.sh` from the repository root to capture `eqclient.ini` and -the per-character player/UI INI files from `~/Games/EverQuestLegends/`. - -Set `EQL_GAME_PATH` to use another EverQuest Legends install directory: - -```bash -EQL_GAME_PATH=/path/to/EverQuestLegends bash eql/update.sh -``` diff --git a/EverQuestLegends/UI_Pleb_oggok_LO1.ini b/EverQuestLegends/UI_Pleb_oggok_LO1.ini index 4232af2..53a2707 100644 --- a/EverQuestLegends/UI_Pleb_oggok_LO1.ini +++ b/EverQuestLegends/UI_Pleb_oggok_LO1.ini @@ -426,8 +426,8 @@ XRef=left YRef=bottom XPos=0.000000% YPos=0.000000% -Width=98 -Height=271 +Width=528 +Height=56 BGTint.red=255 BGTint.green=255 BGTint.blue=255 diff --git a/EverQuestLive/README.md b/EverQuestLive/README.md new file mode 100644 index 0000000..3ea3221 --- /dev/null +++ b/EverQuestLive/README.md @@ -0,0 +1,121 @@ +# Revisiting EverQuest +Whenever I play this game, I invariably change most of the default settings. This document exists to remind of what to do when I will inevitably try to play this game again (and again, and again..). While keeping a copy of eqclient.ini has been sufficient at times, I've noticed the game client tramples much of the config after merging updates. + +### Maximized Fullscreen +The game client can be coerced to run in a borderless windowed mode. Window resizing should be disabled first, otherwise the mouse will be offset incorrectly. You can find this setting in game under: + +*Options* > *Display* > *Allow window resizing* + +Or in eqclient.ini: +```ini +AllowResize=0 +``` + +An alternative way to do this is to leave window resizing enabled, and manually fix the mouse offset. Might be useful if you really want to resize the window to something less than full screen (maybe to run more than one eq client). +```ini +WindowedModeXOffset=2 +WindowedModeYOffset=1 +AllowResize=1 +``` + +The window borders can be removed by a third party utility such as this [autohotkey script](http://gaming.stackexchange.com/a/17307/7413). Just make sure to run the script as Admin, and perform the resize after logging into the character. If the resize occurs on the character selection screen, the renderer won't realize and you'll need to reset the resolution to fix screen stretching. +```autohotkey + ^!h:: + IfWinExist EverQuest + { + WinSet, Style, -0xC00000 ; hide title bar + WinSet, Style, -0x40000 ; hide thickframe/sizebox + WinMove, , , 0, 0, 1920, 1080 + } + return +``` + +### Texture Flickering (aka Z-Fighting) +In-game lighting appears to be [broken by default](https://forums.daybreakgames.com/eq/index.php?threads/technical-question-about-dynamic-lighting.251405/) on modern systems, with no in-game UI option that sufficiently resolves it. The workaround is to use the command: /dynamic off +```ini +ShowDynamicLights=0 +``` +Shadows will eventually begin mis-rendering all over the place and occasionally just start flickering. Turn off shadows in game! + +I like to tweak the default FPS limits. +```ini +MaxFPS=60 +MaxBGFPS=60 +``` + +The windowed gamma feature should be off by default now, but just in case: +```ini +WindowedGamma=0 +``` + +### Controls +Mouselook sensitivity can be adjusted under mouse options. The camera mode can be toggled with F9. Roll the mousewheel while in the default first person camera mode to freely adjust it into a reasonably comfortable third-person mode. + +Definitely enable "Click Through Self" in General Options. Otherwise you'll have a hard time clicking anything remotely near your character. The target selection isn't quite so advanced. + +### Hotkeys +Custom keyboard configuration is in eqclient.ini [KeyMaps] section. Note that on a fresh install, [KeyMaps] won't be there. I tend to unbind keys for infrequently used features that are accessible through the in-game menu. + +### Minimal User Interface +A good UI gets out of your way. [My minimal UI](https://i.imgur.com/mcO4oAA.jpg) is available in this repo in diffs. Maintaining my custom UI changes as a patch set makes integrating official updates easier. I like to set the fade transparency level for most windows to 0%. The exceptions are the hotbars, spellbars and chat windows. I've avoided changing UI panels that are not persistently on screen in order to minimize the long-term maintenance. + +**Out of scope:** `AtlasSkin` / `uiresources/` (guild, achievements, overseer, and other HTML panels) — only the classic `UISkin` XML layer is themed; modern panels stay stock Default. + +Verify patches: `bash ui/verify.sh` (see [AGENTS.md](AGENTS.md)). + +If you've installed [bash on Windows](https://docs.microsoft.com/en-us/windows/wsl/install-win10), you could install the UI with the following commands: + +```bash +git clone https://github.com/isuldor/eqclient-conf +cd eqclient-conf/ui +./install.sh +``` + +Copy UI_Player_server.ini into the game directory and try loading the window layout from it under General settings. + +### Text colors +Chat colors are stored as separate RGB values in eqclient.ini [TextColors] section. The default color scheme is a relatively dark palette. I've implemented the ir_black theme and it can be found in the relevant section in the included eqclient.ini in this repo. This is an ideal light text on dark background scheme, so it'll only make sense if you have dark chat windows like I do. +I have experimented with [gradient backgrounds for chat windows](/docs/gradient-10.jpg), but the result wasn't great because I could only get 10 levels of transparency from a tga texture ingame. + +### (Optionally) Reorganize game data +There's over 4.3 thousand files sitting in the root game directory. If that triggers your OCD (or you hate having to look for the same file over and over again), you can [reorganize everquest](https://www.eqinterface.com/forums/showthread.php?t=21379) using symlinks. This made a little more sense back when I used a slow spinning disk rather than striped ssd storage. + +### Extras +Useful information can be found at various third party websites: + +* [EQLogParser](https://github.com/kauffman12/EQLogParser) open source alternative to [Gamparse](http://gambosoft.eqresource.com/gamparsegettingstarted.php) +* [Brewall](http://www.eqmaps.info/) has detailed maps +* [EQStats](http://www.eqstats.net/) has a spell and item database +* [Raidloot](http://www.raidloot.com/) item and spell info +* [Lucy](http://lucy.allakhazam.com/) is a spell database +* [Allakhazam](http://eq.allakhazam.com/) records items and quests info +* [Magelo](//eq.magelo.com/) has player profiles +* [Traders Corner](http://www.eqtraders.com/) has tradeskill info +* [Beimeith](http://www.elitegamerslounge.com/home/progress/) is a server-wide leaderboard + + +Some of the oldest websites about EverQuest are still around. These are rich archives of EQ lore, quests and discussion: +* [Safehouse](https://thesafehouse.org/forums/forum/everquest-wing) +* [Shaman's Crucible](http://www.shamanscrucible.com/forum/) +* [Druid's Grove](http://thedruidsgrove.org/archive/eq/) +* [Paladins of Norrath](https://www.tapatalk.com/groups/paladinsofnorrath/index.php) +* [GU Comics](http://www.gucomics.com/comic/?cdate=20000710) + +There is plenty of lore in Norrath created by the developers. But the real story is about the [players](https://www.tapatalk.com/groups/sacredomen/we-are-all-old-now-t6985.html) [within](https://www.tapatalk.com/groups/sacredomen/hi-t6995.html) the [game](https://www.tapatalk.com/groups/sacredomen/tholuxe-paells-original-guild-manifestos-t6991.html), and the communities they created. +* There is a brief [lore](https://www.everquest.com/lore) page on the official website +* [Wikia](http://everquest.wikia.com/wiki/Lore) has a nascent lore article + +### Fixing Allakhazam +Ala is an ad-supported website with useful free content. I think they have a subscription membership that goes with Wowhead, which seems like a great idea if you use it frequently. Their advertisements even seem to get through uBlock Origin. Here are some UBO filters that you should totally never use. I'll aim to [contribute to their wiki](http://everquest.allakhazam.com/wiki.html?p=Special%3AUser_contributions&user=Isuldor) rather than be tormented by their advertisements. +``` +everquest.allakhazam.com###logo +everquest.allakhazam.com###row-top +everquest.allakhazam.com###col-right +everquest.allakhazam.com##a.pw-button +everquest.allakhazam.com###gdpr-dashboard +everquest.allakhazam.com###gdpr-toggle +everquest.allakhazam.com###zul-bar +everquest.allakhazam.com###horizontal-bg +everquest.allakhazam.com##footer +everquest.allakhazam.com###wrapperDiv +``` diff --git a/README.md b/README.md index 3ea3221..bb658ed 100644 --- a/README.md +++ b/README.md @@ -1,121 +1,17 @@ -# Revisiting EverQuest -Whenever I play this game, I invariably change most of the default settings. This document exists to remind of what to do when I will inevitably try to play this game again (and again, and again..). While keeping a copy of eqclient.ini has been sufficient at times, I've noticed the game client tramples much of the config after merging updates. +# EQ Player Profile -### Maximized Fullscreen -The game client can be coerced to run in a borderless windowed mode. Window resizing should be disabled first, otherwise the mouse will be offset incorrectly. You can find this setting in game under: +My configuration notes for EverQuest. -*Options* > *Display* > *Allow window resizing* +## Characters -Or in eqclient.ini: -```ini -AllowResize=0 -``` +EQ Legends -An alternative way to do this is to leave window resizing enabled, and manually fix the mouse offset. Might be useful if you really want to resize the window to something less than full screen (maybe to run more than one eq client). -```ini -WindowedModeXOffset=2 -WindowedModeYOffset=1 -AllowResize=1 -``` +* Pleb on Oggok -The window borders can be removed by a third party utility such as this [autohotkey script](http://gaming.stackexchange.com/a/17307/7413). Just make sure to run the script as Admin, and perform the resize after logging into the character. If the resize occurs on the character selection screen, the renderer won't realize and you'll need to reset the resolution to fix screen stretching. -```autohotkey - ^!h:: - IfWinExist EverQuest - { - WinSet, Style, -0xC00000 ; hide title bar - WinSet, Style, -0x40000 ; hide thickframe/sizebox - WinMove, , , 0, 0, 1920, 1080 - } - return -``` +EQ Live -### Texture Flickering (aka Z-Fighting) -In-game lighting appears to be [broken by default](https://forums.daybreakgames.com/eq/index.php?threads/technical-question-about-dynamic-lighting.251405/) on modern systems, with no in-game UI option that sufficiently resolves it. The workaround is to use the command: /dynamic off -```ini -ShowDynamicLights=0 -``` -Shadows will eventually begin mis-rendering all over the place and occasionally just start flickering. Turn off shadows in game! +* Vulch on Coirnav -I like to tweak the default FPS limits. -```ini -MaxFPS=60 -MaxBGFPS=60 -``` +## Links -The windowed gamma feature should be off by default now, but just in case: -```ini -WindowedGamma=0 -``` - -### Controls -Mouselook sensitivity can be adjusted under mouse options. The camera mode can be toggled with F9. Roll the mousewheel while in the default first person camera mode to freely adjust it into a reasonably comfortable third-person mode. - -Definitely enable "Click Through Self" in General Options. Otherwise you'll have a hard time clicking anything remotely near your character. The target selection isn't quite so advanced. - -### Hotkeys -Custom keyboard configuration is in eqclient.ini [KeyMaps] section. Note that on a fresh install, [KeyMaps] won't be there. I tend to unbind keys for infrequently used features that are accessible through the in-game menu. - -### Minimal User Interface -A good UI gets out of your way. [My minimal UI](https://i.imgur.com/mcO4oAA.jpg) is available in this repo in diffs. Maintaining my custom UI changes as a patch set makes integrating official updates easier. I like to set the fade transparency level for most windows to 0%. The exceptions are the hotbars, spellbars and chat windows. I've avoided changing UI panels that are not persistently on screen in order to minimize the long-term maintenance. - -**Out of scope:** `AtlasSkin` / `uiresources/` (guild, achievements, overseer, and other HTML panels) — only the classic `UISkin` XML layer is themed; modern panels stay stock Default. - -Verify patches: `bash ui/verify.sh` (see [AGENTS.md](AGENTS.md)). - -If you've installed [bash on Windows](https://docs.microsoft.com/en-us/windows/wsl/install-win10), you could install the UI with the following commands: - -```bash -git clone https://github.com/isuldor/eqclient-conf -cd eqclient-conf/ui -./install.sh -``` - -Copy UI_Player_server.ini into the game directory and try loading the window layout from it under General settings. - -### Text colors -Chat colors are stored as separate RGB values in eqclient.ini [TextColors] section. The default color scheme is a relatively dark palette. I've implemented the ir_black theme and it can be found in the relevant section in the included eqclient.ini in this repo. This is an ideal light text on dark background scheme, so it'll only make sense if you have dark chat windows like I do. -I have experimented with [gradient backgrounds for chat windows](/docs/gradient-10.jpg), but the result wasn't great because I could only get 10 levels of transparency from a tga texture ingame. - -### (Optionally) Reorganize game data -There's over 4.3 thousand files sitting in the root game directory. If that triggers your OCD (or you hate having to look for the same file over and over again), you can [reorganize everquest](https://www.eqinterface.com/forums/showthread.php?t=21379) using symlinks. This made a little more sense back when I used a slow spinning disk rather than striped ssd storage. - -### Extras -Useful information can be found at various third party websites: - -* [EQLogParser](https://github.com/kauffman12/EQLogParser) open source alternative to [Gamparse](http://gambosoft.eqresource.com/gamparsegettingstarted.php) -* [Brewall](http://www.eqmaps.info/) has detailed maps -* [EQStats](http://www.eqstats.net/) has a spell and item database -* [Raidloot](http://www.raidloot.com/) item and spell info -* [Lucy](http://lucy.allakhazam.com/) is a spell database -* [Allakhazam](http://eq.allakhazam.com/) records items and quests info -* [Magelo](//eq.magelo.com/) has player profiles -* [Traders Corner](http://www.eqtraders.com/) has tradeskill info -* [Beimeith](http://www.elitegamerslounge.com/home/progress/) is a server-wide leaderboard - - -Some of the oldest websites about EverQuest are still around. These are rich archives of EQ lore, quests and discussion: -* [Safehouse](https://thesafehouse.org/forums/forum/everquest-wing) -* [Shaman's Crucible](http://www.shamanscrucible.com/forum/) -* [Druid's Grove](http://thedruidsgrove.org/archive/eq/) -* [Paladins of Norrath](https://www.tapatalk.com/groups/paladinsofnorrath/index.php) -* [GU Comics](http://www.gucomics.com/comic/?cdate=20000710) - -There is plenty of lore in Norrath created by the developers. But the real story is about the [players](https://www.tapatalk.com/groups/sacredomen/we-are-all-old-now-t6985.html) [within](https://www.tapatalk.com/groups/sacredomen/hi-t6995.html) the [game](https://www.tapatalk.com/groups/sacredomen/tholuxe-paells-original-guild-manifestos-t6991.html), and the communities they created. -* There is a brief [lore](https://www.everquest.com/lore) page on the official website -* [Wikia](http://everquest.wikia.com/wiki/Lore) has a nascent lore article - -### Fixing Allakhazam -Ala is an ad-supported website with useful free content. I think they have a subscription membership that goes with Wowhead, which seems like a great idea if you use it frequently. Their advertisements even seem to get through uBlock Origin. Here are some UBO filters that you should totally never use. I'll aim to [contribute to their wiki](http://everquest.allakhazam.com/wiki.html?p=Special%3AUser_contributions&user=Isuldor) rather than be tormented by their advertisements. -``` -everquest.allakhazam.com###logo -everquest.allakhazam.com###row-top -everquest.allakhazam.com###col-right -everquest.allakhazam.com##a.pw-button -everquest.allakhazam.com###gdpr-dashboard -everquest.allakhazam.com###gdpr-toggle -everquest.allakhazam.com###zul-bar -everquest.allakhazam.com###horizontal-bg -everquest.allakhazam.com##footer -everquest.allakhazam.com###wrapperDiv -``` +* Vulch's [Actually minimal](https://www.eqinterface.com/forums/showthread.php?t=22113) UI diff --git a/docs/6117RAK2guL._AC_UY218_.jpg b/docs/6117RAK2guL._AC_UY218_.jpg deleted file mode 100644 index 1b6e662..0000000 Binary files a/docs/6117RAK2guL._AC_UY218_.jpg and /dev/null differ diff --git a/docs/gradient-10.jpg b/docs/UI_Isuldor_brells-gradient.jpg similarity index 100% rename from docs/gradient-10.jpg rename to docs/UI_Isuldor_brells-gradient.jpg diff --git a/docs/UI_Vulch_coirnav.jpg b/docs/UI_Vulch_coirnav.jpg new file mode 100644 index 0000000..36b825e Binary files /dev/null and b/docs/UI_Vulch_coirnav.jpg differ diff --git a/docs/eql-spell-data.md b/docs/eql-spell-data.md new file mode 100644 index 0000000..b320078 --- /dev/null +++ b/docs/eql-spell-data.md @@ -0,0 +1,59 @@ +# EverQuest Legends spell data + +`scripts/eql-spells` creates a local SQLite search index from the EQL client +files. The client installation remains the source of truth: neither the raw +files nor the generated database are tracked in this repository. + +## Sources and cache + +The command reads these files from the EQL game directory: + +* `spells_us.txt` — spell ID, name, and client numeric fields. +* `spells_us_str.txt` — caster, target, and fade messages. +* `dbstr_us.txt` — spell descriptions (`type` `6`). + +It resolves that directory in this order: + +1. `--game-path PATH` +2. `EQL_GAME_PATH` +3. `/home/pleb/Games/EverQuestLegends` + +The default generated index is +`$XDG_CACHE_HOME/eqclient-conf/eql-spells.sqlite`, or +`~/.cache/eqclient-conf/eql-spells.sqlite` when `XDG_CACHE_HOME` is unset. +Use `--db PATH` to use a different index, such as one for a test fixture. + +Each query compares the client files' paths, sizes, nanosecond modification +times, and `:crc` sidecars (when supplied) with index metadata. It rebuilds +automatically after an EQL update. `build` always forces a rebuild. + +## Commands + +```bash +# Force an index build. +scripts/eql-spells build + +# Exact lookup by numeric spell ID or case-insensitive exact name. +scripts/eql-spells get 4 +scripts/eql-spells get 'Summon Waterstone' + +# Include the complete original caret-delimited row and its fields. +scripts/eql-spells get 4 --raw + +# Name substring lookup and full-text name/description search. +scripts/eql-spells find water --limit 10 +scripts/eql-spells search 'underwater breathing' + +# Use a different EQL installation or cache location. +scripts/eql-spells --game-path /path/to/EverQuestLegends --db /tmp/spells.sqlite search fire +``` + +Successful commands emit JSON on stdout. `get` returns `spell`; `find` and +`search` return `results`. All include source fingerprint metadata. Add +`--text` to query commands for compact human-readable output. + +The first version intentionally exposes only reliably joined fields: ID, name, +description, cast messages, and (with `--raw`) the complete original row. The +numeric columns in `spells_us.txt` have no embedded schema, so class, level, +mana, and effect filters should be added only after their mapping has been +verified for this particular client build. diff --git a/scripts/eql-spells b/scripts/eql-spells new file mode 100755 index 0000000..ddbf25e --- /dev/null +++ b/scripts/eql-spells @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +"""Command-line entry point for the local EQL spell-data index.""" + +from eql_spells import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/eql_spells.py b/scripts/eql_spells.py new file mode 100644 index 0000000..6aa6981 --- /dev/null +++ b/scripts/eql_spells.py @@ -0,0 +1,479 @@ +"""Build and query a local EverQuest Legends spell-data index. + +The client files are the source of truth. This module only creates a local, +rebuildable SQLite cache and deliberately does not assign meaning to the +undocumented positional fields in spells_us.txt. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sqlite3 +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + + +SCHEMA_VERSION = 2 +DEFAULT_GAME_PATH = Path("/home/pleb/Games/EverQuestLegends") +SOURCE_FILENAMES = ("spells_us.txt", "spells_us_str.txt", "dbstr_us.txt") +MESSAGE_COLUMNS = ( + "caster_me", + "caster_other", + "casted_me", + "casted_other", + "spell_gone", +) + + +class EqlSpellsError(Exception): + """An expected user-facing command error.""" + + +class UsageError(EqlSpellsError): + """The command line is invalid.""" + + +class DataError(EqlSpellsError): + """The EQL source files or index are invalid.""" + + +class JSONArgumentParser(argparse.ArgumentParser): + def error(self, message: str) -> None: + raise UsageError(message) + + +@dataclass(frozen=True) +class SourceInfo: + game_path: Path + fingerprint: dict[str, Any] + + +def default_cache_path() -> Path: + cache_home = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + return cache_home / "eqclient-conf" / "eql-spells.sqlite" + + +def resolve_game_path(value: str | None) -> Path: + if value: + return Path(value).expanduser().resolve() + if os.environ.get("EQL_GAME_PATH"): + return Path(os.environ["EQL_GAME_PATH"]).expanduser().resolve() + return DEFAULT_GAME_PATH + + +def source_info(game_path: Path) -> SourceInfo: + game_path = game_path.resolve() + if not game_path.is_dir(): + raise DataError( + f"EverQuest Legends directory not found: {game_path}. " + "Set --game-path or EQL_GAME_PATH." + ) + + sources: list[dict[str, Any]] = [] + for filename in SOURCE_FILENAMES: + path = game_path / filename + if not path.is_file(): + raise DataError(f"Required spell data file not found: {path}") + try: + stat = path.stat() + crc_path = Path(f"{path}:crc") + crc = crc_path.read_text(encoding="ascii").strip() if crc_path.is_file() else None + except OSError as error: + raise DataError(f"Unable to inspect {path}: {error}") from error + sources.append( + { + "name": filename, + "path": str(path), + "size": stat.st_size, + "mtime_ns": stat.st_mtime_ns, + "crc": crc, + } + ) + fingerprint = {"schema_version": SCHEMA_VERSION, "sources": sources} + return SourceInfo(game_path=game_path, fingerprint=fingerprint) + + +def connect(path: Path) -> sqlite3.Connection: + connection = sqlite3.connect(path) + connection.row_factory = sqlite3.Row + return connection + + +def create_schema(connection: sqlite3.Connection) -> None: + connection.executescript( + """ + PRAGMA foreign_keys = ON; + CREATE TABLE metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE spells ( + spell_id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + name_fold TEXT NOT NULL, + raw_line TEXT NOT NULL, + fields_json TEXT NOT NULL + ); + CREATE TABLE spell_messages ( + spell_id INTEGER PRIMARY KEY REFERENCES spells(spell_id), + caster_me TEXT NOT NULL DEFAULT '', + caster_other TEXT NOT NULL DEFAULT '', + casted_me TEXT NOT NULL DEFAULT '', + casted_other TEXT NOT NULL DEFAULT '', + spell_gone TEXT NOT NULL DEFAULT '' + ); + CREATE TABLE descriptions ( + spell_id INTEGER PRIMARY KEY REFERENCES spells(spell_id), + description TEXT NOT NULL + ); + CREATE INDEX spells_name_fold_idx ON spells(name_fold); + CREATE VIRTUAL TABLE spell_fts USING fts5(name, description, tokenize = 'porter unicode61'); + """ + ) + + +def source_lines(path: Path) -> Iterable[tuple[int, str]]: + try: + with path.open("r", encoding="utf-8", errors="replace", newline="") as handle: + for line_number, line in enumerate(handle, start=1): + yield line_number, line.rstrip("\r\n") + except OSError as error: + raise DataError(f"Unable to read {path}: {error}") from error + + +def parse_spell_rows(path: Path) -> list[tuple[int, str, str, str, str]]: + rows: list[tuple[int, str, str, str, str]] = [] + seen: set[int] = set() + for line_number, line in source_lines(path): + if not line: + continue + fields = line.split("^") + if len(fields) < 2 or not fields[1]: + raise DataError(f"Malformed spell row in {path}:{line_number}") + try: + spell_id = int(fields[0]) + except ValueError as error: + raise DataError(f"Malformed spell ID in {path}:{line_number}") from error + if spell_id in seen: + raise DataError(f"Duplicate spell ID {spell_id} in {path}:{line_number}") + seen.add(spell_id) + rows.append((spell_id, fields[1], fields[1].casefold(), line, json.dumps(fields))) + if not rows: + raise DataError(f"No spell rows found in {path}") + return rows + + +def parse_messages(path: Path) -> list[tuple[int, str, str, str, str, str]]: + rows: list[tuple[int, str, str, str, str, str]] = [] + seen: set[int] = set() + for line_number, line in source_lines(path): + if not line or line.startswith("#SPELLINDEX^"): + continue + fields = line.split("^") + if len(fields) < 6: + raise DataError(f"Malformed message row in {path}:{line_number}") + try: + spell_id = int(fields[0]) + except ValueError as error: + raise DataError(f"Malformed message spell ID in {path}:{line_number}") from error + if spell_id in seen: + raise DataError(f"Duplicate message spell ID {spell_id} in {path}:{line_number}") + seen.add(spell_id) + rows.append((spell_id, *fields[1:6])) + return rows + + +def parse_descriptions(path: Path, spell_ids: set[int]) -> list[tuple[int, str]]: + rows: list[tuple[int, str]] = [] + seen: set[int] = set() + for line_number, line in source_lines(path): + if not line: + continue + fields = line.split("^") + if len(fields) < 3 or fields[1] != "6": + continue + try: + spell_id = int(fields[0]) + except ValueError as error: + raise DataError(f"Malformed description spell ID in {path}:{line_number}") from error + # dbstr_us.txt is a shared string table. Type-6 entries outside the + # client spell table belong to other UI data and are not spell rows. + if spell_id not in spell_ids: + continue + if spell_id in seen: + raise DataError(f"Duplicate spell description for ID {spell_id} in {path}:{line_number}") + seen.add(spell_id) + rows.append((spell_id, fields[2])) + return rows + + +def validate_joins(spell_ids: set[int], messages: list[tuple[int, str, str, str, str, str]]) -> None: + orphan_messages = [row[0] for row in messages if row[0] not in spell_ids] + if orphan_messages: + raise DataError(f"Message row references unknown spell ID {orphan_messages[0]}") + + +def build_index(db_path: Path, info: SourceInfo) -> dict[str, int]: + spells_path, messages_path, descriptions_path = ( + info.game_path / filename for filename in SOURCE_FILENAMES + ) + spells = parse_spell_rows(spells_path) + messages = parse_messages(messages_path) + spell_ids = {row[0] for row in spells} + descriptions = parse_descriptions(descriptions_path, spell_ids) + validate_joins(spell_ids, messages) + + db_path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{db_path.name}.", suffix=".tmp", dir=db_path.parent + ) + os.close(descriptor) + temporary_path = Path(temporary_name) + try: + connection = connect(temporary_path) + try: + create_schema(connection) + with connection: + connection.executemany( + "INSERT INTO spells (spell_id, name, name_fold, raw_line, fields_json) VALUES (?, ?, ?, ?, ?)", + spells, + ) + connection.executemany( + """INSERT INTO spell_messages + (spell_id, caster_me, caster_other, casted_me, casted_other, spell_gone) + VALUES (?, ?, ?, ?, ?, ?)""", + messages, + ) + connection.executemany( + "INSERT INTO descriptions (spell_id, description) VALUES (?, ?)", descriptions, + ) + connection.execute( + """INSERT INTO spell_fts (rowid, name, description) + SELECT spells.spell_id, spells.name, COALESCE(descriptions.description, '') + FROM spells LEFT JOIN descriptions USING (spell_id)""" + ) + connection.execute( + "INSERT INTO metadata (key, value) VALUES (?, ?)", + ("source", json.dumps({"game_path": str(info.game_path), "fingerprint": info.fingerprint}, sort_keys=True)), + ) + count = connection.execute("SELECT count(*) FROM spells").fetchone()[0] + fts_count = connection.execute("SELECT count(*) FROM spell_fts").fetchone()[0] + if count != len(spells) or fts_count != len(spells): + raise DataError("Built index failed row-count validation") + finally: + connection.close() + os.replace(temporary_path, db_path) + except Exception: + temporary_path.unlink(missing_ok=True) + raise + return {"spells": len(spells), "messages": len(messages), "descriptions": len(descriptions)} + + +def stored_source(db_path: Path) -> dict[str, Any] | None: + if not db_path.is_file(): + return None + try: + connection = connect(db_path) + try: + row = connection.execute("SELECT value FROM metadata WHERE key = 'source'").fetchone() + finally: + connection.close() + return json.loads(row[0]) if row else None + except (OSError, sqlite3.Error, json.JSONDecodeError): + return None + + +def ensure_index(db_path: Path, info: SourceInfo) -> tuple[bool, dict[str, int] | None]: + expected = {"game_path": str(info.game_path), "fingerprint": info.fingerprint} + if stored_source(db_path) == expected: + return False, None + return True, build_index(db_path, info) + + +def source_metadata(connection: sqlite3.Connection) -> dict[str, Any]: + row = connection.execute("SELECT value FROM metadata WHERE key = 'source'").fetchone() + if row is None: + raise DataError("Index metadata is missing") + return json.loads(row[0]) + + +def spell_from_row(row: sqlite3.Row, include_raw: bool = False) -> dict[str, Any]: + messages = {column: row[column] for column in MESSAGE_COLUMNS} + result: dict[str, Any] = { + "id": row["spell_id"], + "name": row["name"], + "description": row["description"], + "messages": messages, + } + if include_raw: + result["raw"] = {"line": row["raw_line"], "fields": json.loads(row["fields_json"])} + return result + + +SELECT_SPELL = """ +SELECT spells.spell_id, spells.name, spells.raw_line, spells.fields_json, + descriptions.description, + COALESCE(spell_messages.caster_me, '') AS caster_me, + COALESCE(spell_messages.caster_other, '') AS caster_other, + COALESCE(spell_messages.casted_me, '') AS casted_me, + COALESCE(spell_messages.casted_other, '') AS casted_other, + COALESCE(spell_messages.spell_gone, '') AS spell_gone +FROM spells +LEFT JOIN descriptions USING (spell_id) +LEFT JOIN spell_messages USING (spell_id) +""" + + +def get_spell(connection: sqlite3.Connection, value: str, include_raw: bool) -> dict[str, Any]: + if value.isdecimal(): + row = connection.execute(SELECT_SPELL + " WHERE spells.spell_id = ?", (int(value),)).fetchone() + if row is None: + raise DataError(f"No spell found with ID {value}") + return spell_from_row(row, include_raw) + + rows = connection.execute( + SELECT_SPELL + " WHERE spells.name_fold = ? ORDER BY spells.spell_id", (value.casefold(),) + ).fetchall() + if not rows: + raise DataError(f"No spell found with exact name {value!r}") + if len(rows) > 1: + ids = [row["spell_id"] for row in rows] + raise DataError(f"Exact name {value!r} is ambiguous; matching IDs: {ids}") + return spell_from_row(rows[0], include_raw) + + +def find_spells(connection: sqlite3.Connection, text: str, limit: int) -> list[dict[str, Any]]: + escaped = text.casefold().replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + rows = connection.execute( + """SELECT spells.spell_id, spells.name, descriptions.description + FROM spells LEFT JOIN descriptions USING (spell_id) + WHERE spells.name_fold LIKE ? ESCAPE '\\' + ORDER BY spells.name_fold, spells.spell_id LIMIT ?""", + (f"%{escaped}%", limit), + ).fetchall() + return [{"id": row["spell_id"], "name": row["name"], "description": row["description"]} for row in rows] + + +def fts_query(text: str) -> str: + terms = re.findall(r"[^\W_]+", text, flags=re.UNICODE) + if not terms: + raise UsageError("search text must contain at least one letter or number") + return " AND ".join(f'"{term}"' for term in terms) + + +def search_spells(connection: sqlite3.Connection, text: str, limit: int) -> list[dict[str, Any]]: + rows = connection.execute( + """SELECT spells.spell_id, spells.name, descriptions.description, bm25(spell_fts) AS score + FROM spell_fts + JOIN spells ON spells.spell_id = spell_fts.rowid + LEFT JOIN descriptions USING (spell_id) + WHERE spell_fts MATCH ? + ORDER BY score, spells.name_fold, spells.spell_id LIMIT ?""", + (fts_query(text), limit), + ).fetchall() + return [ + {"id": row["spell_id"], "name": row["name"], "description": row["description"]} + for row in rows + ] + + +def parse_limit(value: str) -> int: + try: + limit = int(value) + except ValueError as error: + raise argparse.ArgumentTypeError("limit must be an integer") from error + if not 1 <= limit <= 100: + raise argparse.ArgumentTypeError("limit must be between 1 and 100") + return limit + + +def parser() -> argparse.ArgumentParser: + argument_parser = JSONArgumentParser(prog="eql-spells", description=__doc__) + argument_parser.add_argument("--game-path", help="EverQuest Legends install directory") + argument_parser.add_argument("--db", type=Path, help="SQLite index path (default: user cache)") + subparsers = argument_parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("build", help="force a complete index rebuild") + + get_parser = subparsers.add_parser("get", help="look up a spell by ID or exact name") + get_parser.add_argument("value", help="numeric spell ID or exact spell name") + get_parser.add_argument("--raw", action="store_true", help="include original client row and fields") + get_parser.add_argument("--text", action="store_true", help="render compact text instead of JSON") + + for command, help_text in (("find", "find spell names"), ("search", "search spell names and descriptions")): + query_parser = subparsers.add_parser(command, help=help_text) + query_parser.add_argument("text") + query_parser.add_argument("--limit", type=parse_limit, default=20) + query_parser.add_argument("--text", dest="as_text", action="store_true", help="render compact text instead of JSON") + return argument_parser + + +def text_result(command: str, payload: dict[str, Any]) -> str: + if command == "get": + spell = payload["spell"] + lines = [f"{spell['id']}\t{spell['name']}"] + if spell["description"]: + lines.append(spell["description"]) + for label, value in spell["messages"].items(): + if value: + lines.append(f"{label}: {value}") + return "\n".join(lines) + return "\n".join( + f"{result['id']}\t{result['name']}" + (f"\t{result['description']}" if result["description"] else "") + for result in payload["results"] + ) + + +def emit_json(value: dict[str, Any], stream: Any = sys.stdout) -> None: + print(json.dumps(value, ensure_ascii=False, sort_keys=True), file=stream) + + +def main(argv: list[str] | None = None) -> int: + try: + arguments = parser().parse_args(argv) + game_path = resolve_game_path(arguments.game_path) + db_path = (arguments.db.expanduser() if arguments.db else default_cache_path()).resolve() + info = source_info(game_path) + + if arguments.command == "build": + counts = build_index(db_path, info) + emit_json({"built": counts, "db": str(db_path), "source": {"game_path": str(info.game_path), "fingerprint": info.fingerprint}}) + return 0 + + rebuilt, _ = ensure_index(db_path, info) + connection = connect(db_path) + try: + source = source_metadata(connection) + if arguments.command == "get": + payload = {"source": source, "spell": get_spell(connection, arguments.value, arguments.raw)} + as_text = arguments.text + elif arguments.command == "find": + payload = {"source": source, "results": find_spells(connection, arguments.text, arguments.limit)} + as_text = arguments.as_text + else: + payload = {"source": source, "results": search_spells(connection, arguments.text, arguments.limit)} + as_text = arguments.as_text + if rebuilt: + payload["index_rebuilt"] = True + if as_text: + print(text_result(arguments.command, payload)) + else: + emit_json(payload) + finally: + connection.close() + return 0 + except UsageError as error: + emit_json({"error": {"code": "usage", "message": str(error)}}, sys.stderr) + return 2 + except (DataError, OSError, sqlite3.Error) as error: + emit_json({"error": {"code": "data", "message": str(error)}}, sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/eql-spells/dbstr_us.txt b/tests/fixtures/eql-spells/dbstr_us.txt new file mode 100644 index 0000000..90d1f38 --- /dev/null +++ b/tests/fixtures/eql-spells/dbstr_us.txt @@ -0,0 +1,4 @@ +1^6^Launches a bolt of fire at your target.^0^ +2^6^Lets your target breathe underwater for a short time.^0^ +3^6^Launches a ball of fire at your target.^0^ +1^5^Fire^0^ diff --git a/tests/fixtures/eql-spells/spells_us.txt b/tests/fixtures/eql-spells/spells_us.txt new file mode 100644 index 0000000..dcee288 --- /dev/null +++ b/tests/fixtures/eql-spells/spells_us.txt @@ -0,0 +1,3 @@ +1^Fire Bolt^0^alpha +2^Water Breathing^0^beta +3^Fireball^0^gamma diff --git a/tests/fixtures/eql-spells/spells_us_str.txt b/tests/fixtures/eql-spells/spells_us_str.txt new file mode 100644 index 0000000..9347cc2 --- /dev/null +++ b/tests/fixtures/eql-spells/spells_us_str.txt @@ -0,0 +1,4 @@ +#SPELLINDEX^CASTERMETXT^CASTEROTHERTXT^CASTEDMETXT^CASTEDOTHERTXT^SPELLGONE^ +1^You cast Fire Bolt.^%1 casts Fire Bolt.^Fire burns you.^Fire burns %1.^The fire fades.^ +2^You breathe water.^%1 breathes water.^You can breathe water.^%1 can breathe water.^The water magic fades.^ +3^^^^^^ diff --git a/tests/test_eql_spells.py b/tests/test_eql_spells.py new file mode 100644 index 0000000..31a82ed --- /dev/null +++ b/tests/test_eql_spells.py @@ -0,0 +1,108 @@ +"""End-to-end tests for the dependency-free EQL spell CLI.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +CLI = ROOT / "scripts" / "eql-spells" +FIXTURE = ROOT / "tests" / "fixtures" / "eql-spells" + + +class EqlSpellsCliTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.work = Path(self.temporary.name) + self.game = self.work / "game" + shutil.copytree(FIXTURE, self.game) + self.db = self.work / "cache" / "spells.sqlite" + + def tearDown(self) -> None: + self.temporary.cleanup() + + def run_cli(self, *arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(CLI), "--game-path", str(self.game), "--db", str(self.db), *arguments], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + + def json_stdout(self, *arguments: str) -> dict: + result = self.run_cli(*arguments) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads(result.stdout) + + def test_build_and_exact_lookup_by_id_and_name(self) -> None: + built = self.json_stdout("build") + self.assertEqual(built["built"], {"descriptions": 3, "messages": 3, "spells": 3}) + self.assertTrue(self.db.is_file()) + + by_id = self.json_stdout("get", "1") + self.assertEqual(by_id["spell"]["name"], "Fire Bolt") + self.assertEqual(by_id["spell"]["description"], "Launches a bolt of fire at your target.") + self.assertEqual(by_id["spell"]["messages"]["caster_me"], "You cast Fire Bolt.") + self.assertIn("fingerprint", by_id["source"]) + + by_name = self.json_stdout("get", "water breathing") + self.assertEqual(by_name["spell"]["id"], 2) + + def test_find_search_and_raw_output(self) -> None: + found = self.json_stdout("find", "fire") + self.assertEqual([spell["name"] for spell in found["results"]], ["Fire Bolt", "Fireball"]) + + searched = self.json_stdout("search", "breathing underwater") + self.assertEqual([spell["id"] for spell in searched["results"]], [2]) + + raw = self.json_stdout("get", "1", "--raw") + self.assertEqual(raw["spell"]["raw"]["line"], "1^Fire Bolt^0^alpha") + self.assertEqual(raw["spell"]["raw"]["fields"], ["1", "Fire Bolt", "0", "alpha"]) + + def test_query_automatically_rebuilds_after_source_change(self) -> None: + self.json_stdout("build") + descriptions = self.game / "dbstr_us.txt" + with descriptions.open("a", encoding="utf-8") as handle: + handle.write("2^6^Lets your target breathe underwater indefinitely.^0^\n") + + result = self.run_cli("search", "indefinitely") + self.assertEqual(result.returncode, 1) + self.assertIn("Duplicate spell description", json.loads(result.stderr)["error"]["message"]) + + # Replace the existing description rather than adding a duplicate. + descriptions.write_text( + "1^6^Launches a bolt of fire at your target.^0^\n" + "2^6^Lets your target breathe underwater indefinitely.^0^\n" + "3^6^Launches a ball of fire at your target.^0^\n" + "1^5^Fire^0^\n", + encoding="utf-8", + ) + rebuilt = self.json_stdout("search", "indefinitely") + self.assertTrue(rebuilt["index_rebuilt"]) + self.assertEqual(rebuilt["results"][0]["id"], 2) + + def test_malformed_data_and_invalid_query_return_structured_errors(self) -> None: + (self.game / "spells_us.txt").write_text("bad row\n", encoding="utf-8") + malformed = self.run_cli("build") + self.assertEqual(malformed.returncode, 1) + self.assertEqual(json.loads(malformed.stderr)["error"]["code"], "data") + + shutil.copy2(FIXTURE / "spells_us.txt", self.game / "spells_us.txt") + invalid_limit = self.run_cli("find", "fire", "--limit", "101") + self.assertEqual(invalid_limit.returncode, 2) + self.assertEqual(json.loads(invalid_limit.stderr)["error"]["code"], "usage") + + missing = self.run_cli("get", "999") + self.assertEqual(missing.returncode, 1) + self.assertEqual(json.loads(missing.stderr)["error"]["code"], "data") + + +if __name__ == "__main__": + unittest.main()