1#!/usr/bin/env python3
2"""Search nimi.li for toki pona words by English meaning.
3
4Fetches data from https://nimi.li/__data.json (SvelteKit internal endpoint),
5decodes the dedup format, and searches definitions + ku_data keys.
6
7Usage:
8 python3 search-nimi.li.py fetch # Download + cache to cwd
9 python3 search-nimi.li.py friend # Search cached data
10 python3 search-nimi.li.py big important # Multi-word search
11"""
12
13import json
14import os
15import sys
16import urllib.request
17
18ENDPOINT = "https://nimi.li/__data.json"
19RAW_FILE = "nimi-data-raw.json"
20INDEX_FILE = "nimi-words.json"
21
22
23def fetch_data():
24 """Fetch nimi.li data and cache both raw + resolved index to cwd."""
25 req = urllib.request.Request(
26 ENDPOINT, headers={"User-Agent": "nimi-search/1.0"}
27 )
28 with urllib.request.urlopen(req) as resp:
29 raw = json.loads(resp.read())
30
31 with open(RAW_FILE, "w") as f:
32 json.dump(raw, f)
33
34 index = build_index(raw)
35
36 with open(INDEX_FILE, "w") as f:
37 json.dump(index, f, indent=2)
38
39 print(f"Fetched {len(index)} words, cached to {RAW_FILE} and {INDEX_FILE}")
40 return index
41
42
43def resolve(pool, val, depth=0):
44 """Resolve a value from SvelteKit's shared pool (dedup format).
45
46 The pool is a flat array where dicts/lists/strings are stored once and
47 referenced by integer index. This function follows those references
48 recursively to produce plain Python objects.
49 """
50 if depth > 8:
51 return val
52 if isinstance(val, int) and val < len(pool):
53 return resolve(pool, pool[val], depth + 1)
54 if isinstance(val, dict):
55 return {k: resolve(pool, v, depth + 1) for k, v in val.items()}
56 if isinstance(val, list):
57 return [resolve(pool, v, depth + 1) for v in val]
58 return val
59
60
61def build_index(raw):
62 """Extract word entries from the decoded data.
63
64 Returns a dict mapping word name -> {word, definition, ku_data, ...}.
65 """
66 pool = raw["nodes"][1]["data"]
67 words_dict = pool[1] # {word_name: index_into_pool}
68
69 index = {}
70 for name, word_idx in words_dict.items():
71 word_data = resolve(pool, word_idx)
72 trans = word_data.get("translations", {})
73 index[name] = {
74 "word": name,
75 "definition": trans.get("definition", ""),
76 "commentary": trans.get("commentary", ""),
77 "etymology": trans.get("etymology", ""),
78 "ku_data": word_data.get("ku_data", {}),
79 "source_language": word_data.get("source_language", ""),
80 "book": word_data.get("book", ""),
81 "usage_category": word_data.get("usage_category", ""),
82 }
83 return index
84
85
86def load_index():
87 """Load pre-built index from cwd, or fetch if missing."""
88 if os.path.exists(INDEX_FILE):
89 with open(INDEX_FILE) as f:
90 return json.load(f)
91 print(f"No cached data found in cwd. Run `{sys.argv[0]} fetch` first.")
92 sys.exit(1)
93
94
95def search(index, query):
96 """Search for a word by English meaning.
97
98 Searches definition text, ku_data keys (community usage terms), and
99 the word name itself. Returns results sorted by relevance.
100 """
101 q = query.lower()
102 results = []
103 for name, data in index.items():
104 score = 0
105 if q in data["definition"].lower():
106 score += 10
107 for ku_key in data.get("ku_data", {}):
108 if q in ku_key.lower():
109 score += 5
110 break
111 if q in name.lower():
112 score += 3
113 if q in data.get("commentary", "").lower():
114 score += 2
115 if score > 0:
116 results.append((score, data))
117 results.sort(key=lambda x: -x[0])
118 return results
119
120
121def main():
122 if len(sys.argv) < 2:
123 print(f"Usage: {sys.argv[0]} fetch | <english-word> [english-word ...]")
124 sys.exit(1)
125
126 cmd = sys.argv[1]
127
128 if cmd == "fetch":
129 fetch_data()
130 return
131
132 index = load_index()
133 queries = sys.argv[1:]
134
135 for query in queries:
136 results = search(index, query)
137 if not results:
138 print(f"No results for '{query}'.")
139 continue
140 print(f"\n=== '{query}' ===")
141 for _score, r in results:
142 print(f" {r['word']}: {r['definition']}")
143 if r["commentary"]:
144 print(f" ({r['commentary'][:120]})")
145
146
147if __name__ == "__main__":
148 main()