fix: reduce token usage, use short tool descriptions by default (#2679)
Andrey Nering
created
* Follow-up of #2592
We've been testing this for 2 weeks, and it's working great.
This is a very useful enhancement to reduce token usage, cost, and make
smaller models behave better.
This is particularly significant for local models. These models are tiny,
and with a long enough system prompt they have little context window left
for work.
It's still possible to have the old behavior by setting:
export CRUSH_SHORT_TOOL_DESCRIPTIONS=0
@@ -60,11 +60,11 @@ func GetModelNameFromContext(ctx context.Context) string {
}
// FirstLineDescription returns just the first non-empty line from the embedded
-// markdown description when CRUSH_SHORT_TOOL_DESCRIPTIONS is set, significantly-// reducing token usage. Otherwise returns the full description.
+// markdown description. The full description can be used by setting
+// CRUSH_SHORT_TOOL_DESCRIPTIONS=0.
func FirstLineDescription(content []byte) string {
if !testing.Testing() {
- if v, _ := strconv.ParseBool(os.Getenv("CRUSH_SHORT_TOOL_DESCRIPTIONS")); !v {
+ if v, err := strconv.ParseBool(os.Getenv("CRUSH_SHORT_TOOL_DESCRIPTIONS")); err == nil && !v {
return strings.TrimSpace(string(content))
}
}