1//usr/bin/env go run "$0" "$@"; exit
2package main
3
4import (
5 "bytes"
6 "encoding/json"
7 "flag"
8 "fmt"
9 "io"
10 "net/http"
11 "os"
12 "path/filepath"
13 "regexp"
14 "sort"
15 "strings"
16 "sync"
17 "time"
18)
19
20const (
21 syntheticAPI = "https://api.synthetic.new/anthropic/v1/messages/count_tokens"
22 model = "syn:large:text"
23 workerCount = 5 // Number of parallel API workers
24 readmePath = "README.md"
25 skillsHeading = "## Available skills"
26 todoParagraph = "TODO: describe me"
27 maxBodyLines = 500
28)
29
30var httpClient = &http.Client{
31 Timeout: 30 * time.Second,
32}
33
34type Frontmatter struct {
35 Name string
36 Description string
37}
38
39// TokenCount breaks down a skill's token usage. Extra is the sum of every
40// .md file in the skill folder other than SKILL.md itself.
41type TokenCount struct {
42 Name int
43 Description int
44 Body int
45 Extra int
46 Total int
47}
48
49type SkillInfo struct {
50 Dir string
51 Frontmatter Frontmatter
52 BodyLines int
53 Tokens TokenCount
54 Errors []string
55}
56
57type TokenJob struct {
58 ID string
59 Text string
60}
61
62type TokenResult struct {
63 ID string
64 Count int
65 Err error
66}
67
68func main() {
69 workers := flag.Int("workers", workerCount, "Number of parallel API workers")
70 flag.Parse()
71
72 apiKey := os.Getenv("SYNTHETIC_API_KEY")
73 if apiKey == "" {
74 fmt.Fprintln(os.Stderr, "Error: SYNTHETIC_API_KEY environment variable not set")
75 os.Exit(1)
76 }
77
78 counter := newTokenCounter(apiKey, *workers)
79 defer counter.Close()
80
81 skills, err := analyzeSkills(counter)
82 if err != nil {
83 fmt.Fprintf(os.Stderr, "Error: %v\n", err)
84 os.Exit(1)
85 }
86
87 sort.Slice(skills, func(i, j int) bool {
88 return skills[i].Dir < skills[j].Dir
89 })
90
91 stats, err := updateReadme(readmePath, skills)
92 if err != nil {
93 fmt.Fprintf(os.Stderr, "Error updating README: %v\n", err)
94 os.Exit(1)
95 }
96
97 fmt.Fprintf(os.Stderr, "README updated: %d skills maintained, %d errored, %d present in README but missing on disk\n",
98 stats.Maintained, stats.Errored, stats.Missing)
99}
100
101type TokenCounter struct {
102 apiKey string
103 jobs chan TokenJob
104 results chan TokenResult
105 wg sync.WaitGroup
106}
107
108func newTokenCounter(apiKey string, workers int) *TokenCounter {
109 tc := &TokenCounter{
110 apiKey: apiKey,
111 jobs: make(chan TokenJob, 100),
112 results: make(chan TokenResult, 100),
113 }
114 tc.wg.Add(workers)
115 for i := 0; i < workers; i++ {
116 go tc.worker()
117 }
118 go func() {
119 tc.wg.Wait()
120 close(tc.results)
121 }()
122 return tc
123}
124
125func (tc *TokenCounter) worker() {
126 defer tc.wg.Done()
127 for job := range tc.jobs {
128 count, err := countTokensAPI(tc.apiKey, job.Text)
129 tc.results <- TokenResult{ID: job.ID, Count: count, Err: err}
130 }
131}
132
133func (tc *TokenCounter) Count(id, text string) {
134 tc.jobs <- TokenJob{ID: id, Text: text}
135}
136
137func (tc *TokenCounter) GetResult() TokenResult {
138 result, ok := <-tc.results
139 if !ok {
140 return TokenResult{Err: fmt.Errorf("results channel closed")}
141 }
142 return result
143}
144
145func (tc *TokenCounter) TryGetResult() (TokenResult, bool) {
146 select {
147 case result, ok := <-tc.results:
148 if !ok {
149 return TokenResult{}, false
150 }
151 return result, true
152 default:
153 return TokenResult{}, false
154 }
155}
156
157func (tc *TokenCounter) Close() {
158 close(tc.jobs)
159}
160
161func analyzeSkills(counter *TokenCounter) ([]SkillInfo, error) {
162 skillsDir := "skills"
163 entries, err := os.ReadDir(skillsDir)
164 if err != nil {
165 return nil, fmt.Errorf("cannot read skills directory: %w", err)
166 }
167
168 var skills []SkillInfo
169 for _, entry := range entries {
170 if !entry.IsDir() {
171 continue
172 }
173
174 skillPath := filepath.Join(skillsDir, entry.Name())
175 skill, err := analyzeSkill(skillPath, counter)
176 if err != nil {
177 fmt.Fprintf(os.Stderr, "Warning: error analyzing %s: %v\n", entry.Name(), err)
178 continue
179 }
180 skills = append(skills, skill)
181 }
182
183 return skills, nil
184}
185
186func analyzeSkill(path string, counter *TokenCounter) (SkillInfo, error) {
187 skill := SkillInfo{Dir: filepath.Base(path)}
188
189 // Read SKILL.md
190 skillMdPath := filepath.Join(path, "SKILL.md")
191 content, err := os.ReadFile(skillMdPath)
192 if err != nil {
193 skill.Errors = append(skill.Errors, fmt.Sprintf("Cannot read SKILL.md: %v", err))
194 return skill, nil
195 }
196
197 // Parse frontmatter and body
198 fm, body, err := parseFrontmatter(string(content))
199 if err != nil {
200 skill.Errors = append(skill.Errors, fmt.Sprintf("Cannot parse frontmatter: %v", err))
201 return skill, nil
202 }
203 skill.Frontmatter = fm
204 trimmedBody := strings.TrimSpace(body)
205 if trimmedBody == "" {
206 skill.BodyLines = 0
207 } else {
208 skill.BodyLines = len(strings.Split(trimmedBody, "\n"))
209 }
210
211 // Validate
212 skill.Errors = append(skill.Errors, validateSkill(skill)...)
213
214 fmt.Fprintf(os.Stderr, "Analyzing %s...\n", skill.Dir)
215
216 // Collect token-counting jobs. SKILL.md is counted as its frontmatter
217 // name+description fields plus the body; every other .md file in the
218 // folder is counted wholesale.
219 var jobs []TokenJob
220 jobs = append(jobs, TokenJob{ID: fmt.Sprintf("%s:name", skill.Dir), Text: fm.Name})
221 jobs = append(jobs, TokenJob{ID: fmt.Sprintf("%s:description", skill.Dir), Text: fm.Description})
222 jobs = append(jobs, TokenJob{ID: fmt.Sprintf("%s:body", skill.Dir), Text: body})
223
224 err = filepath.WalkDir(path, func(p string, d os.DirEntry, walkErr error) error {
225 if walkErr != nil {
226 skill.Errors = append(skill.Errors, fmt.Sprintf("Cannot walk %s: %v", p, walkErr))
227 return nil
228 }
229 if d.IsDir() {
230 return nil
231 }
232 if !strings.HasSuffix(p, ".md") {
233 return nil
234 }
235 // SKILL.md is already covered by the name/description/body jobs.
236 if p == skillMdPath {
237 return nil
238 }
239 fileContent, err := os.ReadFile(p)
240 if err != nil {
241 skill.Errors = append(skill.Errors, fmt.Sprintf("Cannot read %s: %v", p, err))
242 return nil
243 }
244 relPath, _ := filepath.Rel(path, p)
245 jobs = append(jobs, TokenJob{
246 ID: fmt.Sprintf("%s:file:%s", skill.Dir, relPath),
247 Text: string(fileContent),
248 })
249 return nil
250 })
251 if err != nil {
252 skill.Errors = append(skill.Errors, fmt.Sprintf("Walk error: %v", err))
253 }
254
255 // Interleave enqueue and drain to avoid backpressure on the buffered channels.
256 processResult := func(result TokenResult) {
257 if result.Err != nil {
258 skill.Errors = append(skill.Errors, fmt.Sprintf("Token count failed for %s: %v", result.ID, result.Err))
259 return
260 }
261 parts := strings.SplitN(result.ID, ":", 3)
262 if len(parts) < 2 {
263 return
264 }
265 switch parts[1] {
266 case "name":
267 skill.Tokens.Name = result.Count
268 case "description":
269 skill.Tokens.Description = result.Count
270 case "body":
271 skill.Tokens.Body = result.Count
272 case "file":
273 skill.Tokens.Extra += result.Count
274 }
275 }
276
277 outstanding := 0
278 for _, job := range jobs {
279 counter.Count(job.ID, job.Text)
280 outstanding++
281 for {
282 if result, ok := counter.TryGetResult(); ok {
283 processResult(result)
284 outstanding--
285 } else {
286 break
287 }
288 }
289 }
290 for outstanding > 0 {
291 result := counter.GetResult()
292 processResult(result)
293 outstanding--
294 }
295
296 skill.Tokens.Total = skill.Tokens.Name + skill.Tokens.Description + skill.Tokens.Body + skill.Tokens.Extra
297
298 return skill, nil
299}
300
301func parseFrontmatter(content string) (Frontmatter, string, error) {
302 lines := strings.Split(content, "\n")
303 if len(lines) < 3 || lines[0] != "---" {
304 return Frontmatter{}, "", fmt.Errorf("missing frontmatter")
305 }
306
307 var fm Frontmatter
308 var endIdx int
309 var inDescription bool
310 var descriptionLines []string
311
312 for i := 1; i < len(lines); i++ {
313 if lines[i] == "---" {
314 endIdx = i
315 break
316 }
317
318 line := lines[i]
319
320 // Parse name
321 if strings.HasPrefix(line, "name:") {
322 fm.Name = strings.TrimSpace(strings.TrimPrefix(line, "name:"))
323 continue
324 }
325
326 // Parse description (might be multi-line)
327 if strings.HasPrefix(line, "description:") {
328 descPart := strings.TrimSpace(strings.TrimPrefix(line, "description:"))
329 if descPart != "" {
330 descriptionLines = append(descriptionLines, descPart)
331 }
332 inDescription = true
333 continue
334 }
335
336 // Continue multi-line description
337 if inDescription && strings.HasPrefix(line, " ") {
338 descriptionLines = append(descriptionLines, strings.TrimSpace(line))
339 continue
340 }
341
342 // End of description
343 if inDescription && !strings.HasPrefix(line, " ") {
344 inDescription = false
345 }
346 }
347
348 fm.Description = strings.Join(descriptionLines, " ")
349
350 if endIdx == 0 {
351 return Frontmatter{}, "", fmt.Errorf("unclosed frontmatter")
352 }
353
354 body := strings.Join(lines[endIdx+1:], "\n")
355 return fm, body, nil
356}
357
358func validateSkill(skill SkillInfo) []string {
359 var errors []string
360
361 // Validate name
362 if len(skill.Frontmatter.Name) < 1 || len(skill.Frontmatter.Name) > 64 {
363 errors = append(errors, "name must be 1-64 characters")
364 }
365
366 namePattern := regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*$`)
367 if !namePattern.MatchString(skill.Frontmatter.Name) {
368 errors = append(errors, "name must be lowercase letters, numbers, hyphens only; no leading/trailing/consecutive hyphens")
369 }
370
371 if skill.Frontmatter.Name != skill.Dir {
372 errors = append(errors, fmt.Sprintf("name '%s' doesn't match directory '%s'", skill.Frontmatter.Name, skill.Dir))
373 }
374
375 // Validate description
376 if len(skill.Frontmatter.Description) < 1 {
377 errors = append(errors, "description is empty")
378 } else if len(skill.Frontmatter.Description) > 1024 {
379 errors = append(errors, fmt.Sprintf("description is %d characters (max 1024)", len(skill.Frontmatter.Description)))
380 }
381
382 // Check body line count
383 if skill.BodyLines > maxBodyLines {
384 errors = append(errors, fmt.Sprintf("body has %d lines (recommended: < %d)", skill.BodyLines, maxBodyLines))
385 }
386
387 return errors
388}
389
390func countTokensAPI(apiKey, text string) (int, error) {
391 reqBody := map[string]interface{}{
392 "model": model,
393 "messages": []map[string]string{
394 {
395 "role": "user",
396 "content": text,
397 },
398 },
399 }
400
401 jsonData, err := json.Marshal(reqBody)
402 if err != nil {
403 return 0, fmt.Errorf("marshal request: %w", err)
404 }
405
406 req, err := http.NewRequest("POST", syntheticAPI, bytes.NewBuffer(jsonData))
407 if err != nil {
408 return 0, fmt.Errorf("create request: %w", err)
409 }
410 req.Header.Set("Authorization", "Bearer "+apiKey)
411 req.Header.Set("Content-Type", "application/json")
412 req.Header.Set("anthropic-version", "2023-06-01")
413
414 resp, err := httpClient.Do(req)
415 if err != nil {
416 return 0, fmt.Errorf("HTTP request: %w", err)
417 }
418 defer resp.Body.Close()
419
420 if resp.StatusCode < 200 || resp.StatusCode >= 300 {
421 body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
422 return 0, fmt.Errorf("API status %d: %s", resp.StatusCode, string(body))
423 }
424
425 var result InputTokens
426 if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
427 return 0, fmt.Errorf("decode response: %w", err)
428 }
429
430 return result.InputTokens, nil
431}
432
433type InputTokens struct {
434 InputTokens int `json:"input_tokens"`
435}
436
437// updateStats summarises what updateReadme did.
438type updateStats struct {
439 Maintained int // skills on disk whose section was (re)written
440 Errored int // skills with one or more errors
441 Missing int // sections in README with no matching skill on disk
442}
443
444// parsedSection is one ### skill-name block from the existing README. The
445// script rewrites the heading and bullets of every maintained section but
446// preserves the paragraph verbatim. For sections whose skill has disappeared
447// from disk (orphans), raw is emitted unchanged.
448type parsedSection struct {
449 name string
450 paragraph []string // prose between heading and first bullet (preserved)
451 raw []string // full original section text (used for orphan passthrough)
452}
453
454// updateReadme rewrites the ## Available Skills region of the README: it
455// regenerates each section's heading and bullet list, preserving the
456// author-written paragraph; inserts a TODO scaffold for skills present on disk
457// but not in the README; and leaves sections whose skill is missing from disk
458// untouched (with a stderr warning). Everything outside the region is passed
459// through byte-for-byte.
460func updateReadme(path string, skills []SkillInfo) (updateStats, error) {
461 var stats updateStats
462
463 data, err := os.ReadFile(path)
464 if err != nil {
465 return stats, fmt.Errorf("cannot read %s: %w", path, err)
466 }
467 lines := strings.Split(string(data), "\n")
468
469 headingIdx := -1
470 for i, l := range lines {
471 if strings.TrimSpace(l) == skillsHeading {
472 headingIdx = i
473 break
474 }
475 }
476 if headingIdx == -1 {
477 return stats, fmt.Errorf("could not find %q in %s", skillsHeading, path)
478 }
479
480 regionEnd := len(lines)
481 for i := headingIdx + 1; i < len(lines); i++ {
482 if strings.HasPrefix(strings.TrimSpace(lines[i]), "## ") {
483 regionEnd = i
484 break
485 }
486 }
487
488 // Leading = lines between the heading and the first ### section.
489 firstSectionIdx := regionEnd
490 for i := headingIdx + 1; i < regionEnd; i++ {
491 if strings.HasPrefix(strings.TrimSpace(lines[i]), "### ") {
492 firstSectionIdx = i
493 break
494 }
495 }
496 leading := lines[headingIdx+1 : firstSectionIdx]
497
498 // Parse sections and whatever follows them (trailing).
499 var existing []parsedSection
500 i := firstSectionIdx
501 for i < regionEnd {
502 if !strings.HasPrefix(strings.TrimSpace(lines[i]), "### ") {
503 break // non-section content ends the section run; rest is trailing
504 }
505 name := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(lines[i]), "### "))
506
507 bodyStart := i + 1
508 bodyEnd := regionEnd
509 for j := bodyStart; j < regionEnd; j++ {
510 if strings.HasPrefix(strings.TrimSpace(lines[j]), "### ") {
511 bodyEnd = j
512 break
513 }
514 }
515 raw := lines[i:bodyEnd]
516 body := lines[bodyStart:bodyEnd]
517
518 bulletStart := -1
519 for k, bl := range body {
520 if strings.HasPrefix(strings.TrimSpace(bl), "- ") {
521 bulletStart = k
522 break
523 }
524 }
525 var paragraph []string
526 if bulletStart == -1 {
527 paragraph = body
528 } else {
529 paragraph = body[:bulletStart]
530 }
531 paragraph = trimLeadingBlanks(trimTrailingBlanks(paragraph))
532
533 existing = append(existing, parsedSection{name: name, paragraph: paragraph, raw: raw})
534 i = bodyEnd
535 }
536 trailing := lines[i:regionEnd]
537
538 existingByPos := existing
539 existingByName := make(map[string]parsedSection, len(existing))
540 for _, e := range existing {
541 existingByName[e.name] = e
542 }
543
544 desired := make(map[string]bool, len(skills))
545 for _, s := range skills {
546 desired[s.Dir] = true
547 if len(s.Errors) > 0 {
548 stats.Errored++
549 }
550 }
551 stats.Maintained = len(skills)
552 for _, e := range existingByPos {
553 if !desired[e.name] {
554 stats.Missing++
555 }
556 }
557
558 // Build the rebuilt region.
559 var regionLines []string
560 lead := trimTrailingBlanks(leading)
561 regionLines = append(regionLines, lead...)
562
563 for _, s := range skills {
564 paragraph, ok := existingByName[s.Dir]
565 if ok && len(paragraph.paragraph) > 0 {
566 regionLines = appendBlankSep(regionLines)
567 regionLines = append(regionLines, renderSection(s, paragraph.paragraph)...)
568 } else {
569 // New skill: scaffold with a TODO placeholder.
570 if !ok {
571 fmt.Fprintf(os.Stderr, "Warning: inserting new section for %s (no description yet)\n", s.Dir)
572 }
573 regionLines = appendBlankSep(regionLines)
574 regionLines = append(regionLines, renderSection(s, []string{todoParagraph})...)
575 }
576 }
577
578 // Orphans: sections in the README whose skill is gone from disk. Emit
579 // their raw text unchanged and warn.
580 for _, e := range existingByPos {
581 if desired[e.name] {
582 continue
583 }
584 fmt.Fprintf(os.Stderr, "Warning: section %q is in README but missing from disk; leaving untouched\n", e.name)
585 regionLines = appendBlankSep(regionLines)
586 regionLines = append(regionLines, e.raw...)
587 }
588
589 trail := trimLeadingBlanks(trailing)
590 if len(trail) > 0 {
591 regionLines = appendBlankSep(regionLines)
592 regionLines = append(regionLines, trail...)
593 }
594
595 // Ensure a blank line separates the region from the following ## heading
596 // (if any), matching markdown convention.
597 if len(regionLines) > 0 && regionLines[len(regionLines)-1] != "" && regionEnd < len(lines) {
598 regionLines = append(regionLines, "")
599 }
600
601 var newLines []string
602 newLines = append(newLines, lines[:headingIdx]...)
603 newLines = append(newLines, lines[headingIdx]) // "## Available Skills"
604 newLines = append(newLines, regionLines...)
605 newLines = append(newLines, lines[regionEnd:]...)
606
607 tmp := path + ".tmp"
608 if err := os.WriteFile(tmp, []byte(strings.Join(newLines, "\n")), 0o644); err != nil {
609 return stats, fmt.Errorf("write temp: %w", err)
610 }
611 if err := os.Rename(tmp, path); err != nil {
612 return stats, fmt.Errorf("rename: %w", err)
613 }
614 return stats, nil
615}
616
617// renderSection produces the deterministic heading + (preserved) paragraph +
618// bullets + optional warning block. When the skill has errors, all three
619// token bullets show 0 and a blockquote warning is appended.
620func renderSection(s SkillInfo, paragraph []string) []string {
621 var out []string
622 out = append(out, "### "+s.Dir, "")
623 out = append(out, paragraph...)
624 out = append(out, "")
625
626 nameDesc, bodyTk, totalTk := 0, 0, 0
627 if len(s.Errors) == 0 {
628 nameDesc = s.Tokens.Name + s.Tokens.Description
629 bodyTk = s.Tokens.Body
630 totalTk = s.Tokens.Total
631 }
632 out = append(out,
633 fmt.Sprintf("- Install with: `make %s`", s.Dir),
634 fmt.Sprintf("- Source: [skills/%s/](skills/%s/)", s.Dir, s.Dir),
635 fmt.Sprintf("- Name/desc: %d tok", nameDesc),
636 fmt.Sprintf("- SKILL.md: %d tok", bodyTk),
637 fmt.Sprintf("- Full skill: %d tok (all `.md` files in the folder)", totalTk),
638 )
639
640 if len(s.Errors) > 0 {
641 out = append(out, "", "> ⚠️ **Errors:**")
642 for _, e := range s.Errors {
643 out = append(out, "> - "+e)
644 }
645 }
646 return out
647}
648
649func appendBlankSep(lines []string) []string {
650 if len(lines) == 0 {
651 return lines
652 }
653 if lines[len(lines)-1] != "" {
654 return append(lines, "")
655 }
656 return lines
657}
658
659func trimTrailingBlanks(lines []string) []string {
660 end := len(lines)
661 for end > 0 && strings.TrimSpace(lines[end-1]) == "" {
662 end--
663 }
664 return lines[:end]
665}
666
667func trimLeadingBlanks(lines []string) []string {
668 start := 0
669 for start < len(lines) && strings.TrimSpace(lines[start]) == "" {
670 start++
671 }
672 return lines[start:]
673}