skill-stats.go

  1//usr/bin/env go run "$0" "$@"; exit
  2package main
  3
  4import (
  5	"bytes"
  6	"encoding/json"
  7	"flag"
  8	"fmt"
  9	"io"
 10	"net/http"
 11	"os"
 12	"path/filepath"
 13	"regexp"
 14	"sort"
 15	"strings"
 16	"sync"
 17	"time"
 18)
 19
 20const (
 21	syntheticAPI  = "https://api.synthetic.new/anthropic/v1/messages/count_tokens"
 22	model         = "syn:large:text"
 23	workerCount   = 5 // Number of parallel API workers
 24	readmePath    = "README.md"
 25	skillsHeading = "## Available skills"
 26	todoParagraph = "TODO: describe me"
 27	maxBodyLines  = 500
 28)
 29
 30var httpClient = &http.Client{
 31	Timeout: 30 * time.Second,
 32}
 33
 34type Frontmatter struct {
 35	Name        string
 36	Description string
 37}
 38
 39// TokenCount breaks down a skill's token usage. Extra is the sum of every
 40// .md file in the skill folder other than SKILL.md itself.
 41type TokenCount struct {
 42	Name        int
 43	Description int
 44	Body        int
 45	Extra       int
 46	Total       int
 47}
 48
 49type SkillInfo struct {
 50	Dir         string
 51	Frontmatter Frontmatter
 52	BodyLines   int
 53	Tokens      TokenCount
 54	Errors      []string
 55}
 56
 57type TokenJob struct {
 58	ID   string
 59	Text string
 60}
 61
 62type TokenResult struct {
 63	ID    string
 64	Count int
 65	Err   error
 66}
 67
 68func main() {
 69	workers := flag.Int("workers", workerCount, "Number of parallel API workers")
 70	flag.Parse()
 71
 72	apiKey := os.Getenv("SYNTHETIC_API_KEY")
 73	if apiKey == "" {
 74		fmt.Fprintln(os.Stderr, "Error: SYNTHETIC_API_KEY environment variable not set")
 75		os.Exit(1)
 76	}
 77
 78	counter := newTokenCounter(apiKey, *workers)
 79	defer counter.Close()
 80
 81	skills, err := analyzeSkills(counter)
 82	if err != nil {
 83		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 84		os.Exit(1)
 85	}
 86
 87	sort.Slice(skills, func(i, j int) bool {
 88		return skills[i].Dir < skills[j].Dir
 89	})
 90
 91	stats, err := updateReadme(readmePath, skills)
 92	if err != nil {
 93		fmt.Fprintf(os.Stderr, "Error updating README: %v\n", err)
 94		os.Exit(1)
 95	}
 96
 97	fmt.Fprintf(os.Stderr, "README updated: %d skills maintained, %d errored, %d present in README but missing on disk\n",
 98		stats.Maintained, stats.Errored, stats.Missing)
 99}
100
101type TokenCounter struct {
102	apiKey  string
103	jobs    chan TokenJob
104	results chan TokenResult
105	wg      sync.WaitGroup
106}
107
108func newTokenCounter(apiKey string, workers int) *TokenCounter {
109	tc := &TokenCounter{
110		apiKey:  apiKey,
111		jobs:    make(chan TokenJob, 100),
112		results: make(chan TokenResult, 100),
113	}
114	tc.wg.Add(workers)
115	for i := 0; i < workers; i++ {
116		go tc.worker()
117	}
118	go func() {
119		tc.wg.Wait()
120		close(tc.results)
121	}()
122	return tc
123}
124
125func (tc *TokenCounter) worker() {
126	defer tc.wg.Done()
127	for job := range tc.jobs {
128		count, err := countTokensAPI(tc.apiKey, job.Text)
129		tc.results <- TokenResult{ID: job.ID, Count: count, Err: err}
130	}
131}
132
133func (tc *TokenCounter) Count(id, text string) {
134	tc.jobs <- TokenJob{ID: id, Text: text}
135}
136
137func (tc *TokenCounter) GetResult() TokenResult {
138	result, ok := <-tc.results
139	if !ok {
140		return TokenResult{Err: fmt.Errorf("results channel closed")}
141	}
142	return result
143}
144
145func (tc *TokenCounter) TryGetResult() (TokenResult, bool) {
146	select {
147	case result, ok := <-tc.results:
148		if !ok {
149			return TokenResult{}, false
150		}
151		return result, true
152	default:
153		return TokenResult{}, false
154	}
155}
156
157func (tc *TokenCounter) Close() {
158	close(tc.jobs)
159}
160
161func analyzeSkills(counter *TokenCounter) ([]SkillInfo, error) {
162	skillsDir := "skills"
163	entries, err := os.ReadDir(skillsDir)
164	if err != nil {
165		return nil, fmt.Errorf("cannot read skills directory: %w", err)
166	}
167
168	var skills []SkillInfo
169	for _, entry := range entries {
170		if !entry.IsDir() {
171			continue
172		}
173
174		skillPath := filepath.Join(skillsDir, entry.Name())
175		skill, err := analyzeSkill(skillPath, counter)
176		if err != nil {
177			fmt.Fprintf(os.Stderr, "Warning: error analyzing %s: %v\n", entry.Name(), err)
178			continue
179		}
180		skills = append(skills, skill)
181	}
182
183	return skills, nil
184}
185
186func analyzeSkill(path string, counter *TokenCounter) (SkillInfo, error) {
187	skill := SkillInfo{Dir: filepath.Base(path)}
188
189	// Read SKILL.md
190	skillMdPath := filepath.Join(path, "SKILL.md")
191	content, err := os.ReadFile(skillMdPath)
192	if err != nil {
193		skill.Errors = append(skill.Errors, fmt.Sprintf("Cannot read SKILL.md: %v", err))
194		return skill, nil
195	}
196
197	// Parse frontmatter and body
198	fm, body, err := parseFrontmatter(string(content))
199	if err != nil {
200		skill.Errors = append(skill.Errors, fmt.Sprintf("Cannot parse frontmatter: %v", err))
201		return skill, nil
202	}
203	skill.Frontmatter = fm
204	trimmedBody := strings.TrimSpace(body)
205	if trimmedBody == "" {
206		skill.BodyLines = 0
207	} else {
208		skill.BodyLines = len(strings.Split(trimmedBody, "\n"))
209	}
210
211	// Validate
212	skill.Errors = append(skill.Errors, validateSkill(skill)...)
213
214	fmt.Fprintf(os.Stderr, "Analyzing %s...\n", skill.Dir)
215
216	// Collect token-counting jobs. SKILL.md is counted as its frontmatter
217	// name+description fields plus the body; every other .md file in the
218	// folder is counted wholesale.
219	var jobs []TokenJob
220	jobs = append(jobs, TokenJob{ID: fmt.Sprintf("%s:name", skill.Dir), Text: fm.Name})
221	jobs = append(jobs, TokenJob{ID: fmt.Sprintf("%s:description", skill.Dir), Text: fm.Description})
222	jobs = append(jobs, TokenJob{ID: fmt.Sprintf("%s:body", skill.Dir), Text: body})
223
224	err = filepath.WalkDir(path, func(p string, d os.DirEntry, walkErr error) error {
225		if walkErr != nil {
226			skill.Errors = append(skill.Errors, fmt.Sprintf("Cannot walk %s: %v", p, walkErr))
227			return nil
228		}
229		if d.IsDir() {
230			return nil
231		}
232		if !strings.HasSuffix(p, ".md") {
233			return nil
234		}
235		// SKILL.md is already covered by the name/description/body jobs.
236		if p == skillMdPath {
237			return nil
238		}
239		fileContent, err := os.ReadFile(p)
240		if err != nil {
241			skill.Errors = append(skill.Errors, fmt.Sprintf("Cannot read %s: %v", p, err))
242			return nil
243		}
244		relPath, _ := filepath.Rel(path, p)
245		jobs = append(jobs, TokenJob{
246			ID:   fmt.Sprintf("%s:file:%s", skill.Dir, relPath),
247			Text: string(fileContent),
248		})
249		return nil
250	})
251	if err != nil {
252		skill.Errors = append(skill.Errors, fmt.Sprintf("Walk error: %v", err))
253	}
254
255	// Interleave enqueue and drain to avoid backpressure on the buffered channels.
256	processResult := func(result TokenResult) {
257		if result.Err != nil {
258			skill.Errors = append(skill.Errors, fmt.Sprintf("Token count failed for %s: %v", result.ID, result.Err))
259			return
260		}
261		parts := strings.SplitN(result.ID, ":", 3)
262		if len(parts) < 2 {
263			return
264		}
265		switch parts[1] {
266		case "name":
267			skill.Tokens.Name = result.Count
268		case "description":
269			skill.Tokens.Description = result.Count
270		case "body":
271			skill.Tokens.Body = result.Count
272		case "file":
273			skill.Tokens.Extra += result.Count
274		}
275	}
276
277	outstanding := 0
278	for _, job := range jobs {
279		counter.Count(job.ID, job.Text)
280		outstanding++
281		for {
282			if result, ok := counter.TryGetResult(); ok {
283				processResult(result)
284				outstanding--
285			} else {
286				break
287			}
288		}
289	}
290	for outstanding > 0 {
291		result := counter.GetResult()
292		processResult(result)
293		outstanding--
294	}
295
296	skill.Tokens.Total = skill.Tokens.Name + skill.Tokens.Description + skill.Tokens.Body + skill.Tokens.Extra
297
298	return skill, nil
299}
300
301func parseFrontmatter(content string) (Frontmatter, string, error) {
302	lines := strings.Split(content, "\n")
303	if len(lines) < 3 || lines[0] != "---" {
304		return Frontmatter{}, "", fmt.Errorf("missing frontmatter")
305	}
306
307	var fm Frontmatter
308	var endIdx int
309	var inDescription bool
310	var descriptionLines []string
311
312	for i := 1; i < len(lines); i++ {
313		if lines[i] == "---" {
314			endIdx = i
315			break
316		}
317
318		line := lines[i]
319
320		// Parse name
321		if strings.HasPrefix(line, "name:") {
322			fm.Name = strings.TrimSpace(strings.TrimPrefix(line, "name:"))
323			continue
324		}
325
326		// Parse description (might be multi-line)
327		if strings.HasPrefix(line, "description:") {
328			descPart := strings.TrimSpace(strings.TrimPrefix(line, "description:"))
329			if descPart != "" {
330				descriptionLines = append(descriptionLines, descPart)
331			}
332			inDescription = true
333			continue
334		}
335
336		// Continue multi-line description
337		if inDescription && strings.HasPrefix(line, "  ") {
338			descriptionLines = append(descriptionLines, strings.TrimSpace(line))
339			continue
340		}
341
342		// End of description
343		if inDescription && !strings.HasPrefix(line, "  ") {
344			inDescription = false
345		}
346	}
347
348	fm.Description = strings.Join(descriptionLines, " ")
349
350	if endIdx == 0 {
351		return Frontmatter{}, "", fmt.Errorf("unclosed frontmatter")
352	}
353
354	body := strings.Join(lines[endIdx+1:], "\n")
355	return fm, body, nil
356}
357
358func validateSkill(skill SkillInfo) []string {
359	var errors []string
360
361	// Validate name
362	if len(skill.Frontmatter.Name) < 1 || len(skill.Frontmatter.Name) > 64 {
363		errors = append(errors, "name must be 1-64 characters")
364	}
365
366	namePattern := regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*$`)
367	if !namePattern.MatchString(skill.Frontmatter.Name) {
368		errors = append(errors, "name must be lowercase letters, numbers, hyphens only; no leading/trailing/consecutive hyphens")
369	}
370
371	if skill.Frontmatter.Name != skill.Dir {
372		errors = append(errors, fmt.Sprintf("name '%s' doesn't match directory '%s'", skill.Frontmatter.Name, skill.Dir))
373	}
374
375	// Validate description
376	if len(skill.Frontmatter.Description) < 1 {
377		errors = append(errors, "description is empty")
378	} else if len(skill.Frontmatter.Description) > 1024 {
379		errors = append(errors, fmt.Sprintf("description is %d characters (max 1024)", len(skill.Frontmatter.Description)))
380	}
381
382	// Check body line count
383	if skill.BodyLines > maxBodyLines {
384		errors = append(errors, fmt.Sprintf("body has %d lines (recommended: < %d)", skill.BodyLines, maxBodyLines))
385	}
386
387	return errors
388}
389
390func countTokensAPI(apiKey, text string) (int, error) {
391	reqBody := map[string]interface{}{
392		"model": model,
393		"messages": []map[string]string{
394			{
395				"role":    "user",
396				"content": text,
397			},
398		},
399	}
400
401	jsonData, err := json.Marshal(reqBody)
402	if err != nil {
403		return 0, fmt.Errorf("marshal request: %w", err)
404	}
405
406	req, err := http.NewRequest("POST", syntheticAPI, bytes.NewBuffer(jsonData))
407	if err != nil {
408		return 0, fmt.Errorf("create request: %w", err)
409	}
410	req.Header.Set("Authorization", "Bearer "+apiKey)
411	req.Header.Set("Content-Type", "application/json")
412	req.Header.Set("anthropic-version", "2023-06-01")
413
414	resp, err := httpClient.Do(req)
415	if err != nil {
416		return 0, fmt.Errorf("HTTP request: %w", err)
417	}
418	defer resp.Body.Close()
419
420	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
421		body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
422		return 0, fmt.Errorf("API status %d: %s", resp.StatusCode, string(body))
423	}
424
425	var result InputTokens
426	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
427		return 0, fmt.Errorf("decode response: %w", err)
428	}
429
430	return result.InputTokens, nil
431}
432
433type InputTokens struct {
434	InputTokens int `json:"input_tokens"`
435}
436
437// updateStats summarises what updateReadme did.
438type updateStats struct {
439	Maintained int // skills on disk whose section was (re)written
440	Errored    int // skills with one or more errors
441	Missing    int // sections in README with no matching skill on disk
442}
443
444// parsedSection is one ### skill-name block from the existing README. The
445// script rewrites the heading and bullets of every maintained section but
446// preserves the paragraph verbatim. For sections whose skill has disappeared
447// from disk (orphans), raw is emitted unchanged.
448type parsedSection struct {
449	name      string
450	paragraph []string // prose between heading and first bullet (preserved)
451	raw       []string // full original section text (used for orphan passthrough)
452}
453
454// updateReadme rewrites the ## Available Skills region of the README: it
455// regenerates each section's heading and bullet list, preserving the
456// author-written paragraph; inserts a TODO scaffold for skills present on disk
457// but not in the README; and leaves sections whose skill is missing from disk
458// untouched (with a stderr warning). Everything outside the region is passed
459// through byte-for-byte.
460func updateReadme(path string, skills []SkillInfo) (updateStats, error) {
461	var stats updateStats
462
463	data, err := os.ReadFile(path)
464	if err != nil {
465		return stats, fmt.Errorf("cannot read %s: %w", path, err)
466	}
467	lines := strings.Split(string(data), "\n")
468
469	headingIdx := -1
470	for i, l := range lines {
471		if strings.TrimSpace(l) == skillsHeading {
472			headingIdx = i
473			break
474		}
475	}
476	if headingIdx == -1 {
477		return stats, fmt.Errorf("could not find %q in %s", skillsHeading, path)
478	}
479
480	regionEnd := len(lines)
481	for i := headingIdx + 1; i < len(lines); i++ {
482		if strings.HasPrefix(strings.TrimSpace(lines[i]), "## ") {
483			regionEnd = i
484			break
485		}
486	}
487
488	// Leading = lines between the heading and the first ### section.
489	firstSectionIdx := regionEnd
490	for i := headingIdx + 1; i < regionEnd; i++ {
491		if strings.HasPrefix(strings.TrimSpace(lines[i]), "### ") {
492			firstSectionIdx = i
493			break
494		}
495	}
496	leading := lines[headingIdx+1 : firstSectionIdx]
497
498	// Parse sections and whatever follows them (trailing).
499	var existing []parsedSection
500	i := firstSectionIdx
501	for i < regionEnd {
502		if !strings.HasPrefix(strings.TrimSpace(lines[i]), "### ") {
503			break // non-section content ends the section run; rest is trailing
504		}
505		name := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(lines[i]), "### "))
506
507		bodyStart := i + 1
508		bodyEnd := regionEnd
509		for j := bodyStart; j < regionEnd; j++ {
510			if strings.HasPrefix(strings.TrimSpace(lines[j]), "### ") {
511				bodyEnd = j
512				break
513			}
514		}
515		raw := lines[i:bodyEnd]
516		body := lines[bodyStart:bodyEnd]
517
518		bulletStart := -1
519		for k, bl := range body {
520			if strings.HasPrefix(strings.TrimSpace(bl), "- ") {
521				bulletStart = k
522				break
523			}
524		}
525		var paragraph []string
526		if bulletStart == -1 {
527			paragraph = body
528		} else {
529			paragraph = body[:bulletStart]
530		}
531		paragraph = trimLeadingBlanks(trimTrailingBlanks(paragraph))
532
533		existing = append(existing, parsedSection{name: name, paragraph: paragraph, raw: raw})
534		i = bodyEnd
535	}
536	trailing := lines[i:regionEnd]
537
538	existingByPos := existing
539	existingByName := make(map[string]parsedSection, len(existing))
540	for _, e := range existing {
541		existingByName[e.name] = e
542	}
543
544	desired := make(map[string]bool, len(skills))
545	for _, s := range skills {
546		desired[s.Dir] = true
547		if len(s.Errors) > 0 {
548			stats.Errored++
549		}
550	}
551	stats.Maintained = len(skills)
552	for _, e := range existingByPos {
553		if !desired[e.name] {
554			stats.Missing++
555		}
556	}
557
558	// Build the rebuilt region.
559	var regionLines []string
560	lead := trimTrailingBlanks(leading)
561	regionLines = append(regionLines, lead...)
562
563	for _, s := range skills {
564		paragraph, ok := existingByName[s.Dir]
565		if ok && len(paragraph.paragraph) > 0 {
566			regionLines = appendBlankSep(regionLines)
567			regionLines = append(regionLines, renderSection(s, paragraph.paragraph)...)
568		} else {
569			// New skill: scaffold with a TODO placeholder.
570			if !ok {
571				fmt.Fprintf(os.Stderr, "Warning: inserting new section for %s (no description yet)\n", s.Dir)
572			}
573			regionLines = appendBlankSep(regionLines)
574			regionLines = append(regionLines, renderSection(s, []string{todoParagraph})...)
575		}
576	}
577
578	// Orphans: sections in the README whose skill is gone from disk. Emit
579	// their raw text unchanged and warn.
580	for _, e := range existingByPos {
581		if desired[e.name] {
582			continue
583		}
584		fmt.Fprintf(os.Stderr, "Warning: section %q is in README but missing from disk; leaving untouched\n", e.name)
585		regionLines = appendBlankSep(regionLines)
586		regionLines = append(regionLines, e.raw...)
587	}
588
589	trail := trimLeadingBlanks(trailing)
590	if len(trail) > 0 {
591		regionLines = appendBlankSep(regionLines)
592		regionLines = append(regionLines, trail...)
593	}
594
595	// Ensure a blank line separates the region from the following ## heading
596	// (if any), matching markdown convention.
597	if len(regionLines) > 0 && regionLines[len(regionLines)-1] != "" && regionEnd < len(lines) {
598		regionLines = append(regionLines, "")
599	}
600
601	var newLines []string
602	newLines = append(newLines, lines[:headingIdx]...)
603	newLines = append(newLines, lines[headingIdx]) // "## Available Skills"
604	newLines = append(newLines, regionLines...)
605	newLines = append(newLines, lines[regionEnd:]...)
606
607	tmp := path + ".tmp"
608	if err := os.WriteFile(tmp, []byte(strings.Join(newLines, "\n")), 0o644); err != nil {
609		return stats, fmt.Errorf("write temp: %w", err)
610	}
611	if err := os.Rename(tmp, path); err != nil {
612		return stats, fmt.Errorf("rename: %w", err)
613	}
614	return stats, nil
615}
616
617// renderSection produces the deterministic heading + (preserved) paragraph +
618// bullets + optional warning block. When the skill has errors, all three
619// token bullets show 0 and a blockquote warning is appended.
620func renderSection(s SkillInfo, paragraph []string) []string {
621	var out []string
622	out = append(out, "### "+s.Dir, "")
623	out = append(out, paragraph...)
624	out = append(out, "")
625
626	nameDesc, bodyTk, totalTk := 0, 0, 0
627	if len(s.Errors) == 0 {
628		nameDesc = s.Tokens.Name + s.Tokens.Description
629		bodyTk = s.Tokens.Body
630		totalTk = s.Tokens.Total
631	}
632	out = append(out,
633		fmt.Sprintf("- Install with: `make %s`", s.Dir),
634		fmt.Sprintf("- Source: [skills/%s/](skills/%s/)", s.Dir, s.Dir),
635		fmt.Sprintf("- Name/desc: %d tok", nameDesc),
636		fmt.Sprintf("- SKILL.md: %d tok", bodyTk),
637		fmt.Sprintf("- Full skill: %d tok (all `.md` files in the folder)", totalTk),
638	)
639
640	if len(s.Errors) > 0 {
641		out = append(out, "", "> ⚠️ **Errors:**")
642		for _, e := range s.Errors {
643			out = append(out, "> - "+e)
644		}
645	}
646	return out
647}
648
649func appendBlankSep(lines []string) []string {
650	if len(lines) == 0 {
651		return lines
652	}
653	if lines[len(lines)-1] != "" {
654		return append(lines, "")
655	}
656	return lines
657}
658
659func trimTrailingBlanks(lines []string) []string {
660	end := len(lines)
661	for end > 0 && strings.TrimSpace(lines[end-1]) == "" {
662		end--
663	}
664	return lines[:end]
665}
666
667func trimLeadingBlanks(lines []string) []string {
668	start := 0
669	for start < len(lines) && strings.TrimSpace(lines[start]) == "" {
670		start++
671	}
672	return lines[start:]
673}