Add skills
This commit is contained in:
1014
.config/opencode/skills/.skill-builder.disabled/SKILL.md
Normal file
1014
.config/opencode/skills/.skill-builder.disabled/SKILL.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,615 @@
|
||||
# Test Case Templates
|
||||
|
||||
Copy-paste templates for creating test cases based on skill type.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [File Transform Skills](#file-transform-skills)
|
||||
2. [Code Generation Skills](#code-generation-skills)
|
||||
3. [Workflow Skills](#workflow-skills)
|
||||
4. [Tool Integration Skills](#tool-integration-skills)
|
||||
5. [Documentation Skills](#documentation-skills)
|
||||
|
||||
---
|
||||
|
||||
## File Transform Skills
|
||||
|
||||
For skills that convert, parse, or reformat files.
|
||||
|
||||
### Example: CSV to JSON Converter
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "csv-to-json",
|
||||
"description": "Converts CSV files to JSON format with data type handling",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "Convert the CSV file at ./data/sales.csv to JSON and save it as ./output/sales.json. The CSV has headers: date, product, quantity, price. Make sure dates are in ISO 8601 format.",
|
||||
"expected_output": "A JSON file at ./output/sales.json containing an array of objects, each representing a row from the CSV with properly formatted dates.",
|
||||
"assertions": [
|
||||
"JSON file exists at ./output/sales.json",
|
||||
"File contains valid JSON array",
|
||||
"All dates are in ISO 8601 format (YYYY-MM-DD)",
|
||||
"Numeric fields (quantity, price) are numbers, not strings"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case-large-file",
|
||||
"type": "edge",
|
||||
"prompt": "Convert a CSV file with 50,000 rows at ./data/export.csv to JSON. The file contains some rows with missing values in the 'email' column and special characters (emojis) in the 'notes' column.",
|
||||
"expected_output": "JSON file is created successfully, handling missing values as null or empty strings, and preserving special characters correctly.",
|
||||
"assertions": [
|
||||
"Large file is processed without memory errors",
|
||||
"Missing email values are handled (null or empty string)",
|
||||
"Special characters and emojis are preserved in output",
|
||||
"All 50,000 rows are converted"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing-casual",
|
||||
"type": "variation",
|
||||
"prompt": "Hey, I've got this spreadsheet data/export.csv that I need to turn into JSON. Can you do that for me? Also, the dates are in MM/DD/YYYY format right now - can you make them proper ISO format?",
|
||||
"expected_output": "Same as common case: JSON file with ISO 8601 dates",
|
||||
"assertions": [
|
||||
"Skill triggers with casual language ('turn into', 'proper format')",
|
||||
"Implicit date formatting requirement is handled",
|
||||
"Output matches common case results"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Template: File Transform Skill
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "YOUR_SKILL_NAME",
|
||||
"description": "[Describe what file transformations this skill does]",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "[Realistic request with specific file paths and formats]",
|
||||
"expected_output": "[What the transformed file should contain]",
|
||||
"assertions": [
|
||||
"Output file exists at expected location",
|
||||
"File is valid [format: JSON, XML, etc.]",
|
||||
"Data is correctly transformed",
|
||||
"Specific requirements met (encoding, formatting, etc.)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case",
|
||||
"type": "edge",
|
||||
"prompt": "[Large files, special characters, missing data, malformed input]",
|
||||
"expected_output": "[Graceful handling or error with helpful message]",
|
||||
"assertions": [
|
||||
"Edge case is handled appropriately",
|
||||
"No data loss or corruption",
|
||||
"Error messages are helpful if transformation fails"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing",
|
||||
"type": "variation",
|
||||
"prompt": "[Same request with casual language, different wording]",
|
||||
"expected_output": "[Same as common case]",
|
||||
"assertions": [
|
||||
"Skill triggers with varied phrasing",
|
||||
"Output matches common case",
|
||||
"Implicit requirements are understood"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Code Generation Skills
|
||||
|
||||
For skills that generate code, scripts, or templates.
|
||||
|
||||
### Example: Python Script Generator
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "python-script-gen",
|
||||
"description": "Generates Python scripts for file operations and data processing",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "Create a Python script that recursively finds all .log files in /var/log, compresses them with gzip if they're older than 30 days, and moves them to /archive. Handle permission errors gracefully and log all actions to cleanup.log.",
|
||||
"expected_output": "A Python script that implements the log cleanup functionality with proper error handling, logging, and follows Python best practices.",
|
||||
"assertions": [
|
||||
"Script is syntactically valid Python",
|
||||
"Implements recursive file search",
|
||||
"Compresses files older than 30 days",
|
||||
"Handles permission errors gracefully",
|
||||
"Logs actions to cleanup.log"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case-empty-directory",
|
||||
"type": "edge",
|
||||
"prompt": "Create a Python script that processes all CSV files in ./data and generates a summary report. What should it do if the directory is empty or doesn't exist?",
|
||||
"expected_output": "Script handles empty/non-existent directories gracefully with informative error messages and doesn't crash.",
|
||||
"assertions": [
|
||||
"Script checks if directory exists before processing",
|
||||
"Handles empty directory case gracefully",
|
||||
"Provides informative error message",
|
||||
"Returns appropriate exit code"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing-brief",
|
||||
"type": "variation",
|
||||
"prompt": "Write me a python script to backup my photos. It should copy everything from ~/Pictures to ~/Backups/photos with today's date in the folder name. Skip duplicates if possible.",
|
||||
"expected_output": "Python backup script with timestamped folder and duplicate detection",
|
||||
"assertions": [
|
||||
"Skill works with brief, casual description",
|
||||
"Generates complete script without requiring clarification",
|
||||
"Handles date formatting for folder name",
|
||||
"Implements duplicate detection"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Template: Code Generation Skill
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "YOUR_SKILL_NAME",
|
||||
"description": "[Describe what code this skill generates]",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "[Detailed request with specific requirements and constraints]",
|
||||
"expected_output": "[Description of the generated code]",
|
||||
"assertions": [
|
||||
"Code is syntactically valid",
|
||||
"Implements all requested features",
|
||||
"Follows language best practices",
|
||||
"Includes error handling where appropriate",
|
||||
"Is well-structured and readable"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case",
|
||||
"type": "edge",
|
||||
"prompt": "[Ambiguous requirements, missing data, error conditions]",
|
||||
"expected_output": "[Code handles edge cases or asks for clarification]",
|
||||
"assertions": [
|
||||
"Edge cases are handled appropriately",
|
||||
"Error conditions are managed",
|
||||
"Code doesn't crash on unexpected input"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing",
|
||||
"type": "variation",
|
||||
"prompt": "[Same request with minimal details or casual language]",
|
||||
"expected_output": "[Same quality as common case]",
|
||||
"assertions": [
|
||||
"Skill fills in reasonable defaults",
|
||||
"Generates complete solution",
|
||||
"Output quality matches detailed request"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Workflow Skills
|
||||
|
||||
For skills that guide multi-step processes.
|
||||
|
||||
### Example: Release Workflow
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "release-workflow",
|
||||
"description": "Guides the complete software release process",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "I need to create a new release for my Node.js project. The repo is at ~/projects/myapp. We're currently at version 1.2.3 and this is a minor feature release (1.3.0). I need to update the version, create a changelog entry, commit, tag, and push to GitHub.",
|
||||
"expected_output": "Step-by-step guide covering: version bump in package.json, CHANGELOG.md update, commit creation, annotated tag, and push commands. Should provide copy-pasteable commands.",
|
||||
"assertions": [
|
||||
"All release steps are covered",
|
||||
"Commands are copy-pasteable",
|
||||
"Version numbers are consistent",
|
||||
"Validation steps are included",
|
||||
"Provides rollback guidance"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case-dirty-worktree",
|
||||
"type": "edge",
|
||||
"prompt": "Help me release version 2.0.0 of my project. By the way, I have some uncommitted changes in my working directory that I'm not sure about.",
|
||||
"expected_output": "Workflow detects dirty worktree, suggests stashing or committing changes before proceeding with release. Provides commands to handle the situation.",
|
||||
"assertions": [
|
||||
"Detects uncommitted changes",
|
||||
"Warns about dirty worktree",
|
||||
"Provides options: stash, commit, or abort",
|
||||
"Doesn't proceed without addressing the issue"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing-urgent",
|
||||
"type": "variation",
|
||||
"prompt": "Need to push out v2.1.0 ASAP. Hotfix for critical bug. What's the fastest way to get this released?",
|
||||
"expected_output": "Accelerated workflow prioritizing speed while maintaining essential safety checks",
|
||||
"assertions": [
|
||||
"Recognizes urgency from language ('ASAP', 'fastest')",
|
||||
"Still includes critical safety checks",
|
||||
"Prioritizes speed without skipping validation",
|
||||
"Provides streamlined command sequence"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Template: Workflow Skill
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "YOUR_SKILL_NAME",
|
||||
"description": "[Describe what workflow this skill guides]",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "[Standard workflow request with clear requirements]",
|
||||
"expected_output": "[Complete step-by-step guide with all necessary steps]",
|
||||
"assertions": [
|
||||
"All workflow steps are included",
|
||||
"Steps are in logical order",
|
||||
"Validation points are provided",
|
||||
"Commands are copy-pasteable where applicable",
|
||||
"Clear success criteria defined"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case",
|
||||
"type": "edge",
|
||||
"prompt": "[Workflow with complications, errors, or unusual state]",
|
||||
"expected_output": "[Workflow detects issues and provides guidance]",
|
||||
"assertions": [
|
||||
"Detects unusual states or errors",
|
||||
"Provides recovery options",
|
||||
"Doesn't proceed blindly",
|
||||
"Offers rollback or alternative paths"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing",
|
||||
"type": "variation",
|
||||
"prompt": "[Workflow request with urgency, casual language, or minimal details]",
|
||||
"expected_output": "[Same workflow adapted to context]",
|
||||
"assertions": [
|
||||
"Adapts to urgency level",
|
||||
"Works with minimal context",
|
||||
"Still provides complete guidance"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tool Integration Skills
|
||||
|
||||
For skills that wrap command-line tools.
|
||||
|
||||
### Example: Docker Helper
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "docker-helper",
|
||||
"description": "Streamlines Docker container and image management",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "I need to deploy my Node.js app using Docker. The Dockerfile is in ~/projects/myapp. Build an image tagged as myapp:v1.0, then run a container named 'myapp-prod' that maps port 3000 to the host. Make sure it restarts automatically if it crashes.",
|
||||
"expected_output": "Docker commands to build the image and run the container with specified configuration, including restart policy.",
|
||||
"assertions": [
|
||||
"Provides correct build command with tag",
|
||||
"Provides correct run command with port mapping",
|
||||
"Includes restart policy (--restart unless-stopped or always)",
|
||||
"Sets container name correctly",
|
||||
"Commands are copy-pasteable"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case-port-conflict",
|
||||
"type": "edge",
|
||||
"prompt": "Run my Docker container on port 3000, but I think something might already be using that port on my machine. How do I check and handle this?",
|
||||
"expected_output": "Commands to check port usage, offer solutions (kill process, use different port, or map to different host port), and proceed accordingly.",
|
||||
"assertions": [
|
||||
"Detects potential port conflict",
|
||||
"Provides command to check port usage",
|
||||
"Offers multiple solutions",
|
||||
"Explains trade-offs of each option"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing-cleanup",
|
||||
"type": "variation",
|
||||
"prompt": "Docker is taking up too much space. Clean up old stuff for me?",
|
||||
"expected_output": "Commands to clean up stopped containers, unused images, and build cache",
|
||||
"assertions": [
|
||||
"Understands implicit request from context",
|
||||
"Provides safe cleanup commands",
|
||||
"Warns about data loss where applicable",
|
||||
"Shows space savings after cleanup"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Template: Tool Integration Skill
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "YOUR_SKILL_NAME",
|
||||
"description": "[Describe what tool this skill wraps]",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "[Standard tool usage with specific options and requirements]",
|
||||
"expected_output": "[Correct command(s) with proper flags]",
|
||||
"assertions": [
|
||||
"Command syntax is correct",
|
||||
"All required flags are included",
|
||||
"Best practices are followed",
|
||||
"Commands are copy-pasteable",
|
||||
"Explains what each part does"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case",
|
||||
"type": "edge",
|
||||
"prompt": "[Tool usage with errors, conflicts, or unusual requirements]",
|
||||
"expected_output": "[Troubleshooting steps and solutions]",
|
||||
"assertions": [
|
||||
"Detects potential issues",
|
||||
"Provides diagnostic commands",
|
||||
"Offers multiple solutions",
|
||||
"Explains risks of each approach"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing",
|
||||
"type": "variation",
|
||||
"prompt": "[Casual request with vague requirements]",
|
||||
"expected_output": "[Tool commands with reasonable defaults]",
|
||||
"assertions": [
|
||||
"Fills in reasonable defaults",
|
||||
"Provides complete solution",
|
||||
"Explains assumptions made"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Documentation Skills
|
||||
|
||||
For skills that help create or review documentation.
|
||||
|
||||
### Example: README Generator
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "readme-generator",
|
||||
"description": "Creates comprehensive README files for projects",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "Create a README for my Python project. It's a CLI tool called 'file-organizer' that sorts files by type and date. The repo is at ~/projects/file-organizer. It supports Python 3.8+, uses click for CLI, and has features for: organizing by extension, organizing by date, dry-run mode, and config file support.",
|
||||
"expected_output": "Complete README with: title, description, installation, usage examples, features list, configuration, and contributing sections. Formatted in Markdown.",
|
||||
"assertions": [
|
||||
"README is well-structured with clear headings",
|
||||
"All requested sections are included",
|
||||
"Installation instructions are clear",
|
||||
"Usage examples show actual commands",
|
||||
"Features are listed comprehensively"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case-minimal-info",
|
||||
"type": "edge",
|
||||
"prompt": "Write a README for my project. It's called 'utils' and it does some stuff with files.",
|
||||
"expected_output": "README template with placeholder sections and prompts for missing information. Asks clarifying questions or provides generic placeholders.",
|
||||
"assertions": [
|
||||
"Creates template structure despite minimal info",
|
||||
"Uses placeholders for missing details",
|
||||
"Suggests what information to add",
|
||||
"Doesn't invent features or functionality"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing-casual",
|
||||
"type": "variation",
|
||||
"prompt": "Hey can you write a readme for my new js library? It's on npm as 'async-queue'. It helps manage async tasks with a queue so you don't overwhelm APIs. Pretty simple but useful.",
|
||||
"expected_output": "README with npm installation, basic usage example, and API overview",
|
||||
"assertions": [
|
||||
"Understands from package name and casual description",
|
||||
"Includes npm install instructions",
|
||||
"Provides JavaScript usage examples",
|
||||
"Explains the problem it solves"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Template: Documentation Skill
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "YOUR_SKILL_NAME",
|
||||
"description": "[Describe what documentation this skill helps with]",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "[Detailed request with project information and requirements]",
|
||||
"expected_output": "[Complete, well-structured documentation]",
|
||||
"assertions": [
|
||||
"Documentation is well-organized",
|
||||
"All required sections are included",
|
||||
"Examples are clear and relevant",
|
||||
"Formatting is consistent",
|
||||
"Language is clear and concise"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case",
|
||||
"type": "edge",
|
||||
"prompt": "[Vague request with minimal information]",
|
||||
"expected_output": "[Template with placeholders or clarifying questions]",
|
||||
"assertions": [
|
||||
"Creates structure despite limited info",
|
||||
"Uses appropriate placeholders",
|
||||
"Identifies missing information",
|
||||
"Doesn't invent false details"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing",
|
||||
"type": "variation",
|
||||
"prompt": "[Casual request with implied requirements]",
|
||||
"expected_output": "[Documentation meeting implicit needs]",
|
||||
"assertions": [
|
||||
"Understands implicit requirements",
|
||||
"Provides complete documentation",
|
||||
"Matches tone of request"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tips for Customizing Templates
|
||||
|
||||
### Making Tests Realistic
|
||||
|
||||
**Bad:**
|
||||
```
|
||||
"Convert CSV to JSON"
|
||||
```
|
||||
|
||||
**Good:**
|
||||
```
|
||||
"Convert the CSV file at ./data/sales.csv to JSON and save it as ./output/sales.json. The CSV has headers: date, product, quantity, price. Make sure dates are in ISO 8601 format."
|
||||
```
|
||||
|
||||
### Assertions Should Be Checkable
|
||||
|
||||
**Vague:**
|
||||
```json
|
||||
"assertions": [
|
||||
"Output looks good"
|
||||
]
|
||||
```
|
||||
|
||||
**Specific:**
|
||||
```json
|
||||
"assertions": [
|
||||
"JSON file exists at ./output/sales.json",
|
||||
"All dates are in ISO 8601 format",
|
||||
"Numeric fields are numbers, not strings"
|
||||
]
|
||||
```
|
||||
|
||||
### Edge Cases to Consider
|
||||
|
||||
1. **Large data** - Files with 10,000+ rows
|
||||
2. **Special characters** - Emojis, Unicode, unusual symbols
|
||||
3. **Missing data** - Empty cells, null values
|
||||
4. **Wrong format** - CSV with inconsistent columns
|
||||
5. **Permissions** - Read/write errors
|
||||
6. **Conflicts** - Port conflicts, file already exists
|
||||
7. **Empty input** - Zero rows, blank files
|
||||
8. **Unexpected state** - Dirty git worktree, missing dependencies
|
||||
|
||||
### Variation Ideas
|
||||
|
||||
1. **Casual language** - "Hey, can you...", "I need to..."
|
||||
2. **Abbreviated** - Minimal words, assumes context
|
||||
3. **Urgent** - "ASAP", "quickly", "fastest way"
|
||||
4. **Uncertain** - "I think", "maybe", "probably"
|
||||
5. **Brief** - Single sentence, minimal details
|
||||
6. **Verbose** - Extra context, backstory
|
||||
|
||||
---
|
||||
|
||||
## Validation Checklist
|
||||
|
||||
Before using your evals.json:
|
||||
|
||||
- [ ] Contains exactly 3 test cases (common, edge, variation)
|
||||
- [ ] Each test has unique id (1, 2, 3)
|
||||
- [ ] Each test has descriptive name
|
||||
- [ ] Prompts are realistic with specific details
|
||||
- [ ] Expected outputs are clear
|
||||
- [ ] Assertions are objectively checkable
|
||||
- [ ] JSON is valid (run through jq or json linter)
|
||||
- [ ] File is saved as evals/evals.json in skill directory
|
||||
|
||||
Run this to validate:
|
||||
```bash
|
||||
jq . evals/evals.json > /dev/null && echo "Valid JSON" || echo "Invalid JSON"
|
||||
```
|
||||
@@ -0,0 +1,607 @@
|
||||
# Grading Guide
|
||||
|
||||
Comprehensive guide for evaluating skill outputs and identifying improvements.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [The Evaluation Framework](#the-evaluation-framework)
|
||||
2. [Correctness](#correctness)
|
||||
3. [Completeness](#completeness)
|
||||
4. [Format](#format)
|
||||
5. [Triggering](#triggering)
|
||||
6. [Efficiency](#efficiency)
|
||||
7. [Identifying Patterns](#identifying-patterns)
|
||||
8. [Decision Framework](#decision-framework)
|
||||
9. [Common Issues and Solutions](#common-issues-and-solutions)
|
||||
10. [When to Stop Iterating](#when-to-stop-iterating)
|
||||
|
||||
---
|
||||
|
||||
## The Evaluation Framework
|
||||
|
||||
Evaluate skill outputs across five dimensions:
|
||||
|
||||
1. **Correctness** - Is the output accurate and correct?
|
||||
2. **Completeness** - Are all requirements met?
|
||||
3. **Format** - Does it follow expected structure?
|
||||
4. **Triggering** - Does it activate appropriately?
|
||||
5. **Efficiency** - Is it concise yet complete?
|
||||
|
||||
Each dimension has specific criteria to check. Use the grade-output.sh script for systematic evaluation.
|
||||
|
||||
---
|
||||
|
||||
## Correctness
|
||||
|
||||
### What to Check
|
||||
|
||||
**Output matches expected result:**
|
||||
- For file transforms: Output file contains correct data
|
||||
- For code generation: Code works as described
|
||||
- For workflows: All steps lead to desired outcome
|
||||
- For documentation: Information is accurate
|
||||
|
||||
**No factual errors:**
|
||||
- Commands use correct syntax
|
||||
- File paths are accurate
|
||||
- Versions are correct
|
||||
- Technical details are right
|
||||
|
||||
**Logic is sound:**
|
||||
- Reasoning makes sense
|
||||
- Steps follow logically
|
||||
- No contradictions
|
||||
- Edge cases considered
|
||||
|
||||
**Edge cases handled appropriately:**
|
||||
- Empty inputs don't crash
|
||||
- Large inputs work
|
||||
- Special characters preserved
|
||||
- Errors handled gracefully
|
||||
|
||||
### Examples
|
||||
|
||||
**Good (Correct):**
|
||||
```bash
|
||||
# Command actually works
|
||||
docker run -p 3000:3000 --name myapp myimage:v1.0
|
||||
```
|
||||
|
||||
**Bad (Incorrect):**
|
||||
```bash
|
||||
# Wrong flag syntax
|
||||
docker run --port 3000:3000 myimage # Should be -p or --publish
|
||||
```
|
||||
|
||||
**Good (Handles Edge Cases):**
|
||||
```python
|
||||
import os
|
||||
if os.path.exists(filename):
|
||||
process_file(filename)
|
||||
else:
|
||||
print(f"Error: {filename} not found")
|
||||
```
|
||||
|
||||
**Bad (No Edge Case Handling):**
|
||||
```python
|
||||
process_file(filename) # Will crash if file doesn't exist
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Completeness
|
||||
|
||||
### What to Check
|
||||
|
||||
**All requested tasks completed:**
|
||||
- Every requirement from prompt addressed
|
||||
- No steps forgotten
|
||||
- All files generated
|
||||
- All transformations applied
|
||||
|
||||
**No steps skipped:**
|
||||
- Validation steps included
|
||||
- Cleanup steps not forgotten
|
||||
- Confirmation steps present
|
||||
- Rollback guidance provided
|
||||
|
||||
**Appropriate level of detail:**
|
||||
- Not too brief (missing context)
|
||||
- Not too verbose (information overload)
|
||||
- Explains "why" not just "what"
|
||||
- Provides examples where helpful
|
||||
|
||||
**Relevant context included:**
|
||||
- Prerequisites mentioned
|
||||
- Dependencies noted
|
||||
- Error scenarios covered
|
||||
- Alternative approaches offered
|
||||
|
||||
### Examples
|
||||
|
||||
**Good (Complete):**
|
||||
```
|
||||
To release version 1.3.0:
|
||||
|
||||
1. Update version in package.json: "version": "1.3.0"
|
||||
2. Add entry to CHANGELOG.md with today's date
|
||||
3. Commit: git commit -am "chore(release): bump version to 1.3.0"
|
||||
4. Create tag: git tag -a v1.3.0 -m "Release 1.3.0"
|
||||
5. Push: git push origin main && git push origin v1.3.0
|
||||
6. Create GitHub release: gh release create v1.3.0 --notes-from-tag
|
||||
|
||||
Prerequisites:
|
||||
- All tests passing
|
||||
- CHANGELOG.md updated
|
||||
- You have push access to repository
|
||||
```
|
||||
|
||||
**Bad (Incomplete):**
|
||||
```
|
||||
To release:
|
||||
1. Update version
|
||||
2. Create tag
|
||||
3. Push
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Format
|
||||
|
||||
### What to Check
|
||||
|
||||
**Output follows specified format:**
|
||||
- If skill defines a template, output matches it
|
||||
- Markdown formatting is correct
|
||||
- Code blocks use proper syntax highlighting
|
||||
- Tables are properly formatted
|
||||
|
||||
**Consistent with examples in skill:**
|
||||
- Format matches examples in SKILL.md
|
||||
- Style is consistent
|
||||
- Terminology is consistent
|
||||
- Structure follows patterns
|
||||
|
||||
**Easy to read and understand:**
|
||||
- Clear headings and sections
|
||||
- Good use of whitespace
|
||||
- Logical organization
|
||||
- No wall of text
|
||||
|
||||
### Examples
|
||||
|
||||
**Good (Well-Formatted):**
|
||||
```markdown
|
||||
## Deployment Steps
|
||||
|
||||
### 1. Build Docker Image
|
||||
|
||||
```bash
|
||||
docker build -t myapp:v1.0 .
|
||||
```
|
||||
|
||||
### 2. Run Container
|
||||
|
||||
```bash
|
||||
docker run -d -p 3000:3000 --name myapp myapp:v1.0
|
||||
```
|
||||
|
||||
### 3. Verify Deployment
|
||||
|
||||
```bash
|
||||
curl http://localhost:3000/health
|
||||
```
|
||||
```
|
||||
|
||||
**Bad (Poor Formatting):**
|
||||
```
|
||||
step 1: build the image with docker build -t myapp:v1.0 . then step 2 run the container with docker run -d -p 3000:3000 --name myapp myapp:v1.0 and step 3 verify with curl
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Triggering
|
||||
|
||||
### What to Check
|
||||
|
||||
**Skill activates when appropriate:**
|
||||
- Relevant triggers work
|
||||
- Similar phrasings activate it
|
||||
- Context clues are recognized
|
||||
- Doesn't require exact keywords
|
||||
|
||||
**Doesn't activate when inappropriate:**
|
||||
- Adjacent domains don't trigger it
|
||||
- Ambiguous queries don't wrongly trigger
|
||||
- Different tools aren't confused
|
||||
- Scope is respected
|
||||
|
||||
### Testing Triggering
|
||||
|
||||
Test with 8 queries (4 should-trigger, 4 should-not-trigger):
|
||||
|
||||
**Example for Docker skill:**
|
||||
|
||||
Should trigger:
|
||||
1. "Create a docker container for my Node.js app"
|
||||
2. "Build an image from this Dockerfile"
|
||||
3. "How do I compose up my services?"
|
||||
4. "I need to deploy this using containers"
|
||||
|
||||
Should not trigger:
|
||||
1. "Install Docker on my machine" (installation vs usage)
|
||||
2. "What is containerization?" (education vs hands-on)
|
||||
3. "Show me Kubernetes commands" (different tool)
|
||||
4. "Write a fibonacci function" (completely unrelated)
|
||||
|
||||
### Examples
|
||||
|
||||
**Good (Appropriate Triggering):**
|
||||
- User: "containerize my app"
|
||||
- Skill: Docker helper activates ✓
|
||||
|
||||
**Bad (Missed Trigger):**
|
||||
- User: "I need to run my app in containers"
|
||||
- Skill: Doesn't activate (too narrow description) ✗
|
||||
|
||||
**Bad (False Trigger):**
|
||||
- User: "Install Docker on Ubuntu"
|
||||
- Skill: Activates but only handles usage, not installation ✗
|
||||
|
||||
---
|
||||
|
||||
## Efficiency
|
||||
|
||||
### What to Check
|
||||
|
||||
**No unnecessary steps:**
|
||||
- Doesn't do redundant work
|
||||
- Doesn't suggest unnecessary commands
|
||||
- Shortcuts used where safe
|
||||
- Process is streamlined
|
||||
|
||||
**Reasonable response length:**
|
||||
- Not excessively verbose
|
||||
- Doesn't repeat information
|
||||
- No filler content
|
||||
- Every sentence adds value
|
||||
|
||||
**Not overly verbose:**
|
||||
- Commands not over-explained
|
||||
- Doesn't explain obvious steps
|
||||
- Concise but complete
|
||||
- Respects user's expertise
|
||||
|
||||
### Examples
|
||||
|
||||
**Good (Efficient):**
|
||||
```bash
|
||||
# Clean up Docker resources
|
||||
docker system prune -f
|
||||
```
|
||||
|
||||
**Bad (Verbose):**
|
||||
```bash
|
||||
# Clean up Docker resources
|
||||
# First, we need to run the docker command
|
||||
# Then we use the system subcommand
|
||||
# Then we use the prune subcommand
|
||||
# The -f flag means force
|
||||
docker system prune -f
|
||||
# This will remove stopped containers, unused networks, and dangling images
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Identifying Patterns
|
||||
|
||||
### Single Test Failure
|
||||
|
||||
**Characteristics:**
|
||||
- Only one test case fails
|
||||
- Issue is unique to that scenario
|
||||
- Other tests pass
|
||||
|
||||
**Action:**
|
||||
- Add targeted instruction
|
||||
- Include example for that edge case
|
||||
- Fix the specific issue
|
||||
- Don't generalize prematurely
|
||||
|
||||
**Example:**
|
||||
```
|
||||
Issue: Large file processing fails
|
||||
Solution: Add instruction about memory-efficient processing
|
||||
for files over 10,000 rows
|
||||
```
|
||||
|
||||
### Multiple Test Failures (Same Issue)
|
||||
|
||||
**Characteristics:**
|
||||
- Same problem across 2+ tests
|
||||
- Pattern suggests root cause
|
||||
- Symptom appears in different forms
|
||||
|
||||
**Action:**
|
||||
- Fix the root cause
|
||||
- Add general principle, not specific fix
|
||||
- Consider extracting to script
|
||||
- Explain the "why"
|
||||
|
||||
**Example:**
|
||||
```
|
||||
Issue: Tests 1 and 3 both have incorrect date formatting
|
||||
Solution: Add general instruction about ISO 8601 date format
|
||||
rather than fixing each case individually
|
||||
```
|
||||
|
||||
### Multiple Test Failures (Different Issues)
|
||||
|
||||
**Characteristics:**
|
||||
- Different problems in each test
|
||||
- No clear pattern
|
||||
- Skill may be too broad
|
||||
|
||||
**Action:**
|
||||
- Clarify scope in description
|
||||
- Add more specific instructions
|
||||
- Consider splitting into multiple skills
|
||||
- Focus on core functionality first
|
||||
|
||||
**Example:**
|
||||
```
|
||||
Issue: Test 1 fails on format, Test 2 fails on logic, Test 3
|
||||
never triggers skill
|
||||
Solution: Clarify what the skill does/doesn't do in description
|
||||
Add validation steps
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Decision Framework
|
||||
|
||||
### Fix Specific Case When:
|
||||
|
||||
- Unique edge case not covered
|
||||
- One-time issue unlikely to recur
|
||||
- Fix is simple and doesn't add complexity
|
||||
- Doesn't indicate systemic problem
|
||||
|
||||
**Example:**
|
||||
```
|
||||
Test: CSV with Unicode characters fails
|
||||
Fix: Add note about UTF-8 encoding for international characters
|
||||
(Don't rewrite entire CSV handling)
|
||||
```
|
||||
|
||||
### Generalize Solution When:
|
||||
|
||||
- Same issue in 2+ tests
|
||||
- Pattern suggests broader applicability
|
||||
- Fix would benefit similar requests
|
||||
- Explains underlying principle
|
||||
|
||||
**Example:**
|
||||
```
|
||||
Tests: Both test 1 and 2 have incorrect error handling
|
||||
Fix: Add general principle about validating inputs before processing
|
||||
with examples of common validation checks
|
||||
```
|
||||
|
||||
### Extract to Script When:
|
||||
|
||||
- Same multi-step process repeated
|
||||
- Deterministic operation (not creative)
|
||||
- Would save time on every invocation
|
||||
- Logic is complex or error-prone
|
||||
|
||||
**Example:**
|
||||
```
|
||||
Pattern: All three tests require converting dates to ISO format
|
||||
Action: Create scripts/convert-dates.py
|
||||
Include in SKILL.md: "Use scripts/convert-dates.py for date formatting"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Common Issues and Solutions
|
||||
|
||||
### Issue: Skill Doesn't Trigger
|
||||
|
||||
**Symptoms:**
|
||||
- User says relevant phrase
|
||||
- Skill doesn't appear in available skills
|
||||
- Model handles request without skill
|
||||
|
||||
**Solutions:**
|
||||
1. Add specific trigger phrases to description
|
||||
2. Use "pushy" language: "Make sure to use this skill whenever..."
|
||||
3. Include synonyms and variations
|
||||
4. Test with 8 trigger queries
|
||||
|
||||
### Issue: Output Format Inconsistent
|
||||
|
||||
**Symptoms:**
|
||||
- Sometimes follows template, sometimes doesn't
|
||||
- Format varies between similar requests
|
||||
- Missing sections or elements
|
||||
|
||||
**Solutions:**
|
||||
1. Add explicit format template in SKILL.md
|
||||
2. Provide multiple examples
|
||||
3. Explain why format matters
|
||||
4. Use imperative: "ALWAYS use this template"
|
||||
|
||||
### Issue: Edge Cases Not Handled
|
||||
|
||||
**Symptoms:**
|
||||
- Large files cause errors
|
||||
- Empty inputs crash
|
||||
- Special characters corrupted
|
||||
- Missing data causes failures
|
||||
|
||||
**Solutions:**
|
||||
1. Add validation step instructions
|
||||
2. Include error handling examples
|
||||
3. Create helper script for complex validation
|
||||
4. Document known limitations
|
||||
|
||||
### Issue: Too Verbose
|
||||
|
||||
**Symptoms:**
|
||||
- Responses are walls of text
|
||||
- Over-explains obvious steps
|
||||
- Repeats information
|
||||
- Takes too long to get to point
|
||||
|
||||
**Solutions:**
|
||||
1. Remove redundant explanations
|
||||
2. Trust user's expertise
|
||||
3. Move details to references/
|
||||
4. Use progressive disclosure
|
||||
|
||||
### Issue: Incomplete Responses
|
||||
|
||||
**Symptoms:**
|
||||
- Misses steps in workflow
|
||||
- Forgets validation
|
||||
- Doesn't mention prerequisites
|
||||
- No error handling
|
||||
|
||||
**Solutions:**
|
||||
1. Add comprehensive checklist in SKILL.md
|
||||
2. Include validation at each step
|
||||
3. Document prerequisites upfront
|
||||
4. Add error handling examples
|
||||
|
||||
### Issue: Incorrect Commands
|
||||
|
||||
**Symptoms:**
|
||||
- Commands don't work when copied
|
||||
- Syntax errors
|
||||
- Wrong flags or options
|
||||
- Outdated versions
|
||||
|
||||
**Solutions:**
|
||||
1. Test all commands before including
|
||||
2. Specify version requirements
|
||||
3. Add validation steps
|
||||
4. Include error messages to expect
|
||||
|
||||
---
|
||||
|
||||
## When to Stop Iterating
|
||||
|
||||
### Stop When:
|
||||
|
||||
1. **User is satisfied**
|
||||
- User says "this works for me"
|
||||
- User stops requesting changes
|
||||
- User starts using skill regularly
|
||||
|
||||
2. **Outputs meet expectations**
|
||||
- All test cases pass
|
||||
- Common scenarios work
|
||||
- Edge cases handled reasonably
|
||||
- No critical issues remain
|
||||
|
||||
3. **No meaningful progress**
|
||||
- Last 2-3 iterations didn't improve results
|
||||
- Changes are cosmetic only
|
||||
- Diminishing returns on effort
|
||||
- 90% solution is good enough
|
||||
|
||||
### Don't Stop When:
|
||||
|
||||
- **Perfect is the enemy of good** - 90% working is better than endlessly tweaking
|
||||
- **Edge cases are theoretical** - Don't over-engineer for unlikely scenarios
|
||||
- **User wants to keep iterating** - Follow user's lead
|
||||
- **New issues emerge** - Fix real problems as they're found
|
||||
|
||||
### The 90% Rule
|
||||
|
||||
A skill that works well for 90% of cases is sufficient. Users can handle:
|
||||
- Rare edge cases manually
|
||||
- Unique situations with custom guidance
|
||||
- Complex scenarios with multiple steps
|
||||
|
||||
**Focus on:**
|
||||
- Common use cases (80% of value)
|
||||
- Clear failure messages (10% of value)
|
||||
- Good documentation (10% of value)
|
||||
|
||||
**Don't focus on:**
|
||||
- Every possible edge case (diminishing returns)
|
||||
- Perfect formatting (good enough is fine)
|
||||
- Handling every possible error (major errors are enough)
|
||||
|
||||
---
|
||||
|
||||
## Grading Checklist
|
||||
|
||||
Use this checklist for each test case:
|
||||
|
||||
### Correctness
|
||||
- [ ] Output matches expected result
|
||||
- [ ] No factual errors
|
||||
- [ ] Logic is sound
|
||||
- [ ] Edge cases handled appropriately
|
||||
|
||||
### Completeness
|
||||
- [ ] All requested tasks completed
|
||||
- [ ] No steps skipped
|
||||
- [ ] Appropriate level of detail
|
||||
- [ ] Relevant context included
|
||||
|
||||
### Format
|
||||
- [ ] Output follows specified format
|
||||
- [ ] Consistent with examples
|
||||
- [ ] Easy to read and understand
|
||||
|
||||
### Triggering
|
||||
- [ ] Skill activated when appropriate
|
||||
- [ ] Did not activate when inappropriate
|
||||
|
||||
### Efficiency
|
||||
- [ ] No unnecessary steps
|
||||
- [ ] Reasonable response length
|
||||
- [ ] Not overly verbose
|
||||
|
||||
### Overall
|
||||
- [ ] Would use this skill again
|
||||
- [ ] Would recommend to others
|
||||
- [ ] Saves time vs manual approach
|
||||
- [ ] Output quality meets needs
|
||||
|
||||
---
|
||||
|
||||
## Recording Results
|
||||
|
||||
After grading, record:
|
||||
|
||||
```json
|
||||
{
|
||||
"test_id": 1,
|
||||
"result": "pass|fail|partial",
|
||||
"issues": [
|
||||
"Description of issue 1",
|
||||
"Description of issue 2"
|
||||
],
|
||||
"suggested_fix": "Brief description of improvement",
|
||||
"extract_script": false,
|
||||
"priority": "high|medium|low"
|
||||
}
|
||||
```
|
||||
|
||||
Use the grade-output.sh script to generate this structure interactively.
|
||||
|
||||
---
|
||||
|
||||
## Next Steps After Grading
|
||||
|
||||
1. **Review all results** - Look for patterns
|
||||
2. **Prioritize fixes** - High priority first
|
||||
3. **Update SKILL.md** - Based on issues found
|
||||
4. **Create scripts** - Extract repeated work
|
||||
5. **Re-run tests** - Verify improvements
|
||||
6. **Repeat** - Until satisfied or good enough
|
||||
@@ -0,0 +1,888 @@
|
||||
# Skill Templates
|
||||
|
||||
Copy-paste templates for common skill patterns. Use these as starting points when creating new skills.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Simple Skill (Knowledge Only)](#simple-skill)
|
||||
2. [Tool Integration Skill](#tool-integration-skill)
|
||||
3. [Workflow Skill](#workflow-skill)
|
||||
4. [Documentation Skill](#documentation-skill)
|
||||
5. [Testing Your Skill](#testing-your-skill)
|
||||
|
||||
---
|
||||
|
||||
## Simple Skill
|
||||
|
||||
For skills that provide knowledge without scripts or complex resources.
|
||||
|
||||
**Use when:** The skill provides guidance, best practices, or reference information that can be explained in text.
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
simple-skill/
|
||||
└── SKILL.md
|
||||
```
|
||||
|
||||
### SKILL.md Template
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: simple-skill
|
||||
description: This skill should be used when the user asks to "TRIGGER_PHRASE_1", "TRIGGER_PHRASE_2", or needs help with SPECIFIC_TASK
|
||||
license: MIT
|
||||
compatibility: opencode
|
||||
metadata:
|
||||
category: CATEGORY
|
||||
version: "1.0.0"
|
||||
---
|
||||
|
||||
# Simple Skill Name
|
||||
|
||||
Brief description of what this skill covers.
|
||||
|
||||
## Overview
|
||||
|
||||
High-level explanation of the domain or topic.
|
||||
|
||||
## Common Tasks
|
||||
|
||||
### Task 1: Description
|
||||
|
||||
Step-by-step instructions:
|
||||
|
||||
1. First step with code example
|
||||
2. Second step with code example
|
||||
3. Third step
|
||||
|
||||
### Task 2: Description
|
||||
|
||||
Step-by-step instructions:
|
||||
|
||||
1. First step
|
||||
2. Second step
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Bullet point 1
|
||||
- Bullet point 2
|
||||
- Bullet point 3
|
||||
|
||||
## Important Notes
|
||||
|
||||
- Critical caveat 1
|
||||
- Critical caveat 2
|
||||
```
|
||||
|
||||
### Example: Git Commit Guidelines
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: git-commit-guide
|
||||
description: This skill should be used when the user asks to "write a commit message", "format a commit", "conventional commits", or needs help with Git commit standards
|
||||
license: MIT
|
||||
compatibility: opencode
|
||||
---
|
||||
|
||||
# Git Commit Guidelines
|
||||
|
||||
Follow conventional commits specification for clear, structured commit messages.
|
||||
|
||||
## Format
|
||||
|
||||
```
|
||||
<type>(<scope>): <description>
|
||||
|
||||
[optional body]
|
||||
|
||||
[optional footer]
|
||||
```
|
||||
|
||||
## Types
|
||||
|
||||
- `feat`: New feature
|
||||
- `fix`: Bug fix
|
||||
- `docs`: Documentation only
|
||||
- `style`: Code style (formatting, semicolons, etc)
|
||||
- `refactor`: Code refactoring
|
||||
- `test`: Adding or updating tests
|
||||
- `chore`: Maintenance tasks
|
||||
|
||||
## Examples
|
||||
|
||||
```bash
|
||||
# Feature
|
||||
feat(auth): add OAuth2 login support
|
||||
|
||||
# Fix with scope
|
||||
fix(api): resolve null pointer in user endpoint
|
||||
|
||||
# Breaking change
|
||||
feat(db)!: migrate to PostgreSQL
|
||||
```
|
||||
|
||||
## Rules
|
||||
|
||||
- Use imperative mood: "add" not "added"
|
||||
- Don't capitalize first letter
|
||||
- No period at end
|
||||
- Keep first line under 72 characters
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tool Integration Skill
|
||||
|
||||
For skills that interact with command-line tools and include automation scripts.
|
||||
|
||||
**Use when:** The skill wraps a CLI tool, provides automation, or requires executable scripts.
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
tool-skill/
|
||||
├── SKILL.md
|
||||
├── scripts/
|
||||
│ └── helper.sh
|
||||
├── references/
|
||||
│ └── advanced-usage.md
|
||||
└── evals/
|
||||
└── evals.json
|
||||
```
|
||||
|
||||
### SKILL.md Template
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: tool-skill
|
||||
description: This skill should be used when the user asks to "TRIGGER_PHRASE_1", "run TOOL_NAME", "TOOL_NAME commands", or work with TOOL_NAME
|
||||
license: MIT
|
||||
compatibility: opencode
|
||||
metadata:
|
||||
category: tools
|
||||
version: "1.0.0"
|
||||
---
|
||||
|
||||
# Tool Name Integration
|
||||
|
||||
Interact with TOOL_NAME for SPECIFIC_PURPOSE.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Basic usage:
|
||||
|
||||
```bash
|
||||
tool-name --option value
|
||||
```
|
||||
|
||||
## Common Operations
|
||||
|
||||
### Operation 1: Description
|
||||
|
||||
```bash
|
||||
# Example command
|
||||
tool-name command --flag
|
||||
```
|
||||
|
||||
Explanation of what this does.
|
||||
|
||||
### Operation 2: Description
|
||||
|
||||
```bash
|
||||
# Example command
|
||||
tool-name another-command
|
||||
```
|
||||
|
||||
## Scripts
|
||||
|
||||
Use helper scripts in `scripts/`:
|
||||
|
||||
- `scripts/helper.sh` - What this script does
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
For detailed options and edge cases, see `references/advanced-usage.md`.
|
||||
|
||||
## Important Notes
|
||||
|
||||
- Requirement 1
|
||||
- Requirement 2
|
||||
- Common pitfall
|
||||
```
|
||||
|
||||
### scripts/helper.sh Template
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
#
|
||||
# Helper script for TOOL_NAME operations
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
DEFAULT_OPTION="value"
|
||||
|
||||
# Functions
|
||||
show_help() {
|
||||
cat << EOF
|
||||
Usage: $0 [OPTIONS]
|
||||
|
||||
Helper script for TOOL_NAME
|
||||
|
||||
Options:
|
||||
-h, --help Show this help message
|
||||
-v, --verbose Enable verbose output
|
||||
-o, --option Specify option value
|
||||
|
||||
Examples:
|
||||
$0 --verbose
|
||||
$0 --option custom-value
|
||||
EOF
|
||||
}
|
||||
|
||||
# Parse arguments
|
||||
VERBOSE=false
|
||||
OPTION="$DEFAULT_OPTION"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
show_help
|
||||
exit 0
|
||||
;;
|
||||
-v|--verbose)
|
||||
VERBOSE=true
|
||||
shift
|
||||
;;
|
||||
-o|--option)
|
||||
OPTION="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
show_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Main logic
|
||||
main() {
|
||||
[[ "$VERBOSE" == true ]] && echo "Running with option: $OPTION"
|
||||
|
||||
# Your tool integration logic here
|
||||
echo "Helper script executed successfully"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
```
|
||||
|
||||
### Example: Docker Helper
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: docker-helper
|
||||
description: This skill should be used when the user asks to "manage docker containers", "build docker images", "docker compose operations", or work with Docker workflows
|
||||
license: MIT
|
||||
compatibility: opencode
|
||||
---
|
||||
|
||||
# Docker Helper
|
||||
|
||||
Streamline Docker container and image management.
|
||||
|
||||
## Quick Commands
|
||||
|
||||
### List Running Containers
|
||||
|
||||
```bash
|
||||
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
||||
```
|
||||
|
||||
### Clean Up System
|
||||
|
||||
```bash
|
||||
# Remove stopped containers, unused networks, dangling images
|
||||
docker system prune -f
|
||||
```
|
||||
|
||||
## Scripts
|
||||
|
||||
Use helper scripts:
|
||||
|
||||
- `scripts/container-logs.sh` - View and follow container logs
|
||||
- `scripts/cleanup.sh` - Safe cleanup of unused resources
|
||||
|
||||
## Docker Compose
|
||||
|
||||
### Start Services
|
||||
|
||||
```bash
|
||||
docker compose up -d SERVICE_NAME
|
||||
```
|
||||
|
||||
### View Logs
|
||||
|
||||
```bash
|
||||
docker compose logs -f SERVICE_NAME
|
||||
```
|
||||
|
||||
## References
|
||||
|
||||
- `references/compose-patterns.md` - Advanced compose configurations
|
||||
- `references/troubleshooting.md` - Common issues and solutions
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Workflow Skill
|
||||
|
||||
For skills that guide multi-step processes or complex procedures.
|
||||
|
||||
**Use when:** The skill orchestrates a sequence of steps, decisions, or collaborative tasks.
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
workflow-skill/
|
||||
├── SKILL.md
|
||||
├── scripts/
|
||||
│ ├── step-validator.sh
|
||||
│ └── automation.sh
|
||||
├── references/
|
||||
│ ├── decision-matrix.md
|
||||
│ └── examples/
|
||||
│ └── sample-workflow.md
|
||||
└── evals/
|
||||
└── evals.json
|
||||
```
|
||||
|
||||
### SKILL.md Template
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: workflow-skill
|
||||
description: This skill should be used when the user asks to "WORKFLOW_NAME", "perform WORKFLOW_TASK", "WORKFLOW_VERB process", or needs help with WORKFLOW_DOMAIN
|
||||
license: MIT
|
||||
compatibility: opencode
|
||||
metadata:
|
||||
category: workflow
|
||||
version: "1.0.0"
|
||||
---
|
||||
|
||||
# Workflow Name
|
||||
|
||||
Guide the WORKFLOW_PROCESS from start to finish with validation at each step.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before starting, ensure:
|
||||
|
||||
1. Requirement 1
|
||||
2. Requirement 2
|
||||
3. Requirement 3
|
||||
|
||||
## Workflow Overview
|
||||
|
||||
```
|
||||
Step 1 → Step 2 → Step 3 → Completion
|
||||
↓ ↓ ↓
|
||||
Validate Validate Validate
|
||||
```
|
||||
|
||||
## Step-by-Step Process
|
||||
|
||||
### Step 1: Preparation
|
||||
|
||||
**Objective:** What to accomplish in this step
|
||||
|
||||
**Actions:**
|
||||
1. Action item 1
|
||||
2. Action item 2
|
||||
|
||||
**Validation:**
|
||||
- [ ] Check item 1
|
||||
- [ ] Check item 2
|
||||
|
||||
**Script:** `scripts/step-validator.sh --step 1`
|
||||
|
||||
### Step 2: Execution
|
||||
|
||||
**Objective:** What to accomplish in this step
|
||||
|
||||
**Actions:**
|
||||
1. Action item 1
|
||||
2. Action item 2
|
||||
|
||||
**Validation:**
|
||||
- [ ] Check item 1
|
||||
- [ ] Check item 2
|
||||
|
||||
**Decision Point:**
|
||||
|
||||
If CONDITION_A:
|
||||
- Follow path A (see `references/path-a.md`)
|
||||
|
||||
If CONDITION_B:
|
||||
- Follow path B (see `references/path-b.md`)
|
||||
|
||||
### Step 3: Verification
|
||||
|
||||
**Objective:** Final checks and completion
|
||||
|
||||
**Actions:**
|
||||
1. Run verification command
|
||||
2. Review output
|
||||
|
||||
**Validation:**
|
||||
- [ ] All checks pass
|
||||
- [ ] No errors in logs
|
||||
|
||||
## Automation
|
||||
|
||||
For automated execution:
|
||||
|
||||
```bash
|
||||
scripts/automation.sh --full
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
See `references/troubleshooting.md` for common issues.
|
||||
|
||||
## Examples
|
||||
|
||||
Complete workflow examples in `references/examples/`.
|
||||
```
|
||||
|
||||
### Example: Release Workflow
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: release-workflow
|
||||
description: This skill should be used when the user asks to "create a release", "publish a new version", "tag a release", or prepare software for deployment
|
||||
license: MIT
|
||||
compatibility: opencode
|
||||
---
|
||||
|
||||
# Release Workflow
|
||||
|
||||
Guide the complete release process from version bump to deployment.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [ ] All tests passing
|
||||
- [ ] CHANGELOG.md updated
|
||||
- [ ] Version number decided
|
||||
|
||||
## Workflow
|
||||
|
||||
### Step 1: Pre-Release Checks
|
||||
|
||||
**Actions:**
|
||||
1. Run full test suite: `npm test`
|
||||
2. Verify CHANGELOG.md is current
|
||||
3. Check for uncommitted changes
|
||||
|
||||
**Validation:**
|
||||
```bash
|
||||
scripts/pre-release-check.sh
|
||||
```
|
||||
|
||||
### Step 2: Version Bump
|
||||
|
||||
**Actions:**
|
||||
1. Update version in package.json
|
||||
2. Update CHANGELOG.md with release date
|
||||
3. Commit with message: `chore(release): bump version to X.Y.Z`
|
||||
|
||||
**Validation:**
|
||||
- [ ] Version updated in all files
|
||||
- [ ] CHANGELOG.md has date
|
||||
- [ ] Commit created
|
||||
|
||||
### Step 3: Create Tag
|
||||
|
||||
**Actions:**
|
||||
1. Create annotated tag: `git tag -a vX.Y.Z -m "Release X.Y.Z"`
|
||||
2. Push tag: `git push origin vX.Y.Z`
|
||||
|
||||
**Validation:**
|
||||
```bash
|
||||
scripts/verify-tag.sh vX.Y.Z
|
||||
```
|
||||
|
||||
### Step 4: GitHub Release
|
||||
|
||||
**Actions:**
|
||||
1. Draft release notes from CHANGELOG
|
||||
2. Create GitHub release
|
||||
3. Attach artifacts if needed
|
||||
|
||||
**Command:**
|
||||
```bash
|
||||
gh release create vX.Y.Z --notes-from-tag
|
||||
```
|
||||
|
||||
## Decision Matrix
|
||||
|
||||
See `references/decision-matrix.md` for:
|
||||
- Version numbering (semver vs calver)
|
||||
- Pre-release vs stable
|
||||
- Hotfix procedures
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Documentation Skill
|
||||
|
||||
For skills focused on writing, reviewing, or managing documentation.
|
||||
|
||||
**Use when:** The skill helps create documentation, enforces standards, or provides writing guidelines.
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
docs-skill/
|
||||
├── SKILL.md
|
||||
├── references/
|
||||
│ ├── style-guide.md
|
||||
│ ├── templates/
|
||||
│ │ ├── README-template.md
|
||||
│ │ └── API-doc-template.md
|
||||
│ └── examples/
|
||||
│ └── sample-docs/
|
||||
├── assets/
|
||||
│ └── diagram-template.png
|
||||
└── evals/
|
||||
└── evals.json
|
||||
```
|
||||
|
||||
### SKILL.md Template
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: docs-skill
|
||||
description: This skill should be used when the user asks to "write documentation", "create a README", "document this code", "review docs", or needs help with technical writing
|
||||
license: MIT
|
||||
compatibility: opencode
|
||||
metadata:
|
||||
category: documentation
|
||||
version: "1.0.0"
|
||||
---
|
||||
|
||||
# Documentation Helper
|
||||
|
||||
Create clear, consistent, and effective technical documentation.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Create README
|
||||
|
||||
Start with the template:
|
||||
|
||||
```bash
|
||||
cp references/templates/README-template.md ./README.md
|
||||
```
|
||||
|
||||
Then fill in:
|
||||
1. Project name and description
|
||||
2. Installation steps
|
||||
3. Usage examples
|
||||
4. API reference (if applicable)
|
||||
|
||||
## Documentation Types
|
||||
|
||||
### README.md
|
||||
|
||||
Essential sections:
|
||||
- Title and description
|
||||
- Installation
|
||||
- Quick start
|
||||
- Usage examples
|
||||
- API reference (if applicable)
|
||||
- Contributing
|
||||
- License
|
||||
|
||||
Use template: `references/templates/README-template.md`
|
||||
|
||||
### API Documentation
|
||||
|
||||
Structure:
|
||||
- Endpoint description
|
||||
- Request format
|
||||
- Response format
|
||||
- Error codes
|
||||
- Examples
|
||||
|
||||
Use template: `references/templates/API-doc-template.md`
|
||||
|
||||
### Code Comments
|
||||
|
||||
Follow style guide in `references/style-guide.md`:
|
||||
- JSDoc for JavaScript
|
||||
- Docstrings for Python
|
||||
- Rustdoc for Rust
|
||||
|
||||
## Style Guide
|
||||
|
||||
See `references/style-guide.md` for:
|
||||
- Tone and voice
|
||||
- Formatting rules
|
||||
- Terminology
|
||||
- Code example standards
|
||||
|
||||
## Review Checklist
|
||||
|
||||
Before finalizing documentation:
|
||||
|
||||
- [ ] Clear and concise
|
||||
- [ ] No typos or grammar errors
|
||||
- [ ] Code examples work
|
||||
- [ ] All links functional
|
||||
- [ ] Proper heading hierarchy
|
||||
- [ ] Consistent formatting
|
||||
|
||||
## Examples
|
||||
|
||||
Review well-documented projects in `references/examples/`.
|
||||
|
||||
## Assets
|
||||
|
||||
- `assets/diagram-template.png` - Template for architecture diagrams
|
||||
```
|
||||
|
||||
### Example: API Documentation Helper
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: api-docs-helper
|
||||
description: This skill should be used when the user asks to "document an API", "create API documentation", "write endpoint docs", or needs help with REST API documentation
|
||||
license: MIT
|
||||
compatibility: opencode
|
||||
---
|
||||
|
||||
# API Documentation Helper
|
||||
|
||||
Create comprehensive REST API documentation following OpenAPI standards.
|
||||
|
||||
## Endpoint Documentation Template
|
||||
|
||||
For each endpoint, document:
|
||||
|
||||
### 1. Overview
|
||||
|
||||
```markdown
|
||||
## POST /api/v1/resource
|
||||
|
||||
Brief description of what this endpoint does.
|
||||
|
||||
**Authorization:** Required (Bearer token)
|
||||
**Rate Limit:** 100 requests/minute
|
||||
```
|
||||
|
||||
### 2. Request
|
||||
|
||||
```markdown
|
||||
### Headers
|
||||
|
||||
| Header | Value | Required |
|
||||
|--------|-------|----------|
|
||||
| Content-Type | application/json | Yes |
|
||||
| Authorization | Bearer {token} | Yes |
|
||||
|
||||
### Body
|
||||
|
||||
```json
|
||||
{
|
||||
"field1": "string (required) - Description",
|
||||
"field2": "number (optional) - Description"
|
||||
}
|
||||
```
|
||||
```
|
||||
|
||||
### 3. Response
|
||||
|
||||
```markdown
|
||||
### Success (200 OK)
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "string",
|
||||
"created_at": "ISO 8601 datetime",
|
||||
"status": "active"
|
||||
}
|
||||
```
|
||||
|
||||
### Error Responses
|
||||
|
||||
- `400 Bad Request` - Invalid input
|
||||
- `401 Unauthorized` - Missing/invalid token
|
||||
- `404 Not Found` - Resource doesn't exist
|
||||
```
|
||||
|
||||
### 4. Examples
|
||||
|
||||
Provide curl and client library examples.
|
||||
|
||||
## Standards
|
||||
|
||||
Follow conventions in `references/api-standards.md`:
|
||||
- Use JSON for request/response bodies
|
||||
- ISO 8601 for dates
|
||||
- snake_case for field names
|
||||
- Include pagination metadata for lists
|
||||
|
||||
## Tools
|
||||
|
||||
Validate documentation:
|
||||
- `scripts/validate-openapi.sh` - Check OpenAPI spec
|
||||
- `scripts/check-examples.sh` - Verify code examples work
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Testing Your Skill
|
||||
|
||||
After creating a skill from these templates, follow this testing workflow:
|
||||
|
||||
### Step 1: Create Test Cases
|
||||
|
||||
Generate test cases for your skill:
|
||||
|
||||
```bash
|
||||
~/.config/opencode/skills/skill-builder/scripts/create-tests.sh ~/.config/opencode/skills/your-skill
|
||||
```
|
||||
|
||||
This creates `evals/evals.json` with 3 template test cases:
|
||||
1. **Common Case** - Typical usage scenario
|
||||
2. **Edge Case** - Tricky or unusual situation
|
||||
3. **Varied Phrasing** - Same intent, different words
|
||||
|
||||
### Step 2: Customize Test Prompts
|
||||
|
||||
Edit `evals/evals.json` and replace placeholder text with realistic test prompts:
|
||||
|
||||
**Good test prompt:**
|
||||
```
|
||||
Convert the CSV file at ./data/sales.csv to JSON and save it as ./output/sales.json.
|
||||
The CSV has headers: date, product, quantity, price. Make sure dates are in ISO 8601 format.
|
||||
```
|
||||
|
||||
**Bad test prompt:**
|
||||
```
|
||||
Convert CSV to JSON
|
||||
```
|
||||
|
||||
See `eval-templates.md` for copy-paste templates for different skill types.
|
||||
|
||||
### Step 3: Run Tests
|
||||
|
||||
Execute the test workflow:
|
||||
|
||||
```bash
|
||||
~/.config/opencode/skills/skill-builder/scripts/run-tests.sh ~/.config/opencode/skills/your-skill
|
||||
```
|
||||
|
||||
This will:
|
||||
- Display each test prompt
|
||||
- Guide you through manual testing in opencode
|
||||
- Record your evaluations (pass/fail/skip)
|
||||
- Save results to `evals/test-results.json`
|
||||
|
||||
### Step 4: Grade Results
|
||||
|
||||
Use the grading checklist for systematic evaluation:
|
||||
|
||||
```bash
|
||||
~/.config/opencode/skills/skill-builder/scripts/grade-output.sh ~/.config/opencode/skills/your-skill
|
||||
```
|
||||
|
||||
This evaluates:
|
||||
- Correctness
|
||||
- Completeness
|
||||
- Format
|
||||
- Triggering
|
||||
- Efficiency
|
||||
|
||||
And generates `evals/grading-report.json` with structured feedback.
|
||||
|
||||
### Step 5: Iterate
|
||||
|
||||
Based on test results, improve your skill:
|
||||
|
||||
1. Review issues in grading report
|
||||
2. Update SKILL.md with fixes
|
||||
3. Re-run tests
|
||||
4. Repeat until satisfied
|
||||
|
||||
See `grading-guide.md` for detailed evaluation criteria and decision frameworks.
|
||||
|
||||
### Testing by Skill Type
|
||||
|
||||
**Simple Skills:**
|
||||
- May not need formal test cases
|
||||
- Test manually with 2-3 realistic prompts
|
||||
- Verify outputs are helpful and accurate
|
||||
|
||||
**Tool Integration Skills:**
|
||||
- Must test all commands work when copied
|
||||
- Verify edge cases (errors, conflicts)
|
||||
- Test with different tool versions if possible
|
||||
|
||||
**Workflow Skills:**
|
||||
- Test complete end-to-end flow
|
||||
- Verify each step produces correct result
|
||||
- Test decision points and branches
|
||||
- Check error recovery paths
|
||||
|
||||
**Code Generation Skills:**
|
||||
- Verify generated code is syntactically valid
|
||||
- Test that code actually runs
|
||||
- Check edge cases (empty input, large data)
|
||||
- Validate error handling
|
||||
|
||||
### When to Stop Testing
|
||||
|
||||
Stop iterating when:
|
||||
- All critical test cases pass
|
||||
- User is satisfied with quality
|
||||
- Common scenarios work reliably
|
||||
- No longer making meaningful improvements
|
||||
|
||||
Remember: A skill that works well for 90% of cases is sufficient. Perfection is the enemy of good.
|
||||
|
||||
---
|
||||
|
||||
## Tips for All Templates
|
||||
|
||||
1. **Replace placeholders immediately** - Search for `TRIGGER_PHRASE`, `TOOL_NAME`, etc.
|
||||
|
||||
2. **Write description first** - Include 3-5 specific trigger phrases in quotes
|
||||
|
||||
3. **Keep examples working** - Test all code examples before finalizing
|
||||
|
||||
4. **Validate early and often** - Run validation after every significant change
|
||||
|
||||
5. **Progressive disclosure** - Move details to references/ as the skill grows
|
||||
|
||||
6. **Be specific** - Generic skills don't trigger well. Make descriptions concrete.
|
||||
|
||||
7. **Test triggers** - After creating, try phrases from your description to verify activation
|
||||
|
||||
## Validation Checklist for New Skills
|
||||
|
||||
Before considering a skill complete:
|
||||
|
||||
- [ ] SKILL.md created with proper frontmatter
|
||||
- [ ] `name` matches directory name
|
||||
- [ ] `description` is 20-1024 characters with trigger phrases
|
||||
- [ ] No second-person writing in body
|
||||
- [ ] All scripts are executable
|
||||
- [ ] Referenced files exist and are mentioned in SKILL.md
|
||||
- [ ] Validation passes: `validate-skill.sh /path/to/skill`
|
||||
- [ ] Test cases created in `evals/evals.json`
|
||||
- [ ] Tests pass or issues documented
|
||||
- [ ] Tested with actual trigger phrases
|
||||
179
.config/opencode/skills/.skill-builder.disabled/scripts/create-tests.sh
Executable file
179
.config/opencode/skills/.skill-builder.disabled/scripts/create-tests.sh
Executable file
@@ -0,0 +1,179 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# create-tests.sh - Interactive test case generator for skills
|
||||
#
|
||||
# Usage: ./create-tests.sh <path/to/skill-directory>
|
||||
#
|
||||
# This script interactively generates evals/evals.json with 3 template test cases.
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Parse arguments
|
||||
SKILL_DIR=""
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <path/to/skill-directory>"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 ~/.config/opencode/skills/my-skill"
|
||||
echo " $0 ./my-skill"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
-*)
|
||||
echo -e "${RED}Error: Unknown option $1${NC}"
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
SKILL_DIR="$1"
|
||||
else
|
||||
echo -e "${RED}Error: Multiple skill directories provided${NC}"
|
||||
usage
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate skill directory
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
echo -e "${RED}Error: Skill directory is required${NC}"
|
||||
usage
|
||||
fi
|
||||
|
||||
# Resolve path
|
||||
SKILL_DIR="$(cd "$SKILL_DIR" 2>/dev/null && pwd)" || {
|
||||
echo -e "${RED}Error: Cannot access directory: $SKILL_DIR${NC}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Get skill name from directory
|
||||
SKILL_NAME="$(basename "$SKILL_DIR")"
|
||||
|
||||
# Verify SKILL.md exists
|
||||
if [[ ! -f "$SKILL_DIR/SKILL.md" ]]; then
|
||||
echo -e "${RED}Error: SKILL.md not found in $SKILL_DIR${NC}"
|
||||
echo "This does not appear to be a valid skill directory."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "========================================"
|
||||
echo "Creating Test Cases for: $SKILL_NAME"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Create evals directory
|
||||
mkdir -p "$SKILL_DIR/evals"
|
||||
echo -e "${GREEN}✓ Created evals/ directory${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if evals.json already exists
|
||||
if [[ -f "$SKILL_DIR/evals/evals.json" ]]; then
|
||||
echo -e "${YELLOW}⚠ evals.json already exists${NC}"
|
||||
read -p "Overwrite? (y/N): " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "Aborted."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Get skill purpose from user
|
||||
echo "Let's create 3 test cases for your skill."
|
||||
echo ""
|
||||
echo "First, tell me about your skill's purpose:"
|
||||
echo " (e.g., 'Helps users convert CSV files to JSON format')"
|
||||
read -r SKILL_PURPOSE
|
||||
|
||||
if [[ -z "$SKILL_PURPOSE" ]]; then
|
||||
SKILL_PURPOSE="[Describe what your skill does]"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Now I'll generate 3 test case templates."
|
||||
echo "You can edit evals/evals.json later to customize them."
|
||||
echo ""
|
||||
|
||||
# Generate evals.json
|
||||
cat > "$SKILL_DIR/evals/evals.json" << EVALSJSON
|
||||
{
|
||||
"skill_name": "$SKILL_NAME",
|
||||
"description": "$SKILL_PURPOSE",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "[REPLACE WITH REALISTIC USER REQUEST]\n\nExample: Convert the CSV file at ./data/sales.csv to JSON format and save it as ./output/sales.json. Make sure dates are in ISO 8601 format.",
|
||||
"expected_output": "[DESCRIBE EXPECTED RESULT]\n\nExample: A JSON file at ./output/sales.json containing the sales data from the CSV, with all dates converted to ISO 8601 format.",
|
||||
"assertions": [
|
||||
"Output file exists at the specified location",
|
||||
"File is valid JSON",
|
||||
"Dates are in ISO 8601 format"
|
||||
],
|
||||
"notes": "This tests the typical, straightforward use case."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case",
|
||||
"type": "edge",
|
||||
"prompt": "[REPLACE WITH EDGE CASE SCENARIO]\n\nExample: Convert a CSV file with 100,000 rows, some containing special characters (emojis, Unicode) and empty values in certain columns. The file is at ./data/large_export.csv.",
|
||||
"expected_output": "[DESCRIBE EXPECTED BEHAVIOR]\n\nExample: JSON file is created successfully, handling all special characters correctly and preserving empty values as null or empty strings.",
|
||||
"assertions": [
|
||||
"Large file is processed without errors",
|
||||
"Special characters are preserved correctly",
|
||||
"Empty values are handled appropriately"
|
||||
],
|
||||
"notes": "This tests edge cases like large files, special characters, or missing data."
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing",
|
||||
"type": "variation",
|
||||
"prompt": "[REPLACE WITH SAME INTENT, DIFFERENT WORDS]\n\nExample: Hey, I've got this spreadsheet in data/sales.csv. Can you turn it into JSON for me? And make sure the dates look right - you know, standard format?",
|
||||
"expected_output": "[SAME AS COMMON CASE]",
|
||||
"assertions": [
|
||||
"Skill triggers correctly with casual language",
|
||||
"Output matches common case results",
|
||||
"Implicit requirements (date formatting) are handled"
|
||||
],
|
||||
"notes": "This tests that the skill works with different phrasings and casual language."
|
||||
}
|
||||
]
|
||||
}
|
||||
EVALSJSON
|
||||
|
||||
echo -e "${GREEN}✓ Created evals/evals.json${NC}"
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Next Steps:"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "1. Edit evals/evals.json"
|
||||
echo " Replace the [PLACEHOLDER] text with realistic test cases"
|
||||
echo ""
|
||||
echo "2. Tips for writing good test prompts:"
|
||||
echo " - Include file paths (e.g., ./data/file.csv)"
|
||||
echo " - Add personal context (e.g., 'my boss sent me')"
|
||||
echo " - Use specific values and column names"
|
||||
echo " - Mix formal and casual language"
|
||||
echo ""
|
||||
echo "3. Run tests when ready:"
|
||||
echo " ~/.config/opencode/skills/skill-builder/scripts/run-tests.sh $SKILL_DIR"
|
||||
echo ""
|
||||
echo -e "${GREEN}Done!${NC}"
|
||||
363
.config/opencode/skills/.skill-builder.disabled/scripts/grade-output.sh
Executable file
363
.config/opencode/skills/.skill-builder.disabled/scripts/grade-output.sh
Executable file
@@ -0,0 +1,363 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# grade-output.sh - Interactive grading checklist for skill outputs
|
||||
#
|
||||
# Usage: ./grade-output.sh <path/to/skill-directory>
|
||||
#
|
||||
# This script provides a structured checklist for evaluating skill test outputs.
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Parse arguments
|
||||
SKILL_DIR=""
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <path/to/skill-directory>"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 ~/.config/opencode/skills/my-skill"
|
||||
echo " $0 ./my-skill"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
-*)
|
||||
echo -e "${RED}Error: Unknown option $1${NC}"
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
SKILL_DIR="$1"
|
||||
else
|
||||
echo -e "${RED}Error: Multiple skill directories provided${NC}"
|
||||
usage
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate skill directory
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
echo -e "${RED}Error: Skill directory is required${NC}"
|
||||
usage
|
||||
fi
|
||||
|
||||
# Resolve path
|
||||
SKILL_DIR="$(cd "$SKILL_DIR" 2>/dev/null && pwd)" || {
|
||||
echo -e "${RED}Error: Cannot access directory: $SKILL_DIR${NC}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Get skill name from directory
|
||||
SKILL_NAME="$(basename "$SKILL_DIR")"
|
||||
|
||||
echo "========================================"
|
||||
echo "Grading Output for: $SKILL_NAME"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Check for test results
|
||||
if [[ -f "$SKILL_DIR/evals/test-results.json" ]]; then
|
||||
echo -e "${BLUE}Found previous test results${NC}"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Initialize grading data
|
||||
declare -a CRITERIA=(
|
||||
"Correctness: Output matches expected result"
|
||||
"Correctness: No factual errors"
|
||||
"Correctness: Logic is sound"
|
||||
"Correctness: Edge cases handled appropriately"
|
||||
"Completeness: All requested tasks completed"
|
||||
"Completeness: No steps skipped"
|
||||
"Completeness: Appropriate level of detail"
|
||||
"Completeness: Relevant context included"
|
||||
"Format: Output follows specified format"
|
||||
"Format: Consistent with examples in skill"
|
||||
"Format: Easy to read and understand"
|
||||
"Triggering: Skill activated when appropriate"
|
||||
"Triggering: Did not activate when inappropriate"
|
||||
"Efficiency: No unnecessary steps"
|
||||
"Efficiency: Reasonable response length"
|
||||
"Efficiency: Not overly verbose"
|
||||
)
|
||||
|
||||
GRADES=()
|
||||
ISSUES=()
|
||||
|
||||
# Function to ask yes/no question
|
||||
ask_yes_no() {
|
||||
local prompt="$1"
|
||||
while true; do
|
||||
read -p "$prompt (y/n): " -n 1 -r
|
||||
echo
|
||||
case $REPLY in
|
||||
[Yy])
|
||||
return 0
|
||||
;;
|
||||
[Nn])
|
||||
return 1
|
||||
;;
|
||||
*)
|
||||
echo "Please enter y or n"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
echo "This checklist will help you systematically evaluate the skill output."
|
||||
echo "Answer each question based on the test results you observed."
|
||||
echo ""
|
||||
read -p "Press Enter to begin grading..."
|
||||
echo ""
|
||||
|
||||
# Grade each criterion
|
||||
echo "========================================"
|
||||
echo -e "${CYAN}Grading Criteria${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
for criterion in "${CRITERIA[@]}"; do
|
||||
category="${criterion%%:*}"
|
||||
description="${criterion#*: }"
|
||||
|
||||
echo -e "${BLUE}[$category]${NC} $description"
|
||||
|
||||
if ask_yes_no " Does it meet this criterion"; then
|
||||
GRADES+=("$criterion: PASS")
|
||||
echo -e " ${GREEN}✓ Pass${NC}"
|
||||
else
|
||||
GRADES+=("$criterion: FAIL")
|
||||
echo -e " ${RED}✗ Fail${NC}"
|
||||
|
||||
# Ask for issue description
|
||||
echo " Briefly describe the issue:"
|
||||
read -r issue
|
||||
if [[ -n "$issue" ]]; then
|
||||
ISSUES+=("[$category] $description: $issue")
|
||||
fi
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Overall assessment
|
||||
echo "========================================"
|
||||
echo -e "${CYAN}Overall Assessment${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
echo "Overall Result:"
|
||||
echo " [p] Pass - All or most criteria met"
|
||||
echo " [f] Fail - Significant issues found"
|
||||
echo " [i] Incomplete - Needs more testing"
|
||||
echo ""
|
||||
|
||||
while true; do
|
||||
read -p "Overall result (p/f/i): " -n 1 -r
|
||||
echo
|
||||
case $REPLY in
|
||||
[Pp])
|
||||
OVERALL_RESULT="pass"
|
||||
break
|
||||
;;
|
||||
[Ff])
|
||||
OVERALL_RESULT="fail"
|
||||
break
|
||||
;;
|
||||
[Ii])
|
||||
OVERALL_RESULT="incomplete"
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Please enter p, f, or i"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
# Priority assessment
|
||||
echo "Priority of fixes needed:"
|
||||
echo " [h] High - Critical issues, skill not usable"
|
||||
echo " [m] Medium - Important issues, skill partially works"
|
||||
echo " [l] Low - Minor issues, skill mostly works"
|
||||
echo ""
|
||||
|
||||
while true; do
|
||||
read -p "Priority (h/m/l): " -n 1 -r
|
||||
echo
|
||||
case $REPLY in
|
||||
[Hh])
|
||||
PRIORITY="high"
|
||||
break
|
||||
;;
|
||||
[Mm])
|
||||
PRIORITY="medium"
|
||||
break
|
||||
;;
|
||||
[Ll])
|
||||
PRIORITY="low"
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Please enter h, m, or l"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
# Suggested fixes
|
||||
echo -e "${BLUE}Suggested Improvements (optional):${NC}"
|
||||
echo "Describe what changes would address the issues:"
|
||||
read -r SUGGESTED_FIXES
|
||||
|
||||
echo ""
|
||||
|
||||
# Pattern analysis
|
||||
echo "========================================"
|
||||
echo -e "${CYAN}Pattern Analysis${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
if ask_yes_no "Did the same issue appear in multiple test cases"; then
|
||||
echo "This suggests a systemic problem. Consider:"
|
||||
echo " - Fixing the root cause rather than symptoms"
|
||||
echo " - Adding a helper script for repeated tasks"
|
||||
echo " - Clarifying instructions in SKILL.md"
|
||||
PATTERN="systemic"
|
||||
else
|
||||
echo "Issues appear to be isolated to specific cases."
|
||||
PATTERN="isolated"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Extract to script recommendation
|
||||
if ask_yes_no "Should any repeated work be extracted to a script"; then
|
||||
echo "Consider creating a script in scripts/ directory."
|
||||
EXTRACT_SCRIPT="true"
|
||||
else
|
||||
EXTRACT_SCRIPT="false"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Generate grading report
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# Build issues array
|
||||
ISSUES_JSON="["
|
||||
for i in "${!ISSUES[@]}"; do
|
||||
if [[ $i -gt 0 ]]; then
|
||||
ISSUES_JSON+=","
|
||||
fi
|
||||
ISSUES_JSON+="\"${ISSUES[$i]}\""
|
||||
done
|
||||
ISSUES_JSON+="]"
|
||||
|
||||
# Build grades array
|
||||
GRADES_JSON="["
|
||||
for i in "${!GRADES[@]}"; do
|
||||
if [[ $i -gt 0 ]]; then
|
||||
GRADES_JSON+=","
|
||||
fi
|
||||
GRADES_JSON+="\"${GRADES[$i]}\""
|
||||
done
|
||||
GRADES_JSON+="]"
|
||||
|
||||
# Calculate pass rate
|
||||
TOTAL_CRITERIA=${#CRITERIA[@]}
|
||||
PASSED_COUNT=0
|
||||
for grade in "${GRADES[@]}"; do
|
||||
if [[ "$grade" == *"PASS" ]]; then
|
||||
((PASSED_COUNT++))
|
||||
fi
|
||||
done
|
||||
|
||||
PASS_RATE=$((PASSED_COUNT * 100 / TOTAL_CRITERIA))
|
||||
|
||||
cat > "$SKILL_DIR/evals/grading-report.json" << EOF
|
||||
{
|
||||
"skill_name": "$SKILL_NAME",
|
||||
"timestamp": "$TIMESTAMP",
|
||||
"overall_result": "$OVERALL_RESULT",
|
||||
"priority": "$PRIORITY",
|
||||
"pass_rate": $PASS_RATE,
|
||||
"criteria_passed": $PASSED_COUNT,
|
||||
"criteria_total": $TOTAL_CRITERIA,
|
||||
"pattern_analysis": "$PATTERN",
|
||||
"extract_script_recommended": $EXTRACT_SCRIPT,
|
||||
"detailed_grades": $GRADES_JSON,
|
||||
"issues": $ISSUES_JSON,
|
||||
"suggested_fixes": "$SUGGESTED_FIXES"
|
||||
}
|
||||
EOF
|
||||
|
||||
echo "========================================"
|
||||
echo -e "${GREEN}Grading Report Generated${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Saved to: evals/grading-report.json"
|
||||
echo ""
|
||||
echo "Summary:"
|
||||
echo " Overall: $OVERALL_RESULT"
|
||||
echo " Priority: $PRIORITY"
|
||||
echo " Pass Rate: $PASS_RATE% ($PASSED_COUNT/$TOTAL_CRITERIA criteria)"
|
||||
echo " Pattern: $PATTERN issues"
|
||||
echo ""
|
||||
|
||||
if [[ ${#ISSUES[@]} -gt 0 ]]; then
|
||||
echo -e "${YELLOW}Issues Found:${NC}"
|
||||
for issue in "${ISSUES[@]}"; do
|
||||
echo " - $issue"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Next steps
|
||||
echo "========================================"
|
||||
echo "Next Steps"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
if [[ "$OVERALL_RESULT" == "pass" ]]; then
|
||||
echo -e "${GREEN}✓ Skill is working well!${NC}"
|
||||
echo ""
|
||||
echo "Consider:"
|
||||
echo " - Adding more edge case tests"
|
||||
echo " - Optimizing the description"
|
||||
echo " - Documenting the skill"
|
||||
else
|
||||
echo "To improve the skill:"
|
||||
echo ""
|
||||
echo "1. Review grading-report.json for details"
|
||||
echo "2. Update SKILL.md based on the issues found"
|
||||
echo ""
|
||||
if [[ "$EXTRACT_SCRIPT" == "true" ]]; then
|
||||
echo "3. Create helper scripts for repeated tasks"
|
||||
echo " - Place scripts in scripts/ directory"
|
||||
echo " - Update SKILL.md to reference them"
|
||||
echo ""
|
||||
fi
|
||||
echo "4. Re-run tests to verify improvements:"
|
||||
echo " ~/.config/opencode/skills/skill-builder/scripts/run-tests.sh $SKILL_DIR"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}Done!${NC}"
|
||||
391
.config/opencode/skills/.skill-builder.disabled/scripts/init-skill.sh
Executable file
391
.config/opencode/skills/.skill-builder.disabled/scripts/init-skill.sh
Executable file
@@ -0,0 +1,391 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# init-skill.sh - Scaffold a new skill with proper structure
|
||||
#
|
||||
# Usage: ./init-skill.sh <skill-name> [options]
|
||||
#
|
||||
# Options:
|
||||
# --local Create in current project (.opencode/skills/) instead of global
|
||||
# --with-scripts Include scripts/ directory
|
||||
# --with-refs Include references/ directory
|
||||
# --with-assets Include assets/ directory
|
||||
# --with-tests Include evals/ directory for test cases
|
||||
# --full Include all optional directories
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Get script directory
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SKILL_BUILDER_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
# Parse arguments
|
||||
SKILL_NAME=""
|
||||
LOCAL=false
|
||||
WITH_SCRIPTS=false
|
||||
WITH_REFS=false
|
||||
WITH_ASSETS=false
|
||||
WITH_TESTS=false
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <skill-name> [options]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --local Create in current project (.opencode/skills/)"
|
||||
echo " --with-scripts Include scripts/ directory"
|
||||
echo " --with-refs Include references/ directory"
|
||||
echo " --with-assets Include assets/ directory"
|
||||
echo " --with-tests Include evals/ directory for test cases"
|
||||
echo " --full Include all optional directories"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 my-skill"
|
||||
echo " $0 my-skill --full"
|
||||
echo " $0 my-skill --local --with-scripts"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Validate skill name
|
||||
validate_name() {
|
||||
local name="$1"
|
||||
|
||||
if [[ -z "$name" ]]; then
|
||||
echo -e "${RED}Error: Skill name is required${NC}"
|
||||
usage
|
||||
fi
|
||||
|
||||
if [[ ! "$name" =~ ^[a-z0-9-]+$ ]]; then
|
||||
echo -e "${RED}Error: Skill name must be lowercase alphanumeric with hyphens only${NC}"
|
||||
echo " Valid: my-skill, docker-helper, test-runner"
|
||||
echo " Invalid: My Skill, docker_helper, testRunner"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ${#name} -lt 1 ]]; then
|
||||
echo -e "${RED}Error: Skill name cannot be empty${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ${#name} -gt 64 ]]; then
|
||||
echo -e "${RED}Error: Skill name must be under 64 characters${NC}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--local)
|
||||
LOCAL=true
|
||||
shift
|
||||
;;
|
||||
--with-scripts)
|
||||
WITH_SCRIPTS=true
|
||||
shift
|
||||
;;
|
||||
--with-refs)
|
||||
WITH_REFS=true
|
||||
shift
|
||||
;;
|
||||
--with-assets)
|
||||
WITH_ASSETS=true
|
||||
shift
|
||||
;;
|
||||
--with-tests)
|
||||
WITH_TESTS=true
|
||||
shift
|
||||
;;
|
||||
--full)
|
||||
WITH_SCRIPTS=true
|
||||
WITH_REFS=true
|
||||
WITH_ASSETS=true
|
||||
WITH_TESTS=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
-*)
|
||||
echo -e "${RED}Error: Unknown option $1${NC}"
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$SKILL_NAME" ]]; then
|
||||
SKILL_NAME="$1"
|
||||
else
|
||||
echo -e "${RED}Error: Multiple skill names provided${NC}"
|
||||
usage
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate skill name
|
||||
validate_name "$SKILL_NAME"
|
||||
|
||||
# Determine target directory
|
||||
if [[ "$LOCAL" == true ]]; then
|
||||
# Check if we're in a git repository
|
||||
if ! git rev-parse --git-dir > /dev/null 2>&1; then
|
||||
echo -e "${RED}Error: --local requires being in a git repository${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TARGET_DIR=".opencode/skills/$SKILL_NAME"
|
||||
|
||||
# Check for AGENTS.md in project
|
||||
if [[ -f "AGENTS.md" ]]; then
|
||||
echo -e "${GREEN}✓ Found AGENTS.md in project root${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ No AGENTS.md found in project root${NC}"
|
||||
echo " Consider creating one for project-specific rules"
|
||||
fi
|
||||
else
|
||||
TARGET_DIR="$HOME/.config/opencode/skills/$SKILL_NAME"
|
||||
fi
|
||||
|
||||
# Check if skill already exists
|
||||
if [[ -d "$TARGET_DIR" ]]; then
|
||||
echo -e "${RED}Error: Skill already exists at $TARGET_DIR${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}Creating skill: $SKILL_NAME${NC}"
|
||||
echo " Location: $TARGET_DIR"
|
||||
echo ""
|
||||
|
||||
# Create skill directory
|
||||
mkdir -p "$TARGET_DIR"
|
||||
|
||||
# Create SKILL.md
|
||||
cat > "$TARGET_DIR/SKILL.md" << 'SKILL_TEMPLATE'
|
||||
---
|
||||
name: SKILL_NAME_PLACEHOLDER
|
||||
description: This skill should be used when the user asks to "DESCRIPTION_HERE". Add specific trigger phrases that would activate this skill.
|
||||
license: MIT
|
||||
compatibility: opencode
|
||||
metadata:
|
||||
category: general
|
||||
version: "1.0.0"
|
||||
---
|
||||
|
||||
# SKILL_NAME_PLACEHOLDER
|
||||
|
||||
Brief description of what this skill does and its purpose.
|
||||
|
||||
## What This Skill Provides
|
||||
|
||||
1. **Feature 1** - Brief description
|
||||
2. **Feature 2** - Brief description
|
||||
3. **Feature 3** - Brief description
|
||||
|
||||
## Quick Start
|
||||
|
||||
Basic usage example:
|
||||
|
||||
```bash
|
||||
# Example command
|
||||
some-tool --option value
|
||||
```
|
||||
|
||||
## Common Tasks
|
||||
|
||||
### Task 1: Description
|
||||
|
||||
Step-by-step instructions:
|
||||
|
||||
1. First step
|
||||
2. Second step
|
||||
3. Third step
|
||||
|
||||
### Task 2: Description
|
||||
|
||||
Step-by-step instructions:
|
||||
|
||||
1. First step
|
||||
2. Second step
|
||||
|
||||
## Important Notes
|
||||
|
||||
- Keep this section brief
|
||||
- Use bullet points for clarity
|
||||
- Reference supporting files if available
|
||||
|
||||
## Resources
|
||||
|
||||
SKILL_TEMPLATE
|
||||
|
||||
# Add resource section if directories were created
|
||||
if [[ "$WITH_REFS" == true ]]; then
|
||||
cat >> "$TARGET_DIR/SKILL.md" << 'REF_SECTION'
|
||||
|
||||
### Reference Files
|
||||
|
||||
- `references/detailed-guide.md` - Detailed documentation
|
||||
|
||||
REF_SECTION
|
||||
fi
|
||||
|
||||
if [[ "$WITH_SCRIPTS" == true ]]; then
|
||||
cat >> "$TARGET_DIR/SKILL.md" << 'SCRIPT_SECTION'
|
||||
|
||||
### Scripts
|
||||
|
||||
- `scripts/helper.sh` - Utility script
|
||||
|
||||
SCRIPT_SECTION
|
||||
fi
|
||||
|
||||
if [[ "$WITH_ASSETS" == true ]]; then
|
||||
cat >> "$TARGET_DIR/SKILL.md" << 'ASSET_SECTION'
|
||||
|
||||
### Assets
|
||||
|
||||
- `assets/template.txt` - Template file
|
||||
|
||||
ASSET_SECTION
|
||||
fi
|
||||
|
||||
# Replace placeholder with actual skill name
|
||||
sed -i "s/SKILL_NAME_PLACEHOLDER/$SKILL_NAME/g" "$TARGET_DIR/SKILL.md"
|
||||
|
||||
# Create optional directories
|
||||
if [[ "$WITH_SCRIPTS" == true ]]; then
|
||||
mkdir -p "$TARGET_DIR/scripts"
|
||||
|
||||
# Create example script
|
||||
cat > "$TARGET_DIR/scripts/helper.sh" << 'SCRIPT_EXAMPLE'
|
||||
#!/bin/bash
|
||||
#
|
||||
# Helper script for SKILL_NAME_PLACEHOLDER
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo "Helper script for SKILL_NAME_PLACEHOLDER"
|
||||
echo "Modify this script for your needs"
|
||||
SCRIPT_EXAMPLE
|
||||
|
||||
sed -i "s/SKILL_NAME_PLACEHOLDER/$SKILL_NAME/g" "$TARGET_DIR/scripts/helper.sh"
|
||||
chmod +x "$TARGET_DIR/scripts/helper.sh"
|
||||
|
||||
echo -e "${GREEN} ✓ Created scripts/ directory${NC}"
|
||||
fi
|
||||
|
||||
if [[ "$WITH_REFS" == true ]]; then
|
||||
mkdir -p "$TARGET_DIR/references"
|
||||
|
||||
# Create example reference
|
||||
cat > "$TARGET_DIR/references/detailed-guide.md" << 'REF_EXAMPLE'
|
||||
# Detailed Guide for SKILL_NAME_PLACEHOLDER
|
||||
|
||||
This file contains detailed documentation that can be referenced on-demand.
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
Detailed instructions go here...
|
||||
|
||||
## Configuration
|
||||
|
||||
Configuration options and examples...
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
Common issues and solutions...
|
||||
REF_EXAMPLE
|
||||
|
||||
sed -i "s/SKILL_NAME_PLACEHOLDER/$SKILL_NAME/g" "$TARGET_DIR/references/detailed-guide.md"
|
||||
|
||||
echo -e "${GREEN} ✓ Created references/ directory${NC}"
|
||||
fi
|
||||
|
||||
if [[ "$WITH_ASSETS" == true ]]; then
|
||||
mkdir -p "$TARGET_DIR/assets"
|
||||
|
||||
# Create example asset
|
||||
cat > "$TARGET_DIR/assets/template.txt" << 'ASSET_EXAMPLE'
|
||||
# Template file for SKILL_NAME_PLACEHOLDER
|
||||
|
||||
This is a template file that can be used as a starting point.
|
||||
Modify it according to your needs.
|
||||
ASSET_EXAMPLE
|
||||
|
||||
sed -i "s/SKILL_NAME_PLACEHOLDER/$SKILL_NAME/g" "$TARGET_DIR/assets/template.txt"
|
||||
|
||||
echo -e "${GREEN} ✓ Created assets/ directory${NC}"
|
||||
fi
|
||||
|
||||
if [[ "$WITH_TESTS" == true ]]; then
|
||||
mkdir -p "$TARGET_DIR/evals"
|
||||
|
||||
# Create template evals.json
|
||||
cat > "$TARGET_DIR/evals/evals.json" << 'EVALS_TEMPLATE'
|
||||
{
|
||||
"skill_name": "SKILL_NAME_PLACEHOLDER",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"prompt": "Typical user request with realistic context, file paths, and specific details. This represents the most common way users will ask for help.",
|
||||
"expected_output": "Description of what the skill should produce for this request",
|
||||
"assertions": [
|
||||
"Check that output contains specific expected content",
|
||||
"Verify file is created at expected location (if applicable)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case",
|
||||
"prompt": "Unusual or tricky scenario that tests edge cases, error handling, or complex requirements. Include something that might break the skill.",
|
||||
"expected_output": "Description of expected behavior for this edge case",
|
||||
"assertions": [
|
||||
"Verify edge case is handled appropriately",
|
||||
"Check for graceful error handling if applicable"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing",
|
||||
"prompt": "Same intent as the common case but expressed with different words, casual language, or alternative phrasing. Tests skill robustness to language variation.",
|
||||
"expected_output": "Same expected output as common case",
|
||||
"assertions": [
|
||||
"Output should match common case results",
|
||||
"Skill triggers correctly with different phrasing"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EVALS_TEMPLATE
|
||||
|
||||
sed -i "s/SKILL_NAME_PLACEHOLDER/$SKILL_NAME/g" "$TARGET_DIR/evals/evals.json"
|
||||
|
||||
echo -e "${GREEN} ✓ Created evals/ directory with template evals.json${NC}"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}✓ Skill '$SKILL_NAME' created successfully!${NC}"
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. Edit $TARGET_DIR/SKILL.md"
|
||||
echo " 2. Fill in the description with specific trigger phrases"
|
||||
echo " 3. Add your skill content"
|
||||
if [[ "$WITH_REFS" == true ]]; then
|
||||
echo " 4. Add detailed content to references/ files"
|
||||
fi
|
||||
if [[ "$WITH_SCRIPTS" == true ]]; then
|
||||
echo " 5. Implement scripts in scripts/ directory"
|
||||
fi
|
||||
if [[ "$WITH_TESTS" == true ]]; then
|
||||
echo " 6. Customize test cases in evals/evals.json"
|
||||
echo " 7. Run tests: ${SKILL_BUILDER_DIR}/scripts/run-tests.sh $TARGET_DIR"
|
||||
fi
|
||||
echo ""
|
||||
echo "Validate your skill:"
|
||||
echo " ${SKILL_BUILDER_DIR}/scripts/validate-skill.sh $TARGET_DIR"
|
||||
325
.config/opencode/skills/.skill-builder.disabled/scripts/run-tests.sh
Executable file
325
.config/opencode/skills/.skill-builder.disabled/scripts/run-tests.sh
Executable file
@@ -0,0 +1,325 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# run-tests.sh - Execute test workflow and capture results
|
||||
#
|
||||
# Usage: ./run-tests.sh <path/to/skill-directory>
|
||||
#
|
||||
# This script guides you through testing each test case manually and records results.
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Parse arguments
|
||||
SKILL_DIR=""
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <path/to/skill-directory>"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 ~/.config/opencode/skills/my-skill"
|
||||
echo " $0 ./my-skill"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
-*)
|
||||
echo -e "${RED}Error: Unknown option $1${NC}"
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
SKILL_DIR="$1"
|
||||
else
|
||||
echo -e "${RED}Error: Multiple skill directories provided${NC}"
|
||||
usage
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate skill directory
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
echo -e "${RED}Error: Skill directory is required${NC}"
|
||||
usage
|
||||
fi
|
||||
|
||||
# Resolve path
|
||||
SKILL_DIR="$(cd "$SKILL_DIR" 2>/dev/null && pwd)" || {
|
||||
echo -e "${RED}Error: Cannot access directory: $SKILL_DIR${NC}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Get skill name from directory
|
||||
SKILL_NAME="$(basename "$SKILL_DIR")"
|
||||
|
||||
# Verify evals.json exists
|
||||
if [[ ! -f "$SKILL_DIR/evals/evals.json" ]]; then
|
||||
echo -e "${RED}Error: evals/evals.json not found${NC}"
|
||||
echo ""
|
||||
echo "Create test cases first:"
|
||||
echo " ~/.config/opencode/skills/skill-builder/scripts/create-tests.sh $SKILL_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "========================================"
|
||||
echo "Running Tests for: $SKILL_NAME"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Check if jq is available
|
||||
if ! command -v jq &> /dev/null; then
|
||||
echo -e "${YELLOW}⚠ jq is not installed${NC}"
|
||||
echo " Install jq for better JSON handling:"
|
||||
echo " Ubuntu/Debian: sudo apt-get install jq"
|
||||
echo " macOS: brew install jq"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Falling back to basic parsing...${NC}"
|
||||
USE_JQ=false
|
||||
else
|
||||
USE_JQ=true
|
||||
fi
|
||||
|
||||
# Initialize results array
|
||||
RESULTS=()
|
||||
TEST_COUNT=0
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
SKIPPED=0
|
||||
|
||||
# Function to extract value from JSON using basic parsing
|
||||
get_json_value() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
local index="${3:-}"
|
||||
|
||||
if [[ "$USE_JQ" == true ]]; then
|
||||
if [[ -n "$index" ]]; then
|
||||
jq -r ".evals[$index].$key" "$file" 2>/dev/null || echo ""
|
||||
else
|
||||
jq -r ".$key" "$file" 2>/dev/null || echo ""
|
||||
fi
|
||||
else
|
||||
# Basic grep-based extraction (fallback)
|
||||
if [[ -n "$index" ]]; then
|
||||
# This is a simplified fallback - won't handle nested structures well
|
||||
grep -A 100 '"evals":' "$file" | grep -A 20 "\"id\": $index" | grep "\"$key\":" | head -1 | sed 's/.*"'$key'": "\(.*\)".*/\1/' | sed 's/",*$//'
|
||||
else
|
||||
grep "\"$key\":" "$file" | head -1 | sed 's/.*"'$key'": "\(.*\)".*/\1/' | sed 's/",*$//'
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Count test cases
|
||||
if [[ "$USE_JQ" == true ]]; then
|
||||
TEST_COUNT=$(jq '.evals | length' "$SKILL_DIR/evals/evals.json")
|
||||
else
|
||||
TEST_COUNT=$(grep -c '"id":' "$SKILL_DIR/evals/evals.json" 2>/dev/null || echo "0")
|
||||
fi
|
||||
|
||||
if [[ "$TEST_COUNT" -eq 0 ]]; then
|
||||
echo -e "${RED}Error: No test cases found in evals.json${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Found $TEST_COUNT test case(s)"
|
||||
echo ""
|
||||
|
||||
# Process each test case
|
||||
for ((i=0; i<TEST_COUNT; i++)); do
|
||||
echo "========================================"
|
||||
echo -e "${CYAN}Test Case $((i+1)) of $TEST_COUNT${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Extract test details
|
||||
if [[ "$USE_JQ" == true ]]; then
|
||||
TEST_ID=$(jq -r ".evals[$i].id" "$SKILL_DIR/evals/evals.json")
|
||||
TEST_NAME=$(jq -r ".evals[$i].name" "$SKILL_DIR/evals/evals.json")
|
||||
TEST_PROMPT=$(jq -r ".evals[$i].prompt" "$SKILL_DIR/evals/evals.json")
|
||||
TEST_EXPECTED=$(jq -r ".evals[$i].expected_output" "$SKILL_DIR/evals/evals.json")
|
||||
TEST_TYPE=$(jq -r ".evals[$i].type // .evals[$i].name" "$SKILL_DIR/evals/evals.json")
|
||||
else
|
||||
# Fallback parsing
|
||||
TEST_ID=$((i+1))
|
||||
TEST_NAME="test-$((i+1))"
|
||||
TEST_PROMPT="[Prompt extraction requires jq - install jq for full functionality]"
|
||||
TEST_EXPECTED="[Expected output extraction requires jq]"
|
||||
TEST_TYPE="unknown"
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}Test Name:${NC} $TEST_NAME"
|
||||
echo -e "${BLUE}Type:${NC} $TEST_TYPE"
|
||||
echo ""
|
||||
echo -e "${BLUE}Prompt:${NC}"
|
||||
echo "----------------------------------------"
|
||||
echo "$TEST_PROMPT"
|
||||
echo "----------------------------------------"
|
||||
echo ""
|
||||
|
||||
echo -e "${BLUE}Expected Output:${NC}"
|
||||
echo "$TEST_EXPECTED"
|
||||
echo ""
|
||||
|
||||
# Instructions for manual testing
|
||||
echo -e "${YELLOW}Instructions:${NC}"
|
||||
echo "1. Copy the prompt above"
|
||||
echo "2. Open a new opencode session"
|
||||
echo "3. Paste the prompt and observe the result"
|
||||
echo "4. Return here to record the outcome"
|
||||
echo ""
|
||||
|
||||
read -p "Press Enter when ready to record results..."
|
||||
echo ""
|
||||
|
||||
# Get test result
|
||||
echo "Test Result:"
|
||||
echo " [p] Pass - Output met expectations"
|
||||
echo " [f] Fail - Output did not meet expectations"
|
||||
echo " [s] Skip - Could not test or not applicable"
|
||||
echo ""
|
||||
|
||||
while true; do
|
||||
read -p "Result (p/f/s): " -n 1 -r
|
||||
echo
|
||||
case $REPLY in
|
||||
[Pp])
|
||||
TEST_RESULT="pass"
|
||||
((PASSED++))
|
||||
break
|
||||
;;
|
||||
[Ff])
|
||||
TEST_RESULT="fail"
|
||||
((FAILED++))
|
||||
break
|
||||
;;
|
||||
[Ss])
|
||||
TEST_RESULT="skip"
|
||||
((SKIPPED++))
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Please enter p, f, or s"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
# Get notes
|
||||
echo -e "${BLUE}Notes (optional):${NC}"
|
||||
echo "Describe any issues, observations, or suggestions:"
|
||||
read -r TEST_NOTES
|
||||
|
||||
# Get timestamp
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# Build result entry
|
||||
RESULT_ENTRY=$(cat << EOF
|
||||
{
|
||||
"test_id": $TEST_ID,
|
||||
"test_name": "$TEST_NAME",
|
||||
"result": "$TEST_RESULT",
|
||||
"notes": "$TEST_NOTES",
|
||||
"timestamp": "$TIMESTAMP"
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
RESULTS+=("$RESULT_ENTRY")
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}✓ Recorded test result${NC}"
|
||||
echo ""
|
||||
|
||||
# Ask to continue or stop
|
||||
if [[ $i -lt $((TEST_COUNT-1)) ]]; then
|
||||
read -p "Continue to next test? (Y/n): " -n 1 -r
|
||||
echo
|
||||
if [[ $REPLY =~ ^[Nn]$ ]]; then
|
||||
echo "Stopping test run..."
|
||||
break
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
|
||||
# Generate test-results.json
|
||||
echo "========================================"
|
||||
echo "Generating Test Results"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Create results JSON
|
||||
RUN_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# Build JSON array from results
|
||||
RESULTS_JSON="["
|
||||
for i in "${!RESULTS[@]}"; do
|
||||
if [[ $i -gt 0 ]]; then
|
||||
RESULTS_JSON+=","
|
||||
fi
|
||||
RESULTS_JSON+="${RESULTS[$i]}"
|
||||
done
|
||||
RESULTS_JSON+="]"
|
||||
|
||||
cat > "$SKILL_DIR/evals/test-results.json" << EOF
|
||||
{
|
||||
"skill_name": "$SKILL_NAME",
|
||||
"run_timestamp": "$RUN_TIMESTAMP",
|
||||
"summary": {
|
||||
"total": $((${#RESULTS[@]})),
|
||||
"passed": $PASSED,
|
||||
"failed": $FAILED,
|
||||
"skipped": $SKIPPED
|
||||
},
|
||||
"results": $RESULTS_JSON
|
||||
}
|
||||
EOF
|
||||
|
||||
echo -e "${GREEN}✓ Saved results to evals/test-results.json${NC}"
|
||||
echo ""
|
||||
|
||||
# Display summary
|
||||
echo "========================================"
|
||||
echo "Test Run Summary"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Total Tests: $((${#RESULTS[@]}))"
|
||||
echo -e "${GREEN}Passed: $PASSED${NC}"
|
||||
echo -e "${RED}Failed: $FAILED${NC}"
|
||||
echo -e "${YELLOW}Skipped: $SKIPPED${NC}"
|
||||
echo ""
|
||||
|
||||
# Next steps
|
||||
if [[ $FAILED -gt 0 ]]; then
|
||||
echo "========================================"
|
||||
echo -e "${YELLOW}Next Steps:${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Some tests failed. To improve your skill:"
|
||||
echo ""
|
||||
echo "1. Review test-results.json for details"
|
||||
echo "2. Update SKILL.md to address issues"
|
||||
echo "3. Run tests again:"
|
||||
echo " $0 $SKILL_DIR"
|
||||
echo ""
|
||||
echo "For detailed grading:"
|
||||
echo " ~/.config/opencode/skills/skill-builder/scripts/grade-output.sh $SKILL_DIR"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}Done!${NC}"
|
||||
308
.config/opencode/skills/.skill-builder.disabled/scripts/validate-skill.sh
Executable file
308
.config/opencode/skills/.skill-builder.disabled/scripts/validate-skill.sh
Executable file
@@ -0,0 +1,308 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# validate-skill.sh - Strictly validate a skill's structure and content
|
||||
#
|
||||
# Usage: ./validate-skill.sh <path/to/skill-directory> [options]
|
||||
#
|
||||
# Options:
|
||||
# --strict Fail on warnings (default)
|
||||
# --lenient Only fail on errors, report warnings
|
||||
# --verbose Show detailed output
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Validation counters
|
||||
ERRORS=0
|
||||
WARNINGS=0
|
||||
|
||||
# Default mode
|
||||
STRICT=true
|
||||
VERBOSE=false
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <path/to/skill-directory> [options]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --strict Fail on warnings (default)"
|
||||
echo " --lenient Only fail on errors"
|
||||
echo " --verbose Show detailed output"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 ~/.config/opencode/skills/my-skill"
|
||||
echo " $0 ./my-skill --verbose"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Error and warning functions
|
||||
error() {
|
||||
echo -e "${RED}✗ ERROR:${NC} $1"
|
||||
((ERRORS++)) || true
|
||||
}
|
||||
|
||||
warn() {
|
||||
if [[ "$STRICT" == true ]]; then
|
||||
echo -e "${YELLOW}✗ WARNING (treated as error in strict mode):${NC} $1"
|
||||
((ERRORS++)) || true
|
||||
else
|
||||
echo -e "${YELLOW}⚠ WARNING:${NC} $1"
|
||||
((WARNINGS++)) || true
|
||||
fi
|
||||
}
|
||||
|
||||
info() {
|
||||
if [[ "$VERBOSE" == true ]]; then
|
||||
echo -e "${BLUE}ℹ${NC} $1"
|
||||
fi
|
||||
}
|
||||
|
||||
success() {
|
||||
echo -e "${GREEN}✓${NC} $1"
|
||||
}
|
||||
|
||||
# Parse arguments
|
||||
SKILL_DIR=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--strict)
|
||||
STRICT=true
|
||||
shift
|
||||
;;
|
||||
--lenient)
|
||||
STRICT=false
|
||||
shift
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
-*)
|
||||
echo -e "${RED}Error: Unknown option $1${NC}"
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
SKILL_DIR="$1"
|
||||
else
|
||||
echo -e "${RED}Error: Multiple skill directories provided${NC}"
|
||||
usage
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate skill directory argument
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
echo -e "${RED}Error: Skill directory is required${NC}"
|
||||
usage
|
||||
fi
|
||||
|
||||
# Resolve path
|
||||
SKILL_DIR="$(cd "$SKILL_DIR" 2>/dev/null && pwd)" || {
|
||||
echo -e "${RED}Error: Cannot access directory: $SKILL_DIR${NC}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Get skill name from directory
|
||||
SKILL_NAME="$(basename "$SKILL_DIR")"
|
||||
|
||||
echo "========================================"
|
||||
echo "Validating skill: $SKILL_NAME"
|
||||
echo "Directory: $SKILL_DIR"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Check 1: Directory exists
|
||||
if [[ ! -d "$SKILL_DIR" ]]; then
|
||||
error "Directory does not exist: $SKILL_DIR"
|
||||
exit 1
|
||||
fi
|
||||
success "Directory exists"
|
||||
|
||||
# Check 2: SKILL.md exists
|
||||
SKILL_MD="$SKILL_DIR/SKILL.md"
|
||||
if [[ ! -f "$SKILL_MD" ]]; then
|
||||
error "SKILL.md file not found in $SKILL_DIR"
|
||||
error "Skill must contain a SKILL.md file"
|
||||
exit 1
|
||||
fi
|
||||
success "SKILL.md file exists"
|
||||
|
||||
# Check 3: SKILL.md starts with YAML frontmatter
|
||||
if ! head -1 "$SKILL_MD" | grep -q '^---\s*$'; then
|
||||
error "SKILL.md must start with YAML frontmatter (---)"
|
||||
else
|
||||
success "SKILL.md starts with YAML frontmatter"
|
||||
fi
|
||||
|
||||
# Extract frontmatter content
|
||||
FRONTMATTER=$(sed -n '/^---$/,/^---$/p' "$SKILL_MD" | head -n -1 | tail -n +2)
|
||||
info "Extracted frontmatter"
|
||||
|
||||
# Check 4: Name field exists and is valid
|
||||
if ! echo "$FRONTMATTER" | grep -q '^name:'; then
|
||||
error "Frontmatter missing required field: 'name'"
|
||||
else
|
||||
FM_NAME=$(echo "$FRONTMATTER" | grep '^name:' | sed 's/^name:\s*//' | tr -d '"' | tr -d "'")
|
||||
|
||||
if [[ -z "$FM_NAME" ]]; then
|
||||
error "'name' field is empty"
|
||||
elif [[ ! "$FM_NAME" =~ ^[a-z0-9-]+$ ]]; then
|
||||
error "'name' must be lowercase alphanumeric with hyphens: '$FM_NAME'"
|
||||
elif [[ "$FM_NAME" != "$SKILL_NAME" ]]; then
|
||||
error "'name' ($FM_NAME) does not match directory name ($SKILL_NAME)"
|
||||
else
|
||||
success "'name' field is valid: $FM_NAME"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check 5: Description field exists and is valid
|
||||
if ! echo "$FRONTMATTER" | grep -q '^description:'; then
|
||||
error "Frontmatter missing required field: 'description'"
|
||||
else
|
||||
FM_DESC=$(echo "$FRONTMATTER" | sed -n '/^description:/,/^\S/{/^description:/p;/^\S/q}' | sed 's/^description:\s*//' | tr -d '"' | tr -d "'" | sed ':a;N;$!ba;s/\n/ /g')
|
||||
DESC_LEN=${#FM_DESC}
|
||||
|
||||
if [[ -z "$FM_DESC" ]]; then
|
||||
error "'description' field is empty"
|
||||
elif [[ $DESC_LEN -lt 20 ]]; then
|
||||
error "'description' must be at least 20 characters (found $DESC_LEN)"
|
||||
elif [[ $DESC_LEN -gt 1024 ]]; then
|
||||
error "'description' must be at most 1024 characters (found $DESC_LEN)"
|
||||
else
|
||||
success "'description' field length is valid ($DESC_LEN chars)"
|
||||
fi
|
||||
|
||||
# Check for common description issues
|
||||
if echo "$FM_DESC" | grep -qi '^use this skill'; then
|
||||
warn "Description uses second person ('Use this skill'). Use third person: 'This skill should be used when...'"
|
||||
fi
|
||||
|
||||
if echo "$FM_DESC" | grep -qi '^this skill provides\|^this skill helps'; then
|
||||
warn "Description is vague. Include specific trigger phrases users would say"
|
||||
fi
|
||||
|
||||
# Check for quoted phrases (allow either single or double quotes in original)
|
||||
if ! echo "$FRONTMATTER" | grep -A 5 '^description:' | grep -q '"\|\"'; then
|
||||
warn "Description should include specific trigger phrases in quotes, e.g.: \"create X\", \"configure Y\""
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check 6: No unknown frontmatter fields (informational)
|
||||
KNOWN_FIELDS="^name:\|^description:\|^license:\|^compatibility:\|^metadata:\|^allowed-tools:"
|
||||
UNKNOWN_FIELDS=$(echo "$FRONTMATTER" | grep -v '^\s*$' | grep -v "$KNOWN_FIELDS" || true)
|
||||
if [[ -n "$UNKNOWN_FIELDS" ]]; then
|
||||
info "Unknown frontmatter fields (will be ignored):"
|
||||
echo "$UNKNOWN_FIELDS" | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
# Check 7: YAML syntax is valid
|
||||
if ! echo "$FRONTMATTER" | python3 -c "import yaml, sys; yaml.safe_load(sys.stdin)" 2>/dev/null; then
|
||||
# Fallback: check with basic pattern matching
|
||||
if echo "$FRONTMATTER" | grep -q '^\s*:\s*$'; then
|
||||
error "Invalid YAML syntax: empty key found"
|
||||
fi
|
||||
|
||||
# Count lines with colons to detect malformed entries
|
||||
MALFORMED=$(echo "$FRONTMATTER" | grep -c '^\s*[^:]*:\s*$' || true)
|
||||
if [[ $MALFORMED -gt 0 ]]; then
|
||||
error "Potential YAML syntax issues detected"
|
||||
fi
|
||||
else
|
||||
success "YAML frontmatter syntax appears valid"
|
||||
fi
|
||||
|
||||
# Check 8: SKILL.md has content after frontmatter
|
||||
# Remove everything from line 1 through the second occurrence of ---
|
||||
BODY_CONTENT=$(sed '0,/^---$/d;0,/^---$/d' "$SKILL_MD")
|
||||
if [[ -z "$(echo "$BODY_CONTENT" | tr -d '[:space:]')" ]]; then
|
||||
error "SKILL.md has no content after frontmatter"
|
||||
else
|
||||
success "SKILL.md has body content"
|
||||
fi
|
||||
|
||||
# Check 9: Check for common writing style issues in body
|
||||
if echo "$BODY_CONTENT" | grep -qE '^You (should|need|can|must)'; then
|
||||
warn "Body uses second person ('You should...'). Prefer imperative form: 'Do this...'"
|
||||
fi
|
||||
|
||||
if echo "$BODY_CONTENT" | grep -q '^When to Use This Skill'; then
|
||||
warn "'When to Use This Skill' section in body is redundant. Include triggers in description field instead"
|
||||
fi
|
||||
|
||||
# Check 10: Verify referenced files exist
|
||||
if [[ -d "$SKILL_DIR/references" ]]; then
|
||||
REF_COUNT=$(find "$SKILL_DIR/references" -type f | wc -l)
|
||||
success "References directory exists with $REF_COUNT file(s)"
|
||||
|
||||
# Check if references are mentioned in SKILL.md
|
||||
for ref_file in "$SKILL_DIR/references"/*; do
|
||||
if [[ -f "$ref_file" ]]; then
|
||||
ref_name=$(basename "$ref_file")
|
||||
if ! grep -q "$ref_name" "$SKILL_MD"; then
|
||||
warn "Reference file '$ref_name' is not mentioned in SKILL.md"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -d "$SKILL_DIR/scripts" ]]; then
|
||||
SCRIPT_COUNT=$(find "$SKILL_DIR/scripts" -type f | wc -l)
|
||||
success "Scripts directory exists with $SCRIPT_COUNT file(s)"
|
||||
|
||||
# Check scripts are executable
|
||||
for script in "$SKILL_DIR/scripts"/*; do
|
||||
if [[ -f "$script" ]] && [[ ! -x "$script" ]]; then
|
||||
warn "Script '$(basename "$script")' is not executable (run: chmod +x)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -d "$SKILL_DIR/assets" ]]; then
|
||||
ASSET_COUNT=$(find "$SKILL_DIR/assets" -type f | wc -l)
|
||||
success "Assets directory exists with $ASSET_COUNT file(s)"
|
||||
fi
|
||||
|
||||
# Check 11: File organization
|
||||
if [[ -f "$SKILL_DIR/README.md" ]]; then
|
||||
warn "README.md found. Skills should not include README.md files"
|
||||
fi
|
||||
|
||||
if [[ -f "$SKILL_DIR/CHANGELOG.md" ]]; then
|
||||
warn "CHANGELOG.md found. Skills should not include auxiliary documentation"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
|
||||
# Final report
|
||||
if [[ $ERRORS -eq 0 ]]; then
|
||||
echo -e "${GREEN}✓ VALIDATION PASSED${NC}"
|
||||
if [[ $WARNINGS -gt 0 ]]; then
|
||||
echo -e "${YELLOW} $WARNINGS warning(s) (not treated as errors)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
echo "Your skill is ready to use!"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}✗ VALIDATION FAILED${NC}"
|
||||
echo -e "${RED} $ERRORS error(s) found${NC}"
|
||||
if [[ $WARNINGS -gt 0 ]]; then
|
||||
echo -e "${YELLOW} $WARNINGS warning(s)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
echo "Fix the errors above and run validation again."
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user