Add skills
This commit is contained in:
179
.config/opencode/skills/.skill-builder.disabled/scripts/create-tests.sh
Executable file
179
.config/opencode/skills/.skill-builder.disabled/scripts/create-tests.sh
Executable file
@@ -0,0 +1,179 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# create-tests.sh - Interactive test case generator for skills
|
||||
#
|
||||
# Usage: ./create-tests.sh <path/to/skill-directory>
|
||||
#
|
||||
# This script interactively generates evals/evals.json with 3 template test cases.
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Parse arguments
|
||||
SKILL_DIR=""
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <path/to/skill-directory>"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 ~/.config/opencode/skills/my-skill"
|
||||
echo " $0 ./my-skill"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
-*)
|
||||
echo -e "${RED}Error: Unknown option $1${NC}"
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
SKILL_DIR="$1"
|
||||
else
|
||||
echo -e "${RED}Error: Multiple skill directories provided${NC}"
|
||||
usage
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate skill directory
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
echo -e "${RED}Error: Skill directory is required${NC}"
|
||||
usage
|
||||
fi
|
||||
|
||||
# Resolve path
|
||||
SKILL_DIR="$(cd "$SKILL_DIR" 2>/dev/null && pwd)" || {
|
||||
echo -e "${RED}Error: Cannot access directory: $SKILL_DIR${NC}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Get skill name from directory
|
||||
SKILL_NAME="$(basename "$SKILL_DIR")"
|
||||
|
||||
# Verify SKILL.md exists
|
||||
if [[ ! -f "$SKILL_DIR/SKILL.md" ]]; then
|
||||
echo -e "${RED}Error: SKILL.md not found in $SKILL_DIR${NC}"
|
||||
echo "This does not appear to be a valid skill directory."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "========================================"
|
||||
echo "Creating Test Cases for: $SKILL_NAME"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Create evals directory
|
||||
mkdir -p "$SKILL_DIR/evals"
|
||||
echo -e "${GREEN}✓ Created evals/ directory${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if evals.json already exists
|
||||
if [[ -f "$SKILL_DIR/evals/evals.json" ]]; then
|
||||
echo -e "${YELLOW}⚠ evals.json already exists${NC}"
|
||||
read -p "Overwrite? (y/N): " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "Aborted."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Get skill purpose from user
|
||||
echo "Let's create 3 test cases for your skill."
|
||||
echo ""
|
||||
echo "First, tell me about your skill's purpose:"
|
||||
echo " (e.g., 'Helps users convert CSV files to JSON format')"
|
||||
read -r SKILL_PURPOSE
|
||||
|
||||
if [[ -z "$SKILL_PURPOSE" ]]; then
|
||||
SKILL_PURPOSE="[Describe what your skill does]"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Now I'll generate 3 test case templates."
|
||||
echo "You can edit evals/evals.json later to customize them."
|
||||
echo ""
|
||||
|
||||
# Generate evals.json
|
||||
cat > "$SKILL_DIR/evals/evals.json" << EVALSJSON
|
||||
{
|
||||
"skill_name": "$SKILL_NAME",
|
||||
"description": "$SKILL_PURPOSE",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"type": "common",
|
||||
"prompt": "[REPLACE WITH REALISTIC USER REQUEST]\n\nExample: Convert the CSV file at ./data/sales.csv to JSON format and save it as ./output/sales.json. Make sure dates are in ISO 8601 format.",
|
||||
"expected_output": "[DESCRIBE EXPECTED RESULT]\n\nExample: A JSON file at ./output/sales.json containing the sales data from the CSV, with all dates converted to ISO 8601 format.",
|
||||
"assertions": [
|
||||
"Output file exists at the specified location",
|
||||
"File is valid JSON",
|
||||
"Dates are in ISO 8601 format"
|
||||
],
|
||||
"notes": "This tests the typical, straightforward use case."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case",
|
||||
"type": "edge",
|
||||
"prompt": "[REPLACE WITH EDGE CASE SCENARIO]\n\nExample: Convert a CSV file with 100,000 rows, some containing special characters (emojis, Unicode) and empty values in certain columns. The file is at ./data/large_export.csv.",
|
||||
"expected_output": "[DESCRIBE EXPECTED BEHAVIOR]\n\nExample: JSON file is created successfully, handling all special characters correctly and preserving empty values as null or empty strings.",
|
||||
"assertions": [
|
||||
"Large file is processed without errors",
|
||||
"Special characters are preserved correctly",
|
||||
"Empty values are handled appropriately"
|
||||
],
|
||||
"notes": "This tests edge cases like large files, special characters, or missing data."
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing",
|
||||
"type": "variation",
|
||||
"prompt": "[REPLACE WITH SAME INTENT, DIFFERENT WORDS]\n\nExample: Hey, I've got this spreadsheet in data/sales.csv. Can you turn it into JSON for me? And make sure the dates look right - you know, standard format?",
|
||||
"expected_output": "[SAME AS COMMON CASE]",
|
||||
"assertions": [
|
||||
"Skill triggers correctly with casual language",
|
||||
"Output matches common case results",
|
||||
"Implicit requirements (date formatting) are handled"
|
||||
],
|
||||
"notes": "This tests that the skill works with different phrasings and casual language."
|
||||
}
|
||||
]
|
||||
}
|
||||
EVALSJSON
|
||||
|
||||
echo -e "${GREEN}✓ Created evals/evals.json${NC}"
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Next Steps:"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "1. Edit evals/evals.json"
|
||||
echo " Replace the [PLACEHOLDER] text with realistic test cases"
|
||||
echo ""
|
||||
echo "2. Tips for writing good test prompts:"
|
||||
echo " - Include file paths (e.g., ./data/file.csv)"
|
||||
echo " - Add personal context (e.g., 'my boss sent me')"
|
||||
echo " - Use specific values and column names"
|
||||
echo " - Mix formal and casual language"
|
||||
echo ""
|
||||
echo "3. Run tests when ready:"
|
||||
echo " ~/.config/opencode/skills/skill-builder/scripts/run-tests.sh $SKILL_DIR"
|
||||
echo ""
|
||||
echo -e "${GREEN}Done!${NC}"
|
||||
363
.config/opencode/skills/.skill-builder.disabled/scripts/grade-output.sh
Executable file
363
.config/opencode/skills/.skill-builder.disabled/scripts/grade-output.sh
Executable file
@@ -0,0 +1,363 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# grade-output.sh - Interactive grading checklist for skill outputs
|
||||
#
|
||||
# Usage: ./grade-output.sh <path/to/skill-directory>
|
||||
#
|
||||
# This script provides a structured checklist for evaluating skill test outputs.
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Parse arguments
|
||||
SKILL_DIR=""
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <path/to/skill-directory>"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 ~/.config/opencode/skills/my-skill"
|
||||
echo " $0 ./my-skill"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
-*)
|
||||
echo -e "${RED}Error: Unknown option $1${NC}"
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
SKILL_DIR="$1"
|
||||
else
|
||||
echo -e "${RED}Error: Multiple skill directories provided${NC}"
|
||||
usage
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate skill directory
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
echo -e "${RED}Error: Skill directory is required${NC}"
|
||||
usage
|
||||
fi
|
||||
|
||||
# Resolve path
|
||||
SKILL_DIR="$(cd "$SKILL_DIR" 2>/dev/null && pwd)" || {
|
||||
echo -e "${RED}Error: Cannot access directory: $SKILL_DIR${NC}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Get skill name from directory
|
||||
SKILL_NAME="$(basename "$SKILL_DIR")"
|
||||
|
||||
echo "========================================"
|
||||
echo "Grading Output for: $SKILL_NAME"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Check for test results
|
||||
if [[ -f "$SKILL_DIR/evals/test-results.json" ]]; then
|
||||
echo -e "${BLUE}Found previous test results${NC}"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Initialize grading data
|
||||
declare -a CRITERIA=(
|
||||
"Correctness: Output matches expected result"
|
||||
"Correctness: No factual errors"
|
||||
"Correctness: Logic is sound"
|
||||
"Correctness: Edge cases handled appropriately"
|
||||
"Completeness: All requested tasks completed"
|
||||
"Completeness: No steps skipped"
|
||||
"Completeness: Appropriate level of detail"
|
||||
"Completeness: Relevant context included"
|
||||
"Format: Output follows specified format"
|
||||
"Format: Consistent with examples in skill"
|
||||
"Format: Easy to read and understand"
|
||||
"Triggering: Skill activated when appropriate"
|
||||
"Triggering: Did not activate when inappropriate"
|
||||
"Efficiency: No unnecessary steps"
|
||||
"Efficiency: Reasonable response length"
|
||||
"Efficiency: Not overly verbose"
|
||||
)
|
||||
|
||||
GRADES=()
|
||||
ISSUES=()
|
||||
|
||||
# Function to ask yes/no question
|
||||
ask_yes_no() {
|
||||
local prompt="$1"
|
||||
while true; do
|
||||
read -p "$prompt (y/n): " -n 1 -r
|
||||
echo
|
||||
case $REPLY in
|
||||
[Yy])
|
||||
return 0
|
||||
;;
|
||||
[Nn])
|
||||
return 1
|
||||
;;
|
||||
*)
|
||||
echo "Please enter y or n"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
echo "This checklist will help you systematically evaluate the skill output."
|
||||
echo "Answer each question based on the test results you observed."
|
||||
echo ""
|
||||
read -p "Press Enter to begin grading..."
|
||||
echo ""
|
||||
|
||||
# Grade each criterion
|
||||
echo "========================================"
|
||||
echo -e "${CYAN}Grading Criteria${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
for criterion in "${CRITERIA[@]}"; do
|
||||
category="${criterion%%:*}"
|
||||
description="${criterion#*: }"
|
||||
|
||||
echo -e "${BLUE}[$category]${NC} $description"
|
||||
|
||||
if ask_yes_no " Does it meet this criterion"; then
|
||||
GRADES+=("$criterion: PASS")
|
||||
echo -e " ${GREEN}✓ Pass${NC}"
|
||||
else
|
||||
GRADES+=("$criterion: FAIL")
|
||||
echo -e " ${RED}✗ Fail${NC}"
|
||||
|
||||
# Ask for issue description
|
||||
echo " Briefly describe the issue:"
|
||||
read -r issue
|
||||
if [[ -n "$issue" ]]; then
|
||||
ISSUES+=("[$category] $description: $issue")
|
||||
fi
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Overall assessment
|
||||
echo "========================================"
|
||||
echo -e "${CYAN}Overall Assessment${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
echo "Overall Result:"
|
||||
echo " [p] Pass - All or most criteria met"
|
||||
echo " [f] Fail - Significant issues found"
|
||||
echo " [i] Incomplete - Needs more testing"
|
||||
echo ""
|
||||
|
||||
while true; do
|
||||
read -p "Overall result (p/f/i): " -n 1 -r
|
||||
echo
|
||||
case $REPLY in
|
||||
[Pp])
|
||||
OVERALL_RESULT="pass"
|
||||
break
|
||||
;;
|
||||
[Ff])
|
||||
OVERALL_RESULT="fail"
|
||||
break
|
||||
;;
|
||||
[Ii])
|
||||
OVERALL_RESULT="incomplete"
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Please enter p, f, or i"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
# Priority assessment
|
||||
echo "Priority of fixes needed:"
|
||||
echo " [h] High - Critical issues, skill not usable"
|
||||
echo " [m] Medium - Important issues, skill partially works"
|
||||
echo " [l] Low - Minor issues, skill mostly works"
|
||||
echo ""
|
||||
|
||||
while true; do
|
||||
read -p "Priority (h/m/l): " -n 1 -r
|
||||
echo
|
||||
case $REPLY in
|
||||
[Hh])
|
||||
PRIORITY="high"
|
||||
break
|
||||
;;
|
||||
[Mm])
|
||||
PRIORITY="medium"
|
||||
break
|
||||
;;
|
||||
[Ll])
|
||||
PRIORITY="low"
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Please enter h, m, or l"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
# Suggested fixes
|
||||
echo -e "${BLUE}Suggested Improvements (optional):${NC}"
|
||||
echo "Describe what changes would address the issues:"
|
||||
read -r SUGGESTED_FIXES
|
||||
|
||||
echo ""
|
||||
|
||||
# Pattern analysis
|
||||
echo "========================================"
|
||||
echo -e "${CYAN}Pattern Analysis${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
if ask_yes_no "Did the same issue appear in multiple test cases"; then
|
||||
echo "This suggests a systemic problem. Consider:"
|
||||
echo " - Fixing the root cause rather than symptoms"
|
||||
echo " - Adding a helper script for repeated tasks"
|
||||
echo " - Clarifying instructions in SKILL.md"
|
||||
PATTERN="systemic"
|
||||
else
|
||||
echo "Issues appear to be isolated to specific cases."
|
||||
PATTERN="isolated"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Extract to script recommendation
|
||||
if ask_yes_no "Should any repeated work be extracted to a script"; then
|
||||
echo "Consider creating a script in scripts/ directory."
|
||||
EXTRACT_SCRIPT="true"
|
||||
else
|
||||
EXTRACT_SCRIPT="false"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Generate grading report
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# Build issues array
|
||||
ISSUES_JSON="["
|
||||
for i in "${!ISSUES[@]}"; do
|
||||
if [[ $i -gt 0 ]]; then
|
||||
ISSUES_JSON+=","
|
||||
fi
|
||||
ISSUES_JSON+="\"${ISSUES[$i]}\""
|
||||
done
|
||||
ISSUES_JSON+="]"
|
||||
|
||||
# Build grades array
|
||||
GRADES_JSON="["
|
||||
for i in "${!GRADES[@]}"; do
|
||||
if [[ $i -gt 0 ]]; then
|
||||
GRADES_JSON+=","
|
||||
fi
|
||||
GRADES_JSON+="\"${GRADES[$i]}\""
|
||||
done
|
||||
GRADES_JSON+="]"
|
||||
|
||||
# Calculate pass rate
|
||||
TOTAL_CRITERIA=${#CRITERIA[@]}
|
||||
PASSED_COUNT=0
|
||||
for grade in "${GRADES[@]}"; do
|
||||
if [[ "$grade" == *"PASS" ]]; then
|
||||
((PASSED_COUNT++))
|
||||
fi
|
||||
done
|
||||
|
||||
PASS_RATE=$((PASSED_COUNT * 100 / TOTAL_CRITERIA))
|
||||
|
||||
cat > "$SKILL_DIR/evals/grading-report.json" << EOF
|
||||
{
|
||||
"skill_name": "$SKILL_NAME",
|
||||
"timestamp": "$TIMESTAMP",
|
||||
"overall_result": "$OVERALL_RESULT",
|
||||
"priority": "$PRIORITY",
|
||||
"pass_rate": $PASS_RATE,
|
||||
"criteria_passed": $PASSED_COUNT,
|
||||
"criteria_total": $TOTAL_CRITERIA,
|
||||
"pattern_analysis": "$PATTERN",
|
||||
"extract_script_recommended": $EXTRACT_SCRIPT,
|
||||
"detailed_grades": $GRADES_JSON,
|
||||
"issues": $ISSUES_JSON,
|
||||
"suggested_fixes": "$SUGGESTED_FIXES"
|
||||
}
|
||||
EOF
|
||||
|
||||
echo "========================================"
|
||||
echo -e "${GREEN}Grading Report Generated${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Saved to: evals/grading-report.json"
|
||||
echo ""
|
||||
echo "Summary:"
|
||||
echo " Overall: $OVERALL_RESULT"
|
||||
echo " Priority: $PRIORITY"
|
||||
echo " Pass Rate: $PASS_RATE% ($PASSED_COUNT/$TOTAL_CRITERIA criteria)"
|
||||
echo " Pattern: $PATTERN issues"
|
||||
echo ""
|
||||
|
||||
if [[ ${#ISSUES[@]} -gt 0 ]]; then
|
||||
echo -e "${YELLOW}Issues Found:${NC}"
|
||||
for issue in "${ISSUES[@]}"; do
|
||||
echo " - $issue"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Next steps
|
||||
echo "========================================"
|
||||
echo "Next Steps"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
if [[ "$OVERALL_RESULT" == "pass" ]]; then
|
||||
echo -e "${GREEN}✓ Skill is working well!${NC}"
|
||||
echo ""
|
||||
echo "Consider:"
|
||||
echo " - Adding more edge case tests"
|
||||
echo " - Optimizing the description"
|
||||
echo " - Documenting the skill"
|
||||
else
|
||||
echo "To improve the skill:"
|
||||
echo ""
|
||||
echo "1. Review grading-report.json for details"
|
||||
echo "2. Update SKILL.md based on the issues found"
|
||||
echo ""
|
||||
if [[ "$EXTRACT_SCRIPT" == "true" ]]; then
|
||||
echo "3. Create helper scripts for repeated tasks"
|
||||
echo " - Place scripts in scripts/ directory"
|
||||
echo " - Update SKILL.md to reference them"
|
||||
echo ""
|
||||
fi
|
||||
echo "4. Re-run tests to verify improvements:"
|
||||
echo " ~/.config/opencode/skills/skill-builder/scripts/run-tests.sh $SKILL_DIR"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}Done!${NC}"
|
||||
391
.config/opencode/skills/.skill-builder.disabled/scripts/init-skill.sh
Executable file
391
.config/opencode/skills/.skill-builder.disabled/scripts/init-skill.sh
Executable file
@@ -0,0 +1,391 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# init-skill.sh - Scaffold a new skill with proper structure
|
||||
#
|
||||
# Usage: ./init-skill.sh <skill-name> [options]
|
||||
#
|
||||
# Options:
|
||||
# --local Create in current project (.opencode/skills/) instead of global
|
||||
# --with-scripts Include scripts/ directory
|
||||
# --with-refs Include references/ directory
|
||||
# --with-assets Include assets/ directory
|
||||
# --with-tests Include evals/ directory for test cases
|
||||
# --full Include all optional directories
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Get script directory
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SKILL_BUILDER_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
# Parse arguments
|
||||
SKILL_NAME=""
|
||||
LOCAL=false
|
||||
WITH_SCRIPTS=false
|
||||
WITH_REFS=false
|
||||
WITH_ASSETS=false
|
||||
WITH_TESTS=false
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <skill-name> [options]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --local Create in current project (.opencode/skills/)"
|
||||
echo " --with-scripts Include scripts/ directory"
|
||||
echo " --with-refs Include references/ directory"
|
||||
echo " --with-assets Include assets/ directory"
|
||||
echo " --with-tests Include evals/ directory for test cases"
|
||||
echo " --full Include all optional directories"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 my-skill"
|
||||
echo " $0 my-skill --full"
|
||||
echo " $0 my-skill --local --with-scripts"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Validate skill name
|
||||
validate_name() {
|
||||
local name="$1"
|
||||
|
||||
if [[ -z "$name" ]]; then
|
||||
echo -e "${RED}Error: Skill name is required${NC}"
|
||||
usage
|
||||
fi
|
||||
|
||||
if [[ ! "$name" =~ ^[a-z0-9-]+$ ]]; then
|
||||
echo -e "${RED}Error: Skill name must be lowercase alphanumeric with hyphens only${NC}"
|
||||
echo " Valid: my-skill, docker-helper, test-runner"
|
||||
echo " Invalid: My Skill, docker_helper, testRunner"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ${#name} -lt 1 ]]; then
|
||||
echo -e "${RED}Error: Skill name cannot be empty${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ${#name} -gt 64 ]]; then
|
||||
echo -e "${RED}Error: Skill name must be under 64 characters${NC}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--local)
|
||||
LOCAL=true
|
||||
shift
|
||||
;;
|
||||
--with-scripts)
|
||||
WITH_SCRIPTS=true
|
||||
shift
|
||||
;;
|
||||
--with-refs)
|
||||
WITH_REFS=true
|
||||
shift
|
||||
;;
|
||||
--with-assets)
|
||||
WITH_ASSETS=true
|
||||
shift
|
||||
;;
|
||||
--with-tests)
|
||||
WITH_TESTS=true
|
||||
shift
|
||||
;;
|
||||
--full)
|
||||
WITH_SCRIPTS=true
|
||||
WITH_REFS=true
|
||||
WITH_ASSETS=true
|
||||
WITH_TESTS=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
-*)
|
||||
echo -e "${RED}Error: Unknown option $1${NC}"
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$SKILL_NAME" ]]; then
|
||||
SKILL_NAME="$1"
|
||||
else
|
||||
echo -e "${RED}Error: Multiple skill names provided${NC}"
|
||||
usage
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate skill name
|
||||
validate_name "$SKILL_NAME"
|
||||
|
||||
# Determine target directory
|
||||
if [[ "$LOCAL" == true ]]; then
|
||||
# Check if we're in a git repository
|
||||
if ! git rev-parse --git-dir > /dev/null 2>&1; then
|
||||
echo -e "${RED}Error: --local requires being in a git repository${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TARGET_DIR=".opencode/skills/$SKILL_NAME"
|
||||
|
||||
# Check for AGENTS.md in project
|
||||
if [[ -f "AGENTS.md" ]]; then
|
||||
echo -e "${GREEN}✓ Found AGENTS.md in project root${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ No AGENTS.md found in project root${NC}"
|
||||
echo " Consider creating one for project-specific rules"
|
||||
fi
|
||||
else
|
||||
TARGET_DIR="$HOME/.config/opencode/skills/$SKILL_NAME"
|
||||
fi
|
||||
|
||||
# Check if skill already exists
|
||||
if [[ -d "$TARGET_DIR" ]]; then
|
||||
echo -e "${RED}Error: Skill already exists at $TARGET_DIR${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}Creating skill: $SKILL_NAME${NC}"
|
||||
echo " Location: $TARGET_DIR"
|
||||
echo ""
|
||||
|
||||
# Create skill directory
|
||||
mkdir -p "$TARGET_DIR"
|
||||
|
||||
# Create SKILL.md
|
||||
cat > "$TARGET_DIR/SKILL.md" << 'SKILL_TEMPLATE'
|
||||
---
|
||||
name: SKILL_NAME_PLACEHOLDER
|
||||
description: This skill should be used when the user asks to "DESCRIPTION_HERE". Add specific trigger phrases that would activate this skill.
|
||||
license: MIT
|
||||
compatibility: opencode
|
||||
metadata:
|
||||
category: general
|
||||
version: "1.0.0"
|
||||
---
|
||||
|
||||
# SKILL_NAME_PLACEHOLDER
|
||||
|
||||
Brief description of what this skill does and its purpose.
|
||||
|
||||
## What This Skill Provides
|
||||
|
||||
1. **Feature 1** - Brief description
|
||||
2. **Feature 2** - Brief description
|
||||
3. **Feature 3** - Brief description
|
||||
|
||||
## Quick Start
|
||||
|
||||
Basic usage example:
|
||||
|
||||
```bash
|
||||
# Example command
|
||||
some-tool --option value
|
||||
```
|
||||
|
||||
## Common Tasks
|
||||
|
||||
### Task 1: Description
|
||||
|
||||
Step-by-step instructions:
|
||||
|
||||
1. First step
|
||||
2. Second step
|
||||
3. Third step
|
||||
|
||||
### Task 2: Description
|
||||
|
||||
Step-by-step instructions:
|
||||
|
||||
1. First step
|
||||
2. Second step
|
||||
|
||||
## Important Notes
|
||||
|
||||
- Keep this section brief
|
||||
- Use bullet points for clarity
|
||||
- Reference supporting files if available
|
||||
|
||||
## Resources
|
||||
|
||||
SKILL_TEMPLATE
|
||||
|
||||
# Add resource section if directories were created
|
||||
if [[ "$WITH_REFS" == true ]]; then
|
||||
cat >> "$TARGET_DIR/SKILL.md" << 'REF_SECTION'
|
||||
|
||||
### Reference Files
|
||||
|
||||
- `references/detailed-guide.md` - Detailed documentation
|
||||
|
||||
REF_SECTION
|
||||
fi
|
||||
|
||||
if [[ "$WITH_SCRIPTS" == true ]]; then
|
||||
cat >> "$TARGET_DIR/SKILL.md" << 'SCRIPT_SECTION'
|
||||
|
||||
### Scripts
|
||||
|
||||
- `scripts/helper.sh` - Utility script
|
||||
|
||||
SCRIPT_SECTION
|
||||
fi
|
||||
|
||||
if [[ "$WITH_ASSETS" == true ]]; then
|
||||
cat >> "$TARGET_DIR/SKILL.md" << 'ASSET_SECTION'
|
||||
|
||||
### Assets
|
||||
|
||||
- `assets/template.txt` - Template file
|
||||
|
||||
ASSET_SECTION
|
||||
fi
|
||||
|
||||
# Replace placeholder with actual skill name
|
||||
sed -i "s/SKILL_NAME_PLACEHOLDER/$SKILL_NAME/g" "$TARGET_DIR/SKILL.md"
|
||||
|
||||
# Create optional directories
|
||||
if [[ "$WITH_SCRIPTS" == true ]]; then
|
||||
mkdir -p "$TARGET_DIR/scripts"
|
||||
|
||||
# Create example script
|
||||
cat > "$TARGET_DIR/scripts/helper.sh" << 'SCRIPT_EXAMPLE'
|
||||
#!/bin/bash
|
||||
#
|
||||
# Helper script for SKILL_NAME_PLACEHOLDER
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo "Helper script for SKILL_NAME_PLACEHOLDER"
|
||||
echo "Modify this script for your needs"
|
||||
SCRIPT_EXAMPLE
|
||||
|
||||
sed -i "s/SKILL_NAME_PLACEHOLDER/$SKILL_NAME/g" "$TARGET_DIR/scripts/helper.sh"
|
||||
chmod +x "$TARGET_DIR/scripts/helper.sh"
|
||||
|
||||
echo -e "${GREEN} ✓ Created scripts/ directory${NC}"
|
||||
fi
|
||||
|
||||
if [[ "$WITH_REFS" == true ]]; then
|
||||
mkdir -p "$TARGET_DIR/references"
|
||||
|
||||
# Create example reference
|
||||
cat > "$TARGET_DIR/references/detailed-guide.md" << 'REF_EXAMPLE'
|
||||
# Detailed Guide for SKILL_NAME_PLACEHOLDER
|
||||
|
||||
This file contains detailed documentation that can be referenced on-demand.
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
Detailed instructions go here...
|
||||
|
||||
## Configuration
|
||||
|
||||
Configuration options and examples...
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
Common issues and solutions...
|
||||
REF_EXAMPLE
|
||||
|
||||
sed -i "s/SKILL_NAME_PLACEHOLDER/$SKILL_NAME/g" "$TARGET_DIR/references/detailed-guide.md"
|
||||
|
||||
echo -e "${GREEN} ✓ Created references/ directory${NC}"
|
||||
fi
|
||||
|
||||
if [[ "$WITH_ASSETS" == true ]]; then
|
||||
mkdir -p "$TARGET_DIR/assets"
|
||||
|
||||
# Create example asset
|
||||
cat > "$TARGET_DIR/assets/template.txt" << 'ASSET_EXAMPLE'
|
||||
# Template file for SKILL_NAME_PLACEHOLDER
|
||||
|
||||
This is a template file that can be used as a starting point.
|
||||
Modify it according to your needs.
|
||||
ASSET_EXAMPLE
|
||||
|
||||
sed -i "s/SKILL_NAME_PLACEHOLDER/$SKILL_NAME/g" "$TARGET_DIR/assets/template.txt"
|
||||
|
||||
echo -e "${GREEN} ✓ Created assets/ directory${NC}"
|
||||
fi
|
||||
|
||||
if [[ "$WITH_TESTS" == true ]]; then
|
||||
mkdir -p "$TARGET_DIR/evals"
|
||||
|
||||
# Create template evals.json
|
||||
cat > "$TARGET_DIR/evals/evals.json" << 'EVALS_TEMPLATE'
|
||||
{
|
||||
"skill_name": "SKILL_NAME_PLACEHOLDER",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "common-case",
|
||||
"prompt": "Typical user request with realistic context, file paths, and specific details. This represents the most common way users will ask for help.",
|
||||
"expected_output": "Description of what the skill should produce for this request",
|
||||
"assertions": [
|
||||
"Check that output contains specific expected content",
|
||||
"Verify file is created at expected location (if applicable)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "edge-case",
|
||||
"prompt": "Unusual or tricky scenario that tests edge cases, error handling, or complex requirements. Include something that might break the skill.",
|
||||
"expected_output": "Description of expected behavior for this edge case",
|
||||
"assertions": [
|
||||
"Verify edge case is handled appropriately",
|
||||
"Check for graceful error handling if applicable"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "varied-phrasing",
|
||||
"prompt": "Same intent as the common case but expressed with different words, casual language, or alternative phrasing. Tests skill robustness to language variation.",
|
||||
"expected_output": "Same expected output as common case",
|
||||
"assertions": [
|
||||
"Output should match common case results",
|
||||
"Skill triggers correctly with different phrasing"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EVALS_TEMPLATE
|
||||
|
||||
sed -i "s/SKILL_NAME_PLACEHOLDER/$SKILL_NAME/g" "$TARGET_DIR/evals/evals.json"
|
||||
|
||||
echo -e "${GREEN} ✓ Created evals/ directory with template evals.json${NC}"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}✓ Skill '$SKILL_NAME' created successfully!${NC}"
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. Edit $TARGET_DIR/SKILL.md"
|
||||
echo " 2. Fill in the description with specific trigger phrases"
|
||||
echo " 3. Add your skill content"
|
||||
if [[ "$WITH_REFS" == true ]]; then
|
||||
echo " 4. Add detailed content to references/ files"
|
||||
fi
|
||||
if [[ "$WITH_SCRIPTS" == true ]]; then
|
||||
echo " 5. Implement scripts in scripts/ directory"
|
||||
fi
|
||||
if [[ "$WITH_TESTS" == true ]]; then
|
||||
echo " 6. Customize test cases in evals/evals.json"
|
||||
echo " 7. Run tests: ${SKILL_BUILDER_DIR}/scripts/run-tests.sh $TARGET_DIR"
|
||||
fi
|
||||
echo ""
|
||||
echo "Validate your skill:"
|
||||
echo " ${SKILL_BUILDER_DIR}/scripts/validate-skill.sh $TARGET_DIR"
|
||||
325
.config/opencode/skills/.skill-builder.disabled/scripts/run-tests.sh
Executable file
325
.config/opencode/skills/.skill-builder.disabled/scripts/run-tests.sh
Executable file
@@ -0,0 +1,325 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# run-tests.sh - Execute test workflow and capture results
|
||||
#
|
||||
# Usage: ./run-tests.sh <path/to/skill-directory>
|
||||
#
|
||||
# This script guides you through testing each test case manually and records results.
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Parse arguments
|
||||
SKILL_DIR=""
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <path/to/skill-directory>"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 ~/.config/opencode/skills/my-skill"
|
||||
echo " $0 ./my-skill"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
-*)
|
||||
echo -e "${RED}Error: Unknown option $1${NC}"
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
SKILL_DIR="$1"
|
||||
else
|
||||
echo -e "${RED}Error: Multiple skill directories provided${NC}"
|
||||
usage
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate skill directory
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
echo -e "${RED}Error: Skill directory is required${NC}"
|
||||
usage
|
||||
fi
|
||||
|
||||
# Resolve path
|
||||
SKILL_DIR="$(cd "$SKILL_DIR" 2>/dev/null && pwd)" || {
|
||||
echo -e "${RED}Error: Cannot access directory: $SKILL_DIR${NC}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Get skill name from directory
|
||||
SKILL_NAME="$(basename "$SKILL_DIR")"
|
||||
|
||||
# Verify evals.json exists
|
||||
if [[ ! -f "$SKILL_DIR/evals/evals.json" ]]; then
|
||||
echo -e "${RED}Error: evals/evals.json not found${NC}"
|
||||
echo ""
|
||||
echo "Create test cases first:"
|
||||
echo " ~/.config/opencode/skills/skill-builder/scripts/create-tests.sh $SKILL_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "========================================"
|
||||
echo "Running Tests for: $SKILL_NAME"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Check if jq is available
|
||||
if ! command -v jq &> /dev/null; then
|
||||
echo -e "${YELLOW}⚠ jq is not installed${NC}"
|
||||
echo " Install jq for better JSON handling:"
|
||||
echo " Ubuntu/Debian: sudo apt-get install jq"
|
||||
echo " macOS: brew install jq"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Falling back to basic parsing...${NC}"
|
||||
USE_JQ=false
|
||||
else
|
||||
USE_JQ=true
|
||||
fi
|
||||
|
||||
# Initialize results array
|
||||
RESULTS=()
|
||||
TEST_COUNT=0
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
SKIPPED=0
|
||||
|
||||
# Function to extract value from JSON using basic parsing
|
||||
get_json_value() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
local index="${3:-}"
|
||||
|
||||
if [[ "$USE_JQ" == true ]]; then
|
||||
if [[ -n "$index" ]]; then
|
||||
jq -r ".evals[$index].$key" "$file" 2>/dev/null || echo ""
|
||||
else
|
||||
jq -r ".$key" "$file" 2>/dev/null || echo ""
|
||||
fi
|
||||
else
|
||||
# Basic grep-based extraction (fallback)
|
||||
if [[ -n "$index" ]]; then
|
||||
# This is a simplified fallback - won't handle nested structures well
|
||||
grep -A 100 '"evals":' "$file" | grep -A 20 "\"id\": $index" | grep "\"$key\":" | head -1 | sed 's/.*"'$key'": "\(.*\)".*/\1/' | sed 's/",*$//'
|
||||
else
|
||||
grep "\"$key\":" "$file" | head -1 | sed 's/.*"'$key'": "\(.*\)".*/\1/' | sed 's/",*$//'
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Count test cases
|
||||
if [[ "$USE_JQ" == true ]]; then
|
||||
TEST_COUNT=$(jq '.evals | length' "$SKILL_DIR/evals/evals.json")
|
||||
else
|
||||
TEST_COUNT=$(grep -c '"id":' "$SKILL_DIR/evals/evals.json" 2>/dev/null || echo "0")
|
||||
fi
|
||||
|
||||
if [[ "$TEST_COUNT" -eq 0 ]]; then
|
||||
echo -e "${RED}Error: No test cases found in evals.json${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Found $TEST_COUNT test case(s)"
|
||||
echo ""
|
||||
|
||||
# Process each test case
|
||||
for ((i=0; i<TEST_COUNT; i++)); do
|
||||
echo "========================================"
|
||||
echo -e "${CYAN}Test Case $((i+1)) of $TEST_COUNT${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Extract test details
|
||||
if [[ "$USE_JQ" == true ]]; then
|
||||
TEST_ID=$(jq -r ".evals[$i].id" "$SKILL_DIR/evals/evals.json")
|
||||
TEST_NAME=$(jq -r ".evals[$i].name" "$SKILL_DIR/evals/evals.json")
|
||||
TEST_PROMPT=$(jq -r ".evals[$i].prompt" "$SKILL_DIR/evals/evals.json")
|
||||
TEST_EXPECTED=$(jq -r ".evals[$i].expected_output" "$SKILL_DIR/evals/evals.json")
|
||||
TEST_TYPE=$(jq -r ".evals[$i].type // .evals[$i].name" "$SKILL_DIR/evals/evals.json")
|
||||
else
|
||||
# Fallback parsing
|
||||
TEST_ID=$((i+1))
|
||||
TEST_NAME="test-$((i+1))"
|
||||
TEST_PROMPT="[Prompt extraction requires jq - install jq for full functionality]"
|
||||
TEST_EXPECTED="[Expected output extraction requires jq]"
|
||||
TEST_TYPE="unknown"
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}Test Name:${NC} $TEST_NAME"
|
||||
echo -e "${BLUE}Type:${NC} $TEST_TYPE"
|
||||
echo ""
|
||||
echo -e "${BLUE}Prompt:${NC}"
|
||||
echo "----------------------------------------"
|
||||
echo "$TEST_PROMPT"
|
||||
echo "----------------------------------------"
|
||||
echo ""
|
||||
|
||||
echo -e "${BLUE}Expected Output:${NC}"
|
||||
echo "$TEST_EXPECTED"
|
||||
echo ""
|
||||
|
||||
# Instructions for manual testing
|
||||
echo -e "${YELLOW}Instructions:${NC}"
|
||||
echo "1. Copy the prompt above"
|
||||
echo "2. Open a new opencode session"
|
||||
echo "3. Paste the prompt and observe the result"
|
||||
echo "4. Return here to record the outcome"
|
||||
echo ""
|
||||
|
||||
read -p "Press Enter when ready to record results..."
|
||||
echo ""
|
||||
|
||||
# Get test result
|
||||
echo "Test Result:"
|
||||
echo " [p] Pass - Output met expectations"
|
||||
echo " [f] Fail - Output did not meet expectations"
|
||||
echo " [s] Skip - Could not test or not applicable"
|
||||
echo ""
|
||||
|
||||
while true; do
|
||||
read -p "Result (p/f/s): " -n 1 -r
|
||||
echo
|
||||
case $REPLY in
|
||||
[Pp])
|
||||
TEST_RESULT="pass"
|
||||
((PASSED++))
|
||||
break
|
||||
;;
|
||||
[Ff])
|
||||
TEST_RESULT="fail"
|
||||
((FAILED++))
|
||||
break
|
||||
;;
|
||||
[Ss])
|
||||
TEST_RESULT="skip"
|
||||
((SKIPPED++))
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Please enter p, f, or s"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
# Get notes
|
||||
echo -e "${BLUE}Notes (optional):${NC}"
|
||||
echo "Describe any issues, observations, or suggestions:"
|
||||
read -r TEST_NOTES
|
||||
|
||||
# Get timestamp
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# Build result entry
|
||||
RESULT_ENTRY=$(cat << EOF
|
||||
{
|
||||
"test_id": $TEST_ID,
|
||||
"test_name": "$TEST_NAME",
|
||||
"result": "$TEST_RESULT",
|
||||
"notes": "$TEST_NOTES",
|
||||
"timestamp": "$TIMESTAMP"
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
RESULTS+=("$RESULT_ENTRY")
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}✓ Recorded test result${NC}"
|
||||
echo ""
|
||||
|
||||
# Ask to continue or stop
|
||||
if [[ $i -lt $((TEST_COUNT-1)) ]]; then
|
||||
read -p "Continue to next test? (Y/n): " -n 1 -r
|
||||
echo
|
||||
if [[ $REPLY =~ ^[Nn]$ ]]; then
|
||||
echo "Stopping test run..."
|
||||
break
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
|
||||
# Generate test-results.json
|
||||
echo "========================================"
|
||||
echo "Generating Test Results"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Create results JSON
|
||||
RUN_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# Build JSON array from results
|
||||
RESULTS_JSON="["
|
||||
for i in "${!RESULTS[@]}"; do
|
||||
if [[ $i -gt 0 ]]; then
|
||||
RESULTS_JSON+=","
|
||||
fi
|
||||
RESULTS_JSON+="${RESULTS[$i]}"
|
||||
done
|
||||
RESULTS_JSON+="]"
|
||||
|
||||
cat > "$SKILL_DIR/evals/test-results.json" << EOF
|
||||
{
|
||||
"skill_name": "$SKILL_NAME",
|
||||
"run_timestamp": "$RUN_TIMESTAMP",
|
||||
"summary": {
|
||||
"total": $((${#RESULTS[@]})),
|
||||
"passed": $PASSED,
|
||||
"failed": $FAILED,
|
||||
"skipped": $SKIPPED
|
||||
},
|
||||
"results": $RESULTS_JSON
|
||||
}
|
||||
EOF
|
||||
|
||||
echo -e "${GREEN}✓ Saved results to evals/test-results.json${NC}"
|
||||
echo ""
|
||||
|
||||
# Display summary
|
||||
echo "========================================"
|
||||
echo "Test Run Summary"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Total Tests: $((${#RESULTS[@]}))"
|
||||
echo -e "${GREEN}Passed: $PASSED${NC}"
|
||||
echo -e "${RED}Failed: $FAILED${NC}"
|
||||
echo -e "${YELLOW}Skipped: $SKIPPED${NC}"
|
||||
echo ""
|
||||
|
||||
# Next steps
|
||||
if [[ $FAILED -gt 0 ]]; then
|
||||
echo "========================================"
|
||||
echo -e "${YELLOW}Next Steps:${NC}"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Some tests failed. To improve your skill:"
|
||||
echo ""
|
||||
echo "1. Review test-results.json for details"
|
||||
echo "2. Update SKILL.md to address issues"
|
||||
echo "3. Run tests again:"
|
||||
echo " $0 $SKILL_DIR"
|
||||
echo ""
|
||||
echo "For detailed grading:"
|
||||
echo " ~/.config/opencode/skills/skill-builder/scripts/grade-output.sh $SKILL_DIR"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}Done!${NC}"
|
||||
308
.config/opencode/skills/.skill-builder.disabled/scripts/validate-skill.sh
Executable file
308
.config/opencode/skills/.skill-builder.disabled/scripts/validate-skill.sh
Executable file
@@ -0,0 +1,308 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# validate-skill.sh - Strictly validate a skill's structure and content
|
||||
#
|
||||
# Usage: ./validate-skill.sh <path/to/skill-directory> [options]
|
||||
#
|
||||
# Options:
|
||||
# --strict Fail on warnings (default)
|
||||
# --lenient Only fail on errors, report warnings
|
||||
# --verbose Show detailed output
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Validation counters
|
||||
ERRORS=0
|
||||
WARNINGS=0
|
||||
|
||||
# Default mode
|
||||
STRICT=true
|
||||
VERBOSE=false
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <path/to/skill-directory> [options]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --strict Fail on warnings (default)"
|
||||
echo " --lenient Only fail on errors"
|
||||
echo " --verbose Show detailed output"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 ~/.config/opencode/skills/my-skill"
|
||||
echo " $0 ./my-skill --verbose"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Error and warning functions
|
||||
error() {
|
||||
echo -e "${RED}✗ ERROR:${NC} $1"
|
||||
((ERRORS++)) || true
|
||||
}
|
||||
|
||||
warn() {
|
||||
if [[ "$STRICT" == true ]]; then
|
||||
echo -e "${YELLOW}✗ WARNING (treated as error in strict mode):${NC} $1"
|
||||
((ERRORS++)) || true
|
||||
else
|
||||
echo -e "${YELLOW}⚠ WARNING:${NC} $1"
|
||||
((WARNINGS++)) || true
|
||||
fi
|
||||
}
|
||||
|
||||
info() {
|
||||
if [[ "$VERBOSE" == true ]]; then
|
||||
echo -e "${BLUE}ℹ${NC} $1"
|
||||
fi
|
||||
}
|
||||
|
||||
success() {
|
||||
echo -e "${GREEN}✓${NC} $1"
|
||||
}
|
||||
|
||||
# Parse arguments
|
||||
SKILL_DIR=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--strict)
|
||||
STRICT=true
|
||||
shift
|
||||
;;
|
||||
--lenient)
|
||||
STRICT=false
|
||||
shift
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
-*)
|
||||
echo -e "${RED}Error: Unknown option $1${NC}"
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
SKILL_DIR="$1"
|
||||
else
|
||||
echo -e "${RED}Error: Multiple skill directories provided${NC}"
|
||||
usage
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate skill directory argument
|
||||
if [[ -z "$SKILL_DIR" ]]; then
|
||||
echo -e "${RED}Error: Skill directory is required${NC}"
|
||||
usage
|
||||
fi
|
||||
|
||||
# Resolve path
|
||||
SKILL_DIR="$(cd "$SKILL_DIR" 2>/dev/null && pwd)" || {
|
||||
echo -e "${RED}Error: Cannot access directory: $SKILL_DIR${NC}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Get skill name from directory
|
||||
SKILL_NAME="$(basename "$SKILL_DIR")"
|
||||
|
||||
echo "========================================"
|
||||
echo "Validating skill: $SKILL_NAME"
|
||||
echo "Directory: $SKILL_DIR"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Check 1: Directory exists
|
||||
if [[ ! -d "$SKILL_DIR" ]]; then
|
||||
error "Directory does not exist: $SKILL_DIR"
|
||||
exit 1
|
||||
fi
|
||||
success "Directory exists"
|
||||
|
||||
# Check 2: SKILL.md exists
|
||||
SKILL_MD="$SKILL_DIR/SKILL.md"
|
||||
if [[ ! -f "$SKILL_MD" ]]; then
|
||||
error "SKILL.md file not found in $SKILL_DIR"
|
||||
error "Skill must contain a SKILL.md file"
|
||||
exit 1
|
||||
fi
|
||||
success "SKILL.md file exists"
|
||||
|
||||
# Check 3: SKILL.md starts with YAML frontmatter
|
||||
if ! head -1 "$SKILL_MD" | grep -q '^---\s*$'; then
|
||||
error "SKILL.md must start with YAML frontmatter (---)"
|
||||
else
|
||||
success "SKILL.md starts with YAML frontmatter"
|
||||
fi
|
||||
|
||||
# Extract frontmatter content
|
||||
FRONTMATTER=$(sed -n '/^---$/,/^---$/p' "$SKILL_MD" | head -n -1 | tail -n +2)
|
||||
info "Extracted frontmatter"
|
||||
|
||||
# Check 4: Name field exists and is valid
|
||||
if ! echo "$FRONTMATTER" | grep -q '^name:'; then
|
||||
error "Frontmatter missing required field: 'name'"
|
||||
else
|
||||
FM_NAME=$(echo "$FRONTMATTER" | grep '^name:' | sed 's/^name:\s*//' | tr -d '"' | tr -d "'")
|
||||
|
||||
if [[ -z "$FM_NAME" ]]; then
|
||||
error "'name' field is empty"
|
||||
elif [[ ! "$FM_NAME" =~ ^[a-z0-9-]+$ ]]; then
|
||||
error "'name' must be lowercase alphanumeric with hyphens: '$FM_NAME'"
|
||||
elif [[ "$FM_NAME" != "$SKILL_NAME" ]]; then
|
||||
error "'name' ($FM_NAME) does not match directory name ($SKILL_NAME)"
|
||||
else
|
||||
success "'name' field is valid: $FM_NAME"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check 5: Description field exists and is valid
|
||||
if ! echo "$FRONTMATTER" | grep -q '^description:'; then
|
||||
error "Frontmatter missing required field: 'description'"
|
||||
else
|
||||
FM_DESC=$(echo "$FRONTMATTER" | sed -n '/^description:/,/^\S/{/^description:/p;/^\S/q}' | sed 's/^description:\s*//' | tr -d '"' | tr -d "'" | sed ':a;N;$!ba;s/\n/ /g')
|
||||
DESC_LEN=${#FM_DESC}
|
||||
|
||||
if [[ -z "$FM_DESC" ]]; then
|
||||
error "'description' field is empty"
|
||||
elif [[ $DESC_LEN -lt 20 ]]; then
|
||||
error "'description' must be at least 20 characters (found $DESC_LEN)"
|
||||
elif [[ $DESC_LEN -gt 1024 ]]; then
|
||||
error "'description' must be at most 1024 characters (found $DESC_LEN)"
|
||||
else
|
||||
success "'description' field length is valid ($DESC_LEN chars)"
|
||||
fi
|
||||
|
||||
# Check for common description issues
|
||||
if echo "$FM_DESC" | grep -qi '^use this skill'; then
|
||||
warn "Description uses second person ('Use this skill'). Use third person: 'This skill should be used when...'"
|
||||
fi
|
||||
|
||||
if echo "$FM_DESC" | grep -qi '^this skill provides\|^this skill helps'; then
|
||||
warn "Description is vague. Include specific trigger phrases users would say"
|
||||
fi
|
||||
|
||||
# Check for quoted phrases (allow either single or double quotes in original)
|
||||
if ! echo "$FRONTMATTER" | grep -A 5 '^description:' | grep -q '"\|\"'; then
|
||||
warn "Description should include specific trigger phrases in quotes, e.g.: \"create X\", \"configure Y\""
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check 6: No unknown frontmatter fields (informational)
|
||||
KNOWN_FIELDS="^name:\|^description:\|^license:\|^compatibility:\|^metadata:\|^allowed-tools:"
|
||||
UNKNOWN_FIELDS=$(echo "$FRONTMATTER" | grep -v '^\s*$' | grep -v "$KNOWN_FIELDS" || true)
|
||||
if [[ -n "$UNKNOWN_FIELDS" ]]; then
|
||||
info "Unknown frontmatter fields (will be ignored):"
|
||||
echo "$UNKNOWN_FIELDS" | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
# Check 7: YAML syntax is valid
|
||||
if ! echo "$FRONTMATTER" | python3 -c "import yaml, sys; yaml.safe_load(sys.stdin)" 2>/dev/null; then
|
||||
# Fallback: check with basic pattern matching
|
||||
if echo "$FRONTMATTER" | grep -q '^\s*:\s*$'; then
|
||||
error "Invalid YAML syntax: empty key found"
|
||||
fi
|
||||
|
||||
# Count lines with colons to detect malformed entries
|
||||
MALFORMED=$(echo "$FRONTMATTER" | grep -c '^\s*[^:]*:\s*$' || true)
|
||||
if [[ $MALFORMED -gt 0 ]]; then
|
||||
error "Potential YAML syntax issues detected"
|
||||
fi
|
||||
else
|
||||
success "YAML frontmatter syntax appears valid"
|
||||
fi
|
||||
|
||||
# Check 8: SKILL.md has content after frontmatter
|
||||
# Remove everything from line 1 through the second occurrence of ---
|
||||
BODY_CONTENT=$(sed '0,/^---$/d;0,/^---$/d' "$SKILL_MD")
|
||||
if [[ -z "$(echo "$BODY_CONTENT" | tr -d '[:space:]')" ]]; then
|
||||
error "SKILL.md has no content after frontmatter"
|
||||
else
|
||||
success "SKILL.md has body content"
|
||||
fi
|
||||
|
||||
# Check 9: Check for common writing style issues in body
|
||||
if echo "$BODY_CONTENT" | grep -qE '^You (should|need|can|must)'; then
|
||||
warn "Body uses second person ('You should...'). Prefer imperative form: 'Do this...'"
|
||||
fi
|
||||
|
||||
if echo "$BODY_CONTENT" | grep -q '^When to Use This Skill'; then
|
||||
warn "'When to Use This Skill' section in body is redundant. Include triggers in description field instead"
|
||||
fi
|
||||
|
||||
# Check 10: Verify referenced files exist
|
||||
if [[ -d "$SKILL_DIR/references" ]]; then
|
||||
REF_COUNT=$(find "$SKILL_DIR/references" -type f | wc -l)
|
||||
success "References directory exists with $REF_COUNT file(s)"
|
||||
|
||||
# Check if references are mentioned in SKILL.md
|
||||
for ref_file in "$SKILL_DIR/references"/*; do
|
||||
if [[ -f "$ref_file" ]]; then
|
||||
ref_name=$(basename "$ref_file")
|
||||
if ! grep -q "$ref_name" "$SKILL_MD"; then
|
||||
warn "Reference file '$ref_name' is not mentioned in SKILL.md"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -d "$SKILL_DIR/scripts" ]]; then
|
||||
SCRIPT_COUNT=$(find "$SKILL_DIR/scripts" -type f | wc -l)
|
||||
success "Scripts directory exists with $SCRIPT_COUNT file(s)"
|
||||
|
||||
# Check scripts are executable
|
||||
for script in "$SKILL_DIR/scripts"/*; do
|
||||
if [[ -f "$script" ]] && [[ ! -x "$script" ]]; then
|
||||
warn "Script '$(basename "$script")' is not executable (run: chmod +x)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -d "$SKILL_DIR/assets" ]]; then
|
||||
ASSET_COUNT=$(find "$SKILL_DIR/assets" -type f | wc -l)
|
||||
success "Assets directory exists with $ASSET_COUNT file(s)"
|
||||
fi
|
||||
|
||||
# Check 11: File organization
|
||||
if [[ -f "$SKILL_DIR/README.md" ]]; then
|
||||
warn "README.md found. Skills should not include README.md files"
|
||||
fi
|
||||
|
||||
if [[ -f "$SKILL_DIR/CHANGELOG.md" ]]; then
|
||||
warn "CHANGELOG.md found. Skills should not include auxiliary documentation"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
|
||||
# Final report
|
||||
if [[ $ERRORS -eq 0 ]]; then
|
||||
echo -e "${GREEN}✓ VALIDATION PASSED${NC}"
|
||||
if [[ $WARNINGS -gt 0 ]]; then
|
||||
echo -e "${YELLOW} $WARNINGS warning(s) (not treated as errors)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
echo "Your skill is ready to use!"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}✗ VALIDATION FAILED${NC}"
|
||||
echo -e "${RED} $ERRORS error(s) found${NC}"
|
||||
if [[ $WARNINGS -gt 0 ]]; then
|
||||
echo -e "${YELLOW} $WARNINGS warning(s)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
echo "Fix the errors above and run validation again."
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user