mirror of
https://github.com/bigbeartechworld/big-bear-scripts.git
synced 2026-08-01 17:58:48 -04:00
Add allowlist support with ranges and comments (#70)
* Add allowlist support with ranges and comments Introduces enhanced allowlist parsing in the Unicode Security Scanner, supporting single codes, code ranges, and inline comments. Updates documentation and test suite to verify allowlist functionality, including range and inline comment handling. Also adds a GitHub Actions workflow for automated scanner testing on Ubuntu and macOS. * Simplify test command in CI workflow Refactored the test step to use 'if ./run.sh ...' directly, removing the explicit check of the exit code for improved readability. * Unify test workflow for multiple OS platforms Replaces separate macOS and Ubuntu jobs with a matrix strategy to run tests on both platforms. Adds shell defaults and improves test steps with more robust output validation.
This commit is contained in:
Christopher
GitHub
98
.github/workflows/test-unicode-scanner.yml
vendored
Normal file
98
.github/workflows/test-unicode-scanner.yml
vendored
Normal file
@@ -0,0 +1,98 @@
|
||||
name: Test Unicode Security Scanner
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'check-for-unicode/**'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'check-for-unicode/**'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
test-scanner:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Make scripts executable
|
||||
run: |
|
||||
chmod +x check-for-unicode/run.sh
|
||||
chmod +x check-for-unicode/test-suite/run-tests.sh
|
||||
|
||||
- name: Display scanner version
|
||||
working-directory: check-for-unicode
|
||||
run: ./run.sh --version
|
||||
|
||||
- name: Run test suite
|
||||
working-directory: check-for-unicode/test-suite
|
||||
run: bash run-tests.sh
|
||||
|
||||
- name: Test scanner help
|
||||
working-directory: check-for-unicode
|
||||
run: |
|
||||
output=$(./run.sh --help)
|
||||
if echo "$output" | grep -q "ALLOWLIST FORMAT"; then
|
||||
echo "✓ Help includes allowlist documentation"
|
||||
else
|
||||
echo "✗ Help missing allowlist documentation"
|
||||
exit 1
|
||||
fi
|
||||
if echo "$output" | grep -q "exclude-emojis"; then
|
||||
echo "✓ Help includes emoji exclusion flag"
|
||||
else
|
||||
echo "✗ Help missing emoji exclusion flag"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Test scanning a clean file
|
||||
working-directory: check-for-unicode
|
||||
run: |
|
||||
if ./run.sh test-suite/clean-test.js; then
|
||||
echo "✓ Clean file test passed"
|
||||
else
|
||||
echo "✗ Clean file test failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Test detection of malicious file
|
||||
working-directory: check-for-unicode
|
||||
run: |
|
||||
if ./run.sh test-suite/trojan-source-test.js; then
|
||||
echo "✗ Malicious file was not detected"
|
||||
exit 1
|
||||
else
|
||||
echo "✓ Malicious file correctly detected"
|
||||
fi
|
||||
|
||||
- name: Test JSON output
|
||||
working-directory: check-for-unicode
|
||||
run: |
|
||||
output=$(./run.sh --json test-suite/clean-test.js)
|
||||
echo "$output" | jq .
|
||||
if echo "$output" | jq -e '.scanner' > /dev/null; then
|
||||
echo "✓ JSON output test passed"
|
||||
else
|
||||
echo "✗ JSON output test failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Test quiet mode
|
||||
working-directory: check-for-unicode
|
||||
run: |
|
||||
output=$(./run.sh --quiet test-suite/clean-test.js 2>&1)
|
||||
if [ -z "$output" ]; then
|
||||
echo "✓ Quiet mode test passed (no output)"
|
||||
else
|
||||
echo "✗ Quiet mode test failed (unexpected output: $output)"
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,11 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Unicode Security Scanner v2.1.0 AI+
|
||||
# Unicode Security Scanner v2025.11.0 AI+
|
||||
# Detects dangerous Unicode characters that can be used in security attacks
|
||||
# Including Trojan Source attacks (CVE-2021-42574) and other invisible characters
|
||||
|
||||
# Script configuration
|
||||
VERSION="2.1.1"
|
||||
VERSION="2025.11.0"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Command-line options (defaults)
|
||||
@@ -17,6 +17,10 @@ EXCLUDE_EMOJIS=false
|
||||
EXCLUDE_COMMON_UNICODE=false
|
||||
INCLUDE_BINARY=false
|
||||
|
||||
# Allowlist storage (global)
|
||||
ALLOWLIST_CODES=""
|
||||
ALLOWLIST_RANGES=()
|
||||
|
||||
# Check dependencies
|
||||
check_dependencies() {
|
||||
local missing=()
|
||||
@@ -53,6 +57,13 @@ OPTIONS:
|
||||
--include-binary Include binary files (archives, images, executables, etc.)
|
||||
By default, only text files are scanned to avoid false positives
|
||||
|
||||
ALLOWLIST FORMAT:
|
||||
The allowlist file supports:
|
||||
- Single codes: U+200B or 200B
|
||||
- Inline comments: U+200B # Zero-width space for i18n
|
||||
- Ranges: U+0400-U+04FF # Allow entire Cyrillic block
|
||||
- Full-line comments starting with #
|
||||
|
||||
EXAMPLES:
|
||||
$0 ./src/ # Scan directory (text files only)
|
||||
$0 script.py # Scan single file
|
||||
@@ -61,6 +72,7 @@ EXAMPLES:
|
||||
$0 --exclude-emojis ./ui/ # Skip emoji characters in UI code
|
||||
$0 --exclude-common ./docs/ # Very permissive for documentation
|
||||
$0 --include-binary ./ # Scan all files including binaries
|
||||
$0 --allowlist .unicode-allowlist ./ # Use custom allowlist
|
||||
|
||||
EXIT CODES:
|
||||
0 - No threats detected
|
||||
@@ -81,15 +93,61 @@ show_version() {
|
||||
|
||||
# Load allowlist (Unicode codes to ignore)
|
||||
load_allowlist() {
|
||||
# Store allowlisted codes in a simple variable (space-separated)
|
||||
# Reset global allowlist variables
|
||||
ALLOWLIST_CODES=""
|
||||
ALLOWLIST_RANGES=()
|
||||
local line_num=0
|
||||
|
||||
if [ -f "$ALLOWLIST_FILE" ]; then
|
||||
while IFS= read -r line; do
|
||||
# Skip comments and empty lines
|
||||
[[ "$line" =~ ^#.*$ || -z "$line" ]] && continue
|
||||
# Extract Unicode code (e.g., U+200B or 200B)
|
||||
code=$(echo "$line" | grep -oE 'U\+[0-9A-Fa-f]+|^[0-9A-Fa-f]+' | tr -d 'U+' | tr '[:lower:]' '[:upper:]')
|
||||
[ -n "$code" ] && ALLOWLIST_CODES="$ALLOWLIST_CODES $code "
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
((line_num++))
|
||||
|
||||
# Skip empty lines
|
||||
[[ -z "$line" ]] && continue
|
||||
|
||||
# Skip full-line comments
|
||||
[[ "$line" =~ ^[[:space:]]*# ]] && continue
|
||||
|
||||
# Strip inline comments (everything after #)
|
||||
line="${line%%#*}"
|
||||
|
||||
# Trim whitespace
|
||||
line=$(echo "$line" | xargs)
|
||||
[[ -z "$line" ]] && continue
|
||||
|
||||
# Check for range syntax (U+XXXX-U+YYYY or XXXX-YYYY)
|
||||
if [[ "$line" =~ ^(U\+)?([0-9A-Fa-f]+)-(U\+)?([0-9A-Fa-f]+)$ ]]; then
|
||||
local start_code="${BASH_REMATCH[2]}"
|
||||
local end_code="${BASH_REMATCH[4]}"
|
||||
start_code=$(echo "$start_code" | tr '[:lower:]' '[:upper:]')
|
||||
end_code=$(echo "$end_code" | tr '[:lower:]' '[:upper:]')
|
||||
|
||||
# Validate range
|
||||
local start_dec=$((16#$start_code))
|
||||
local end_dec=$((16#$end_code))
|
||||
|
||||
if [ $start_dec -gt $end_dec ]; then
|
||||
[ "$QUIET_MODE" = false ] && echo "Warning: Invalid range at line $line_num in allowlist: $line (start > end)" >&2
|
||||
continue
|
||||
fi
|
||||
|
||||
ALLOWLIST_RANGES+=("$start_dec:$end_dec")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Extract single Unicode code (e.g., U+200B or 200B)
|
||||
code=$(echo "$line" | grep -oE 'U\+[0-9A-Fa-f]+|^[0-9A-Fa-f]+' | head -1 | tr -d 'U+' | tr '[:lower:]' '[:upper:]')
|
||||
|
||||
if [ -n "$code" ]; then
|
||||
# Validate the code is a valid hex number
|
||||
if ! [[ "$code" =~ ^[0-9A-F]+$ ]]; then
|
||||
[ "$QUIET_MODE" = false ] && echo "Warning: Invalid Unicode code at line $line_num in allowlist: $line" >&2
|
||||
continue
|
||||
fi
|
||||
ALLOWLIST_CODES="$ALLOWLIST_CODES $code "
|
||||
else
|
||||
[ "$QUIET_MODE" = false ] && echo "Warning: Could not parse line $line_num in allowlist: $line" >&2
|
||||
fi
|
||||
done < "$ALLOWLIST_FILE"
|
||||
fi
|
||||
}
|
||||
@@ -97,7 +155,23 @@ load_allowlist() {
|
||||
# Check if Unicode code is in allowlist
|
||||
is_allowed() {
|
||||
local code=$1
|
||||
[[ "$ALLOWLIST_CODES" == *" $code "* ]]
|
||||
|
||||
# Check direct match
|
||||
[[ "$ALLOWLIST_CODES" == *" $code "* ]] && return 0
|
||||
|
||||
# Check ranges
|
||||
if [ ${#ALLOWLIST_RANGES[@]} -gt 0 ]; then
|
||||
local code_dec=$((16#$code))
|
||||
for range in "${ALLOWLIST_RANGES[@]}"; do
|
||||
local start_dec="${range%%:*}"
|
||||
local end_dec="${range##*:}"
|
||||
if [ $code_dec -ge $start_dec ] && [ $code_dec -le $end_dec ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# Check if pattern is emoji-related
|
||||
@@ -404,7 +478,7 @@ harmful_patterns=(
|
||||
"e28088:2008:Punctuation Space (can break tokenization)"
|
||||
"e28089:2009:Thin Space (subtle spacing attack)"
|
||||
"e2808a:200A:Hair Space (micro-spacing attack)"
|
||||
"e2808b:202F:Narrow No-Break Space (line manipulation)"
|
||||
"e280af:202F:Narrow No-Break Space (line manipulation)"
|
||||
"e281a5:2065:Inhibit Arabic Form Shaping (script confusion)"
|
||||
"e281a6:2066:Left-to-Right Isolate (directional confusion)"
|
||||
"e281a7:2067:Right-to-Left Isolate (directional confusion)"
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
# Test allowlist for Unicode Security Scanner
|
||||
# Format: One Unicode code per line (with or without U+ prefix)
|
||||
# Comments start with #
|
||||
# Supports: inline comments, ranges (U+XXXX-U+YYYY)
|
||||
#
|
||||
# Examples:
|
||||
# U+200B - Single code with prefix
|
||||
# 200B - Single code without prefix
|
||||
# U+200B # comment - Code with inline description
|
||||
# U+0400-U+04FF - Range of codes (entire Cyrillic block)
|
||||
|
||||
# Allow zero-width space for this test
|
||||
U+200B
|
||||
U+200B # Zero Width Space - needed for legitimate i18n word boundaries
|
||||
|
||||
# Allow some homograph characters for legitimate internationalization
|
||||
U+0430
|
||||
U+0430 # Cyrillic Small Letter A - for Russian language support
|
||||
|
||||
@@ -127,6 +127,85 @@ fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Test allowlist functionality
|
||||
echo -e "${YELLOW}Testing allowlist functionality...${NC}"
|
||||
|
||||
# Create a temp file with allowlisted dangerous Unicode
|
||||
ALLOWLIST_TEST_FILE="${SCRIPT_DIR}/allowlist-test-temp.txt"
|
||||
cat > "$ALLOWLIST_TEST_FILE" << 'EOF'
|
||||
# This file has zero-width space and Cyrillic 'a' which are allowlisted
|
||||
Test with zero-width: testword
|
||||
Test with Cyrillic a: аdmin
|
||||
EOF
|
||||
|
||||
if [ -f "${SCRIPT_DIR}/.unicode-allowlist-test" ]; then
|
||||
((total++))
|
||||
# Test that allowlisted characters are NOT detected
|
||||
if "$SCANNER" --allowlist "${SCRIPT_DIR}/.unicode-allowlist-test" "$ALLOWLIST_TEST_FILE" > /dev/null 2>&1; then
|
||||
echo -e " ${GREEN}✓ PASS${NC}: allowlist-test (allowlisted characters ignored)"
|
||||
((passed++))
|
||||
else
|
||||
echo -e " ${RED}✗ FAIL${NC}: allowlist-test (allowlisted characters still detected)"
|
||||
((failed++))
|
||||
fi
|
||||
|
||||
((total++))
|
||||
# Test that same file WITHOUT allowlist DOES detect the characters
|
||||
if "$SCANNER" "$ALLOWLIST_TEST_FILE" > /dev/null 2>&1; then
|
||||
echo -e " ${RED}✗ FAIL${NC}: allowlist-verify (dangerous chars not detected without allowlist)"
|
||||
((failed++))
|
||||
else
|
||||
echo -e " ${GREEN}✓ PASS${NC}: allowlist-verify (dangerous chars detected without allowlist)"
|
||||
((passed++))
|
||||
fi
|
||||
fi
|
||||
|
||||
# Test range syntax in allowlist
|
||||
RANGE_ALLOWLIST="${SCRIPT_DIR}/.allowlist-range-test"
|
||||
cat > "$RANGE_ALLOWLIST" << 'EOF'
|
||||
# Allow all basic Cyrillic (U+0400-U+04FF)
|
||||
U+0400-U+04FF
|
||||
EOF
|
||||
|
||||
# Create a test file with ONLY Cyrillic characters (no other dangerous Unicode)
|
||||
CYRILLIC_TEST_FILE="${SCRIPT_DIR}/cyrillic-only-temp.txt"
|
||||
cat > "$CYRILLIC_TEST_FILE" << 'EOF'
|
||||
# This file only has Cyrillic lookalikes
|
||||
Test with Cyrillic a: аdmin
|
||||
Test with Cyrillic e: еmail
|
||||
EOF
|
||||
|
||||
((total++))
|
||||
# Test that range allowlist works (Cyrillic-only file should pass)
|
||||
if "$SCANNER" --allowlist "$RANGE_ALLOWLIST" "$CYRILLIC_TEST_FILE" > /dev/null 2>&1; then
|
||||
echo -e " ${GREEN}✓ PASS${NC}: range-allowlist (Cyrillic range allowed)"
|
||||
((passed++))
|
||||
else
|
||||
echo -e " ${RED}✗ FAIL${NC}: range-allowlist (Cyrillic range not working)"
|
||||
((failed++))
|
||||
fi
|
||||
|
||||
# Test inline comments in allowlist
|
||||
INLINE_ALLOWLIST="${SCRIPT_DIR}/.allowlist-inline-test"
|
||||
cat > "$INLINE_ALLOWLIST" << 'EOF'
|
||||
U+200B # Zero-width space for i18n purposes
|
||||
0430 # Cyrillic small a - needed for Russian text
|
||||
EOF
|
||||
|
||||
((total++))
|
||||
if "$SCANNER" --allowlist "$INLINE_ALLOWLIST" "$ALLOWLIST_TEST_FILE" > /dev/null 2>&1; then
|
||||
echo -e " ${GREEN}✓ PASS${NC}: inline-comments (allowlist with inline comments works)"
|
||||
((passed++))
|
||||
else
|
||||
echo -e " ${RED}✗ FAIL${NC}: inline-comments (allowlist with inline comments failed)"
|
||||
((failed++))
|
||||
fi
|
||||
|
||||
# Cleanup temp files
|
||||
rm -f "$ALLOWLIST_TEST_FILE" "$RANGE_ALLOWLIST" "$INLINE_ALLOWLIST" "$CYRILLIC_TEST_FILE"
|
||||
|
||||
echo ""
|
||||
|
||||
# Test that malicious files are STILL caught even with exclusion flags
|
||||
echo -e "${YELLOW}Testing malicious files with exclusion flags (should still detect)...${NC}"
|
||||
for file in "${SCRIPT_DIR}"/*injection* "${SCRIPT_DIR}"/*trojan*; do
|
||||
|
||||
Reference in New Issue
Block a user