diff --git a/check-for-unicode/CHANGELOG.md b/check-for-unicode/CHANGELOG.md index 19371bf..5b1e2fc 100644 --- a/check-for-unicode/CHANGELOG.md +++ b/check-for-unicode/CHANGELOG.md @@ -7,6 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.1.1] - 2025-10-30 Binary File False Positive Fix + +### Added +- ๐Ÿšซ **Automatic Binary File Skipping**: Scanner now automatically skips binary files by default to prevent false positives + - Skips archives: `.jar`, `.war`, `.ear`, `.zip`, `.tar`, `.gz`, etc. + - Skips images: `.jpg`, `.png`, `.gif`, `.svg`, `.webp`, etc. + - Skips videos: `.mp4`, `.avi`, `.mov`, `.mkv`, etc. + - Skips audio: `.mp3`, `.wav`, `.ogg`, `.flac`, etc. + - Skips executables: `.exe`, `.dll`, `.so`, `.dylib`, etc. + - Skips fonts: `.ttf`, `.otf`, `.woff`, `.woff2`, etc. + - Skips binary documents: `.pdf`, `.doc`, `.docx`, `.xls`, etc. +- ๐Ÿ”ง **`--include-binary` Flag**: Force scanning of binary files when needed +- ๐Ÿงช **Binary File Tests**: Added test suite coverage for binary file handling + - Test binary files are skipped by default + - Test binary files are scanned with `--include-binary` flag + +### Fixed +- ๐Ÿ› **False Positives in Binary Files**: Files like `.jar`, `.zip`, `.png`, `.pdf` no longer trigger false detections +- ๐Ÿ› **Shell Script Detection**: Fixed issue where shell scripts were incorrectly classified as binary executables +- ๐Ÿ”ง **File Type Detection**: Improved binary detection using both file extension and MIME type checks + +### Changed +- ๐Ÿ“ˆ **Version**: Bumped from 2.1.0 to 2.1.1 +- ๐Ÿ“š **Documentation**: Updated README with binary file handling information +- ๐Ÿ“š **Help Text**: Added `--include-binary` flag documentation +- ๐Ÿ” **is_binary_file()**: New function for comprehensive binary file detection +- ๐Ÿงช **Test Suite**: Now includes 11 tests (up from 9) + +### Security +- โœ… **Maintained Security**: All text-based threats are still detected +- โœ… **Smart Defaults**: Only scans text files where Unicode attacks are relevant +- โœ… **User Control**: Users can still force scan binary files if needed with `--include-binary` + ## [2.1.0] - 2025-10-23 False Positive Fix ### Added diff --git a/check-for-unicode/README.md b/check-for-unicode/README.md index f3ffbcd..fef8117 100644 --- a/check-for-unicode/README.md +++ b/check-for-unicode/README.md @@ -4,22 +4,23 @@ A comprehensive security scanner that detects dangerous Unicode characters used ## โš ๏ธ Avoiding False Positives -**NEW in v2.1.0**: The scanner now includes context-aware detection to reduce false positives: +**NEW in v2.1.1**: The scanner now automatically skips binary files to prevent false positives: +- **Binary files** (`.jar`, `.zip`, `.png`, `.pdf`, etc.) are automatically skipped by default +- **Text files only** are scanned unless you use `--include-binary` flag - **Emoji characters** (๐Ÿท๏ธ, ๐Ÿช, etc.) used in UI elements are automatically detected and can be excluded - **Smart quotes and common Unicode** used in documentation can be excluded -- Use `--exclude-emojis` flag when scanning UI/frontend code -- Use `--exclude-common` flag for documentation or content files - Use `.unicode-allowlist` file to whitelist specific Unicode characters for your project -**Common False Positives:** +**Common False Positives Fixed:** +- โœ… **Binary files**: Now skipped automatically (archives, images, executables, etc.) - โœ… **Emojis in UI**: Use `--exclude-emojis` flag - โœ… **Smart quotes in docs**: Use `--exclude-common` flag - โœ… **Intentional Unicode in i18n**: Add to `.unicode-allowlist` ## Purpose -This enhanced script (v2.1.0 AI+) identifies Unicode characters that can: +This enhanced script (v2.1.1 AI+) identifies Unicode characters that can: - **AI Injection Attacks**: Characters used to manipulate AI model responses - **Homograph Attacks**: Visually similar characters from different scripts (CVE-2017-5116) @@ -248,9 +249,12 @@ chmod +x run.sh # Scan a single file ./run.sh /path/to/file.txt -# Scan a directory recursively +# Scan a directory recursively (automatically skips binary files) ./run.sh /path/to/directory +# Scan including binary files (archives, images, etc.) +./run.sh --include-binary ./ + # Scan UI/frontend code (exclude emojis to avoid false positives) ./run.sh --exclude-emojis ./src/components/ diff --git a/check-for-unicode/run.sh b/check-for-unicode/run.sh index 3f55b70..3228989 100755 --- a/check-for-unicode/run.sh +++ b/check-for-unicode/run.sh @@ -5,7 +5,7 @@ # Including Trojan Source attacks (CVE-2021-42574) and other invisible characters # Script configuration -VERSION="2.1.0" +VERSION="2.1.1" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Command-line options (defaults) @@ -15,6 +15,7 @@ SEVERITY_FILTER="" ALLOWLIST_FILE="${SCRIPT_DIR}/.unicode-allowlist" EXCLUDE_EMOJIS=false EXCLUDE_COMMON_UNICODE=false +INCLUDE_BINARY=false # Check dependencies check_dependencies() { @@ -49,14 +50,17 @@ OPTIONS: --allowlist FILE Path to allowlist file (default: .unicode-allowlist) --exclude-emojis Exclude emoji characters and variation selectors (reduces false positives) --exclude-common Exclude common Unicode like smart quotes, dashes (very permissive) + --include-binary Include binary files (archives, images, executables, etc.) + By default, only text files are scanned to avoid false positives EXAMPLES: - $0 ./src/ # Scan directory + $0 ./src/ # Scan directory (text files only) $0 script.py # Scan single file $0 --quiet --json ./app/ > results.json # JSON output for CI $0 --severity critical,high ./ # Only show critical/high $0 --exclude-emojis ./ui/ # Skip emoji characters in UI code $0 --exclude-common ./docs/ # Very permissive for documentation + $0 --include-binary ./ # Scan all files including binaries EXIT CODES: 0 - No threats detected @@ -127,6 +131,93 @@ is_common_unicode() { return 1 } +# Check if file is binary (non-text) +is_binary_file() { + local file=$1 + + # Check by file extension first (faster) + case "${file##*.}" in + # Archives + jar|war|ear|zip|tar|gz|tgz|bz2|xz|7z|rar|iso) + return 0 + ;; + # Images + jpg|jpeg|png|gif|bmp|ico|svg|webp|tiff|tif|psd) + return 0 + ;; + # Videos + mp4|avi|mov|wmv|flv|mkv|webm|m4v) + return 0 + ;; + # Audio + mp3|wav|ogg|flac|aac|wma|m4a) + return 0 + ;; + # Executables and libraries + exe|dll|so|dylib|a|o|class|pyc|pyo|beam) + return 0 + ;; + # Documents (binary formats) + pdf|doc|docx|xls|xlsx|ppt|pptx|odt|ods|odp) + return 0 + ;; + # Fonts + ttf|otf|woff|woff2|eot) + return 0 + ;; + # Databases + db|sqlite|sqlite3) + return 0 + ;; + # Other binary formats + bin|dat|pkg|deb|rpm|dmg) + return 0 + ;; + esac + + # Use file command to check MIME type + local mime_type + mime_type=$(file -b --mime-type "$file" 2>/dev/null) + + # Check if MIME type indicates binary + case "$mime_type" in + application/x-executable|\ + application/x-sharedlib|\ + application/x-object|\ + application/x-archive|\ + application/zip|\ + application/gzip|\ + application/x-tar|\ + application/x-bzip2|\ + application/x-xz|\ + application/java-archive|\ + application/pdf|\ + application/octet-stream|\ + image/*|\ + video/*|\ + audio/*|\ + font/*) + return 0 + ;; + esac + + # Check if file command says it's binary + # Note: Avoid matching "shell script text executable" - we want actual binary executables + local file_desc + file_desc=$(file "$file" 2>/dev/null) + if echo "$file_desc" | grep -qiE "executable.*binary|archive|compressed|image data|audio|video"; then + return 0 + fi + + # Exclude shell scripts and other text-based executables + if echo "$file_desc" | grep -qi "text"; then + return 1 + fi + + # Not binary + return 1 +} + # Check if hex pattern appears in an emoji context is_in_emoji_context() { local hex_content=$1 @@ -439,6 +530,10 @@ while [[ $# -gt 0 ]]; do EXCLUDE_COMMON_UNICODE=true shift ;; + --include-binary) + INCLUDE_BINARY=true + shift + ;; -*) echo "Error: Unknown option: $1" >&2 echo "Use --help for usage information" >&2 @@ -464,7 +559,7 @@ load_allowlist # Show header unless in quiet or JSON mode if [ "$QUIET_MODE" = false ] && [ "$JSON_OUTPUT" = false ]; then echo -e "\033[1;35mโ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—\033[0m" - echo -e "\033[1;35mโ•‘ Big Bear Unicode Security Scanner v2.1.0 AI+ โ•‘\033[0m" + echo -e "\033[1;35mโ•‘ Big Bear Unicode Security Scanner v2.1.1 AI+ โ•‘\033[0m" echo -e "\033[1;35mโ•‘ Detecting dangerous Unicode & AI injection attacks โ•‘\033[0m" echo -e "\033[1;35mโ•‘ Please support me! โ•‘\033[0m" echo -e "\033[1;35mโ•‘ https://ko-fi.com/bigbeartechworld โ•‘\033[0m" @@ -483,6 +578,15 @@ declare -a json_results search_file() { file="$1" + # Skip binary files unless --include-binary is set + if [ "$INCLUDE_BINARY" = false ] && is_binary_file "$file"; then + if [ "$QUIET_MODE" = false ] && [ "$JSON_OUTPUT" = false ]; then + echo -e "\n\033[1;34mSkipping:\033[0m $file \033[90m(binary file)\033[0m" + fi + ((total_files++)) + return + fi + if [ "$QUIET_MODE" = false ] && [ "$JSON_OUTPUT" = false ]; then echo -e "\n\033[1;34mScanning:\033[0m $file" fi diff --git a/check-for-unicode/test-suite/run-tests.sh b/check-for-unicode/test-suite/run-tests.sh index 4e65f4f..a9a2705 100755 --- a/check-for-unicode/test-suite/run-tests.sh +++ b/check-for-unicode/test-suite/run-tests.sh @@ -101,6 +101,32 @@ done echo "" +# Test binary file handling +echo -e "${YELLOW}Testing binary file handling...${NC}" +if [ -f "${SCRIPT_DIR}/test.jar" ]; then + ((total++)) + # Test that binary files are skipped by default + if "$SCANNER" "${SCRIPT_DIR}/test.jar" 2>&1 | grep -q "Skipping.*binary"; then + echo -e " ${GREEN}โœ“ PASS${NC}: test.jar (binary file skipped by default)" + ((passed++)) + else + echo -e " ${RED}โœ— FAIL${NC}: test.jar (binary file was not skipped)" + ((failed++)) + fi + + ((total++)) + # Test that binary files are scanned with --include-binary + if "$SCANNER" --include-binary "${SCRIPT_DIR}/test.jar" 2>&1 | grep -q "Scanning.*test.jar"; then + echo -e " ${GREEN}โœ“ PASS${NC}: test.jar (binary file scanned with --include-binary)" + ((passed++)) + else + echo -e " ${RED}โœ— FAIL${NC}: test.jar (binary file not scanned with --include-binary)" + ((failed++)) + fi +fi + +echo "" + # Test that malicious files are STILL caught even with exclusion flags echo -e "${YELLOW}Testing malicious files with exclusion flags (should still detect)...${NC}" for file in "${SCRIPT_DIR}"/*injection* "${SCRIPT_DIR}"/*trojan*; do diff --git a/check-for-unicode/test-suite/test.jar b/check-for-unicode/test-suite/test.jar new file mode 100644 index 0000000..f26ef03 Binary files /dev/null and b/check-for-unicode/test-suite/test.jar differ