Exclusion filters (#22)

* inital implementation - README and e2e scripts need updating still

* update e2e script and update CI

* gitlab-ci yml format

* format e2e script

* add example to README

Author: selfup

.github/workflows/go.yml

@@ -18,6 +18,6 @@ jobs:
     - name: Build Bins
       run: ./scripts/bins.sh
     - name: E2E
-      run: ./scripts/e2e.sh
+      run: ETE_DIR=$HOME ./scripts/e2e.sh
     - name: Print SHA
       run: cat scnnr_bins.zip.sha256

.gitlab-ci.yml

@@ -16,6 +16,7 @@ release:
     - go run main.go -m fsf -s 1MB -d $HOME
     - go run main.go -m fnf -f main,DEFCON -p $HOME
     - go run main.go -k main,const,let,var,for -p $HOME
+    - ETE_DIR=$HOME ./scripts/e2e.sh
     - go run cmd/pack/main.go
     - go run main.go -m fff -k $(go run cmd/checksum/main.go) -p .
     - cat scnnr_bins.zip.sha256
@@ -26,4 +27,3 @@ release:
       - scnnr_bins.zip.sha256
       - LICENSE
       - README.md
-

README.md

@@ -96,6 +96,14 @@ $ scnnr -h
   -s string
         REQUIRED SizeFinder MODE
             size: 1MB,10MB,100MB,1GB,10GB,100GB,1TB
+  -xd string
+        OPTIONAL Scnnr MODE
+            comma-delimited list of directory names to exclude from scanning
+            ex: -xd ".git,node_modules,.venv"
+  -xe string
+        OPTIONAL Scnnr MODE
+            comma-delimited list of file extensions to exclude from scanning
+            ex: -xe ".log,.tmp,.json"
 ```
 
 ### No Keywords
@@ -187,6 +195,37 @@ README.md:383:36:cache
 README.md:393:40:cache
 ```
 
+### Keywords while excluding dirs (-xd) and file extensions (-xe)
+
+```bash
+go run main.go -k main,let,for -p . -c -xd ".git,scnnr_bins,pkg,cmd,.zip" -xe ".yml,.sh,.md" 
+.editorconfig:11:21:let
+.gitignore:3:1:main
+.gitignore:4:1:main
+.dockerignore:3:1:main
+.dockerignore:4:1:main
+Dockerfile:14:21:main
+Dockerfile:15:25:main
+main.go:25:9:main
+main.go:36:6:main
+main.go:46:8:for
+main.go:47:8:for
+main.go:48:8:for
+main.go:49:8:for
+main.go:70:65:for
+main.go:78:57:for
+main.go:84:23:for
+main.go:89:34:for
+main.go:125:3:for
+main.go:129:3:for
+main.go:172:3:for
+main.go:179:42:for
+main.go:186:3:for
+scnnr_bins.zip:2053:93:for
+scnnr_bins.zip:26325:8:let
+scnnr_bins.zip:32123:53:let
+```
+
 # File Name Finder (NameFinder) (fnf)
 
 ```

main.go

@@ -101,6 +101,16 @@ func main() {
 	flag.StringVar(&size, "s", "", `REQUIRED SizeFinder MODE
     size: 1MB,10MB,100MB,1GB,10GB,100GB,1TB`)
 
+	var excludeDirs string
+	flag.StringVar(&excludeDirs, "xd", "", `OPTIONAL Scnnr MODE
+    comma-delimited list of directory names to exclude from scanning
+    ex: -xd ".git,node_modules,.venv"`)
+
+	var excludeExts string
+	flag.StringVar(&excludeExts, "xe", "", `OPTIONAL Scnnr MODE
+    comma-delimited list of file extensions to exclude from scanning
+    ex: -xe ".log,.tmp,.json"`)
+
 	flag.Parse()
 
 	directory = dir
@@ -127,13 +137,26 @@ func main() {
 			log.Fatal("Position tracking flags (-l, -c) require keywords (-k)")
 		}
 
+		var excludeDirsList []string
+		var excludeExtsList []string
+
+		if excludeDirs != "" {
+			excludeDirsList = strings.Split(excludeDirs, ",")
+		}
+
+		if excludeExts != "" {
+			excludeExtsList = strings.Split(excludeExts, ",")
+		}
+
 		scanner := scnnr.Scanner{
 			Regex:          rgx,
 			Keywords:       keywords,
 			Directory:      directory,
 			FileExtensions: extensions,
 			ShowLines:      showLines,
 			ShowCols:       showCols,
+			ExcludeDirs:    excludeDirsList,
+			ExcludeExts:    excludeExtsList,
 		}
 
 		err := scanner.Scan()

pkg/scanner.go

@@ -21,6 +21,8 @@ type Scanner struct {
 	FileExtensions   []string
 	Keywords         []string
 	KeywordMatches   []string
+	ExcludeDirs      []string
+	ExcludeExts      []string
 	AllMatches       []Match
 	MatchedFilePaths []FileData
 }
@@ -112,13 +114,23 @@ func (s *Scanner) scan(path string, info os.FileInfo, err error) error {
 		return err
 	}
 
+	// Check if directory should be excluded
+	if info.IsDir() && s.shouldExcludeDir(filepath.Base(path)) {
+		return filepath.SkipDir
+	}
+
 	if !info.IsDir() {
+		fileExtension := filepath.Ext(path)
+
+		// Check if file extension should be excluded
+		if s.shouldExcludeExt(fileExtension) {
+			return nil
+		}
+
 		if s.FileExtensions[0] == "" {
 			s.MatchedFilePaths = append(s.MatchedFilePaths, FileData{path, info})
 		} else {
 			for _, pattern := range s.FileExtensions {
-				fileExtension := filepath.Ext(path)
-
 				if fileExtension == pattern {
 					s.MatchedFilePaths = append(s.MatchedFilePaths, FileData{path, info})
 				}
@@ -208,6 +220,26 @@ func (s *Scanner) parse(match FileData) {
 	file.Close()
 }
 
+// shouldExcludeDir checks if a directory name should be excluded
+func (s *Scanner) shouldExcludeDir(dirName string) bool {
+	for _, excludeDir := range s.ExcludeDirs {
+		if dirName == excludeDir {
+			return true
+		}
+	}
+	return false
+}
+
+// shouldExcludeExt checks if a file extension should be excluded
+func (s *Scanner) shouldExcludeExt(ext string) bool {
+	for _, excludeExt := range s.ExcludeExts {
+		if ext == excludeExt {
+			return true
+		}
+	}
+	return false
+}
+
 func eachSlice(files []FileData) [][]FileData {
 	var chunks [][]FileData
 	var chunk []FileData

scripts/e2e.sh

@@ -2,9 +2,20 @@
 
 set -eou pipefail
 
+e2e_dir=$HOME
+
+if [[ $ETE_DIR != "" ]]
+then
+    e2e_dir=$ETE_DIR
+fi
+
+##################################################
+##################################################
+##################################################
+
 echo "--- FILE SIZE FINDER DRY RUN: BEGIN ---"
 
-go run main.go -m fsf -s 1MB -d $HOME
+go run main.go -m fsf -s 1MB -d $e2e_dir
 
 echo "--- FILE SIZE FINDER DRY RUN: DONE ---"
 
@@ -16,7 +27,7 @@ sleep 2
 
 echo "--- FILE NAME FINDER DRY RUN: BEGIN---"
 
-go run main.go -m fnf -f main,DEFCON -p $HOME
+go run main.go -m fnf -f main,DEFCON -p $e2e_dir
 
 echo "--- FILE NAME FINDER DRY RUN: DONE ---"
 
@@ -28,12 +39,16 @@ sleep 2
 
 echo "--- SCANNER DRY RUN: BEGIN---"
 
-go run main.go -k main,const,let,var,for -p $HOME
+go run main.go -k main,const,let,var,for -p $e2e_dir
 
 echo "--- SCANNER DRY RUN: DONE ---"
 
 sleep 2
 
+##################################################
+##################################################
+##################################################
+
 echo "--- FILE FINGERPRINT FINDER DRY RUN: BEGIN---"
 
 go run main.go -m fff -k $(go run cmd/checksum/main.go) -d .
@@ -42,16 +57,62 @@ echo "--- FILE FINGERPRINT FINDER DRY RUN: DONE---"
 
 sleep 2
 
+##################################################
+##################################################
+##################################################
+
 echo "--- SCANNER LINE RUN: BEGIN---"
 
-go run main.go -k main,const,let,var,for -p $HOME -l
+go run main.go -k main,const,let,var,for -p $e2e_dir -l
 
 echo "--- SCANNER LINE RUN: DONE ---"
 
 sleep 2
 
+##################################################
+##################################################
+##################################################
+
 echo "--- SCANNER LINE AND COL RUN: BEGIN---"
 
-go run main.go -k main,const,let,var,for -p $HOME -c
+go run main.go -k main,const,let,var,for -p $e2e_dir -c
 
 echo "--- SCANNER LINE AND COL RUN: DONE ---"
+
+sleep 2
+
+##################################################
+##################################################
+##################################################
+
+echo "--- SCANNER DIR EXCLUDE RUN: BEGIN---"
+
+sleep 2
+
+go run main.go -k main,const,let,var,for -p $e2e_dir -c -xd ".git"
+
+echo "--- SCANNER DIR EXCLUDE RUN: DONE ---"
+
+sleep 2
+
+##################################################
+##################################################
+##################################################
+
+echo "--- SCANNER EXTENSION EXCLUDE RUN: BEGIN---"
+
+go run main.go -k main,const,let,var,for -p $e2e_dir -c -xe ".yml,.sh,.md"
+
+echo "--- SCANNER EXTENSION RUN: DONE ---"
+
+sleep 2
+
+##################################################
+##################################################
+##################################################
+
+echo "--- SCANNER DIRECTORY AND EXTENSION EXCLUDE RUN: BEGIN---"
+
+go run main.go -k main,const,let,var,for -p $e2e_dir -c -xd ".git" -xe ".yml,.sh,.md"
+
+echo "--- SCANNER DIRECTORY AND EXTENSION RUN: DONE ---"