using worker pool to boost deletion process

README.md

@@ -7,7 +7,7 @@ Golang CLI to delete GitLab stored artifacts
 When you need to wipe out a Gitlab project's artifacts, this is the tool for you. It will delete all the artifacts in a project. It will not delete the project itself. It will not delete the project's repository.
 
 Because of GitLab API limitations, you need to know how many job pages you need to iterate through to delete all the artifacts. This is how GitLab API works. If you don't know it, just use the standard values:
-  
+
   ```yaml
   per_page: 100
   page: 20
@@ -22,9 +22,10 @@ Parameters:
 - `--project_id`: Project ID or path.
 - `--per-page`: Number of jobs to fetch per page. Default: `100`
 - `--page`: Page number. Default: `1`
+- `--concurrent_requests`: Number of concurrent requests
 
 ```bash
-gitlab-artifacts-cleaner --server https://gitlab.com --token <token> --project_id <project_id> --pages 10 --per_page 100
+gitlab-artifacts-cleaner --server https://gitlab.com --token <token> --project_id <project_id> --pages 10 --per_page 100 --concurrent_requests 5
 ```
 
 This is a work in progress. If you have any suggestions, please open an issue.

main.go

@@ -5,22 +5,48 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"sync"
 )
 
 type Job struct {
-	ID          int    `json:"id"`
+	ID int `json:"id"`
 }
 
-func main () {
+func worker(server string, project_id int, token string, jobs <-chan Job, results chan<- *http.Response, wg *sync.WaitGroup) {
+	for job := range jobs {
+		fmt.Println("job to process:", job.ID)
+		url := fmt.Sprintf("%v/api/v4/projects/%v/jobs/%d/artifacts", server, project_id, job.ID)
+		req, err := http.NewRequest("DELETE", url, nil)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		req.Header.Set("PRIVATE-TOKEN", token)
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		defer resp.Body.Close()
+		fmt.Println("job artifacts deleted:", job.ID)
+
+		results <- resp
+		wg.Done()
+	}
+}
+
+func main() {
 	var project_id int
 	var per_page int
 	var pages int
+	var concurrent_requests int
 	var token string
 	var server string
 
 	flag.IntVar(&project_id, "project_id", 0, "Project ID")
 	flag.IntVar(&per_page, "per_page", 100, "Number of jobs per page")
 	flag.IntVar(&pages, "pages", 1, "Number of pages")
+	flag.IntVar(&concurrent_requests, "concurrent_requests", 5, "Number of concurrent requests")
 	flag.StringVar(&token, "token", "", "Private token")
 	flag.StringVar(&server, "server", "", "Gitlab server")
 	flag.Parse()
@@ -50,22 +76,22 @@ func main () {
 			return
 		}
 
+		pool_jobs := make(chan Job, len(jobs))
+		results := make(chan *http.Response, len(jobs))
+		var wg sync.WaitGroup
+
+		// Start workers
+		for w := 0; w < concurrent_requests; w++ {
+			go worker(server, project_id, token, pool_jobs, results, &wg)
+		}
+
+		// Sending jobs to the worker pool
+		wg.Add(len(jobs))
 		for _, job := range jobs {
-			fmt.Println("job to process:", job.ID)
-			url := fmt.Sprintf("%v/api/v4/projects/%v/jobs/%d/artifacts", server, project_id, job.ID)
-			req, err := http.NewRequest("DELETE", url, nil)
-			if err != nil {
-				fmt.Println(err)
-				return
-			}
-			req.Header.Set("PRIVATE-TOKEN", token)
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				fmt.Println(err)
-				return
-			}
-			defer resp.Body.Close()
-			fmt.Println("job artifacts deleted:", job.ID)
+			pool_jobs <- job
 		}
+		close(pool_jobs)
+		wg.Wait()
+
 	}
 }