reaper: MORE parallel

apps/book-reaper/internal/app/app.go

@@ -48,7 +48,7 @@ func (a *App) Run() error {
 	go func() {
 		defer dbWg.Done()
 
-		batchSize := 50
+		batchSize := 150
 		batch := make([]BookJob, 0, batchSize)
 		flush := func() {
 			addedBooks := 0
@@ -104,7 +104,7 @@ func findZipFiles(basePath string) []string {
 }
 
 func processZipFilesParallel(basePath string, zipFiles []string, jobChan chan<- BookJob, processedCount *uint64, totalZips int) {
-	const workers = 4
+	const workers = 8
 
 	var wg sync.WaitGroup
 	tasks := make(chan string, workers)
@@ -143,32 +143,35 @@ func processZip(basePath string, zipPath string, jobChan chan<- BookJob) {
 		log.Printf("Failed open zip: %v\n", err)
 		return
 	}
+	bookcase, err := filepath.Rel(basePath, zipPath)
+	if err != nil {
+		log.Printf("Error rel path: %v\n", err)
+		return
+	}
 	defer r.Close()
-
+	var wg sync.WaitGroup
 	for _, f := range r.File {
 		if strings.HasSuffix(strings.ToLower(f.Name), ".fb2") {
-			rc, err := f.Open()
+			wg.Add(1)
-			if err != nil {
+			go func() {
-				log.Printf("Unable read file from archive: %v\n", err)
+				defer wg.Done()
-				continue
+				rc, err := f.Open()
-			}
+				if err != nil {
+					log.Printf("Unable read file from archive: %v\n", err)
+					return
+				}
+				rawFB2 := reaper.Parse(rc)
+				_ = rc.Close()
 
-			rawFB2 := reaper.Parse(rc)
+				if rawFB2 == nil {
-			_ = rc.Close()
+					// log.Printf("Не удалось распарсить: %s\n", f.Name)
+					return
+				}
 
-			if rawFB2 == nil {
+				fb2 := reaper.RawToFB2(*rawFB2, f.FileInfo().Name(), &bookcase, f.UncompressedSize64, nil)
-				// log.Printf("Не удалось распарсить: %s\n", f.Name)
+				jobChan <- BookJob{FB2: fb2}
-				continue
+			}()
-			}
-
-			bookcase, err := filepath.Rel(basePath, zipPath)
-			if err != nil {
-				log.Printf("Error rel path: %v\n", err)
-				continue
-			}
-
-			fb2 := reaper.RawToFB2(*rawFB2, f.FileInfo().Name(), &bookcase, f.UncompressedSize64, nil)
-			jobChan <- BookJob{FB2: fb2}
 		}
 	}
+	wg.Wait()
 }