Beesquit/app
1
0
Fork 0
forked from YaBL/app
Code Pull Requests Activity

reaper: я уже не знаю что можно сделать лучше, купите компьютер получше хз

Browse Source
This commit is contained in:
Michael 2025-06-22 13:12:27 +03:00
parent bde518c91d
commit 0b4eb08a3f
1 changed files with 19 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
apps/book-reaper/internal/app/app.go
View File

@ -39,7 +39,7 @@ func (a *App) Run() error {
totalZips := len(zipFiles) totalZips := len(zipFiles)
fmt.Printf("Found %d archives\n", totalZips) fmt.Printf("Found %d archives\n", totalZips)
jobChan := make(chan BookJob, 2500) jobChan := make(chan BookJob, 1000)
var dbWg sync.WaitGroup var dbWg sync.WaitGroup
var processedCount uint64 = 0 var processedCount uint64 = 0
@ -104,7 +104,7 @@ func findZipFiles(basePath string) []string {
} }
func processZipFilesParallel(basePath string, zipFiles []string, jobChan chan<- BookJob, processedCount *uint64, totalZips int) { func processZipFilesParallel(basePath string, zipFiles []string, jobChan chan<- BookJob, processedCount *uint64, totalZips int) {
const workers = 8 const workers = 16
var wg sync.WaitGroup var wg sync.WaitGroup
tasks := make(chan string, workers) tasks := make(chan string, workers)
@ -140,7 +140,7 @@ func processZipFilesParallel(basePath string, zipFiles []string, jobChan chan<-
func processZip(basePath string, zipPath string, jobChan chan<- BookJob) { func processZip(basePath string, zipPath string, jobChan chan<- BookJob) {
r, err := zip.OpenReader(zipPath) r, err := zip.OpenReader(zipPath)
if err != nil { if err != nil {
log.Printf("Failed open zip: %v\n", err) log.Printf("Failed open zip: %v, %s\n", err, zipPath)
return return
} }
bookcase, err := filepath.Rel(basePath, zipPath) bookcase, err := filepath.Rel(basePath, zipPath)
@ -149,29 +149,25 @@ func processZip(basePath string, zipPath string, jobChan chan<- BookJob) {
return return
} }
defer r.Close() defer r.Close()
var wg sync.WaitGroup
for _, f := range r.File { for _, f := range r.File {
if strings.HasSuffix(strings.ToLower(f.Name), ".fb2") { if strings.HasSuffix(strings.ToLower(f.Name), ".fb2") {
wg.Add(1) rc, err := f.Open()
go func() { if err != nil {
defer wg.Done() log.Printf("Unable read file from archive: %v\n", err)
rc, err := f.Open() return
if err != nil { }
log.Printf("Unable read file from archive: %v\n", err) rawFB2 := reaper.Parse(rc)
return _ = rc.Close()
}
rawFB2 := reaper.Parse(rc)
_ = rc.Close()
if rawFB2 == nil { if rawFB2 == nil {
// log.Printf("Не удалось распарсить: %s\n", f.Name) // log.Printf("Не удалось распарсить: %s\n", f.Name)
return return
} }
fb2 := reaper.RawToFB2(*rawFB2, f.FileInfo().Name(), &bookcase, f.UncompressedSize64, nil) fb2 := reaper.RawToFB2(*rawFB2, f.FileInfo().Name(), &bookcase, f.UncompressedSize64, nil)
log.Println("paersed book", fb2.Title) // total := atomic.LoadUint64(&totalAddedBooks)
jobChan <- BookJob{FB2: fb2} // fmt.Printf("\rpaersed book %s %d", "total", total)
}() jobChan <- BookJob{FB2: fb2}
} }
} }
wg.Wait()
} }