YaBL/app
4
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

reaper: я уже не знаю что можно сделать лучше, купите компьютер получше хз

Browse Source
This commit is contained in:
Michael 2025-06-22 13:12:27 +03:00
parent bde518c91d
commit 0b4eb08a3f
1 changed files with 19 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
apps/book-reaper/internal/app/app.go
View File

@ -39,7 +39,7 @@ func (a *App) Run() error {
totalZips := len(zipFiles)
fmt.Printf("Found %d archives\n", totalZips)
jobChan := make(chan BookJob, 2500)
jobChan := make(chan BookJob, 1000)
var dbWg sync.WaitGroup
var processedCount uint64 = 0
@ -104,7 +104,7 @@ func findZipFiles(basePath string) []string {
}
func processZipFilesParallel(basePath string, zipFiles []string, jobChan chan<- BookJob, processedCount *uint64, totalZips int) {
const workers = 8
const workers = 16
var wg sync.WaitGroup
tasks := make(chan string, workers)
@ -140,7 +140,7 @@ func processZipFilesParallel(basePath string, zipFiles []string, jobChan chan<-
func processZip(basePath string, zipPath string, jobChan chan<- BookJob) {
r, err := zip.OpenReader(zipPath)
if err != nil {
log.Printf("Failed open zip: %v\n", err)
log.Printf("Failed open zip: %v, %s\n", err, zipPath)
return
}
bookcase, err := filepath.Rel(basePath, zipPath)
@ -149,29 +149,25 @@ func processZip(basePath string, zipPath string, jobChan chan<- BookJob) {
return
}
defer r.Close()
var wg sync.WaitGroup
for _, f := range r.File {
if strings.HasSuffix(strings.ToLower(f.Name), ".fb2") {
wg.Add(1)
go func() {
defer wg.Done()
rc, err := f.Open()
if err != nil {
log.Printf("Unable read file from archive: %v\n", err)
return
}
rawFB2 := reaper.Parse(rc)
_ = rc.Close()
rc, err := f.Open()
if err != nil {
log.Printf("Unable read file from archive: %v\n", err)
return
}
rawFB2 := reaper.Parse(rc)
_ = rc.Close()
if rawFB2 == nil {
// log.Printf("Не удалось распарсить: %s\n", f.Name)
return
}
fb2 := reaper.RawToFB2(*rawFB2, f.FileInfo().Name(), &bookcase, f.UncompressedSize64, nil)
log.Println("paersed book", fb2.Title)
jobChan <- BookJob{FB2: fb2}
}()
if rawFB2 == nil {
// log.Printf("Не удалось распарсить: %s\n", f.Name)
return
}
fb2 := reaper.RawToFB2(*rawFB2, f.FileInfo().Name(), &bookcase, f.UncompressedSize64, nil)
// total := atomic.LoadUint64(&totalAddedBooks)
// fmt.Printf("\rpaersed book %s %d", "total", total)
jobChan <- BookJob{FB2: fb2}
}
}
wg.Wait()
}