chore: init monorepo

This commit is contained in:
2025-06-21 12:42:09 +03:00
commit 1874ae3ac1
103 changed files with 23946 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
.env
+29
View File
@@ -0,0 +1,29 @@
package main
import (
"book-tools/internal/app"
"book-tools/internal/config"
"book-tools/internal/storage"
"log"
"github.com/joho/godotenv"
)
func main() {
_ = godotenv.Load() // не паникуем, если файла нет
cfg, err := config.Load()
if err != nil {
log.Fatalf("Ошибка загрузки конфига: %v", err)
}
db, err := storage.NewSQLite(cfg.DBPath)
if err != nil {
log.Fatalf("Не удалось открыть базу: %v", err)
}
app := app.NewApp(cfg, db)
if err := app.Run(); err != nil {
log.Fatalf("Ошибка в приложении: %v", err)
}
}
+14
View File
@@ -0,0 +1,14 @@
module book-tools
go 1.24.4
require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/joho/godotenv v1.5.1
github.com/mattn/go-sqlite3 v1.14.28 // indirect
golang.org/x/net v0.41.0
golang.org/x/text v0.26.0 // indirect
gorm.io/driver/sqlite v1.6.0
gorm.io/gorm v1.30.0
)
+16
View File
@@ -0,0 +1,16 @@
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs=
gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
+174
View File
@@ -0,0 +1,174 @@
package app
import (
"archive/zip"
"book-tools/internal/config"
"book-tools/pkg/reaper"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"gorm.io/gorm"
)
type App struct {
cfg *config.Config
db *gorm.DB
}
func NewApp(cfg *config.Config, db *gorm.DB) *App {
return &App{cfg: cfg, db: db}
}
var totalAddedBooks uint64
type BookJob struct {
FB2 reaper.FB2
}
func (a *App) Run() error {
// fmt.Printf("Работаем с папкой: %s\n", a.cfg.BaseDir)
// fmt.Printf("Используем базу: %s\n", a.cfg.DBPath)
zipFiles := findZipFiles(a.cfg.BaseDir)
totalZips := len(zipFiles)
fmt.Printf("Found %d archives\n", totalZips)
jobChan := make(chan BookJob, 500)
var dbWg sync.WaitGroup
var processedCount uint64 = 0
// 🧠 Писатель в БД, добавляем пачками по 50 книг
dbWg.Add(1)
go func() {
defer dbWg.Done()
batchSize := 50
batch := make([]BookJob, 0, batchSize)
flush := func() {
addedBooks := 0
if len(batch) == 0 {
return
}
tx := a.db.Begin()
for _, job := range batch {
bookID := reaper.FB2toDB(tx, job.FB2)
if tx.Error != nil {
tx.Rollback()
log.Printf("Failed add book to transaction: %v", tx.Error)
return
}
if bookID > 0 {
addedBooks++
}
}
tx.Commit()
atomic.AddUint64(&processedCount, uint64(len(batch)))
atomic.AddUint64(&totalAddedBooks, uint64(addedBooks))
batch = batch[:0]
}
for job := range jobChan {
batch = append(batch, job)
if len(batch) >= batchSize {
flush()
}
}
flush()
}()
// Обработка ZIP в параллели
processZipFilesParallel(a.cfg.BaseDir, zipFiles, jobChan, &processedCount, totalZips)
close(jobChan)
dbWg.Wait()
fmt.Printf("\nAll done. Added %d books.\n", totalAddedBooks)
return nil
}
func findZipFiles(basePath string) []string {
var zips []string
_ = filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error {
if err == nil && !info.IsDir() && strings.HasSuffix(strings.ToLower(info.Name()), ".zip") {
zips = append(zips, path)
}
return nil
})
return zips
}
func processZipFilesParallel(basePath string, zipFiles []string, jobChan chan<- BookJob, processedCount *uint64, totalZips int) {
const workers = 4
var wg sync.WaitGroup
tasks := make(chan string, workers)
// Для прогресса - можно сделать так: считаем обработанные zip файлы
var zipProcessed uint64 = 0
// Запускаем воркеров
for i := 0; i < workers; i++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
for zipPath := range tasks {
start := time.Now()
processZip(basePath, zipPath, jobChan)
duration := time.Since(start)
atomic.AddUint64(&zipProcessed, 1)
// Вывод прогресса в одну строку, перезаписывая её
percentage := float64(atomic.LoadUint64(&zipProcessed)) / float64(totalZips) * 100
fmt.Printf("\r[Worker %d] Processed archives: %d/%d (%.1f%%) — %s (%.2fs) (%d books)", id, atomic.LoadUint64(&zipProcessed), totalZips, percentage, filepath.Base(zipPath), duration.Seconds(), totalAddedBooks)
}
}(i + 1)
}
for _, z := range zipFiles {
tasks <- z
}
close(tasks)
wg.Wait()
}
func processZip(basePath string, zipPath string, jobChan chan<- BookJob) {
r, err := zip.OpenReader(zipPath)
if err != nil {
log.Printf("Failed open zip: %v\n", err)
return
}
defer r.Close()
for _, f := range r.File {
if strings.HasSuffix(strings.ToLower(f.Name), ".fb2") {
rc, err := f.Open()
if err != nil {
log.Printf("Unable read file from archive: %v\n", err)
continue
}
rawFB2 := reaper.Parse(rc)
_ = rc.Close()
if rawFB2 == nil {
// log.Printf("Не удалось распарсить: %s\n", f.Name)
continue
}
bookcase, err := filepath.Rel(basePath, zipPath)
if err != nil {
log.Printf("Error rel path: %v\n", err)
continue
}
fb2 := reaper.RawToFB2(*rawFB2, f.FileInfo().Name(), &bookcase, f.UncompressedSize64, nil)
jobChan <- BookJob{FB2: fb2}
}
}
}
@@ -0,0 +1,25 @@
package config
import (
"errors"
"os"
)
type Config struct {
BaseDir string
DBPath string
}
func Load() (*Config, error) {
dir := os.Getenv("BASE_PATH")
db := os.Getenv("DB_LINK")
if dir == "" || db == "" {
return nil, errors.New("must set BASE_PATH and DB_LINK in env")
}
return &Config{
BaseDir: dir,
DBPath: db,
}, nil
}
@@ -0,0 +1,21 @@
package storage
import (
"fmt"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
func NewSQLite(path string) (*gorm.DB, error) {
fmt.Println("connecting...")
var db *gorm.DB
db, err := gorm.Open(sqlite.Open(path), &gorm.Config{
Logger: logger.Default.LogMode(logger.Error),
})
if err != nil {
return nil, err
}
return db, nil
}
@@ -0,0 +1,51 @@
package reaper
type Person struct {
FirstName string `xml:"first-name"`
MiddleName *string `xml:"middle-name"`
LastName *string `xml:"last-name"`
}
type Sequences struct {
Name *string `xml:"name,attr"`
Number *int `xml:"number,attr"`
}
type FB2Read struct {
Title string `xml:"title-info>book-title"`
Genres []string `xml:"title-info>genre"`
Authors []Person `xml:"title-info>author"`
Lang string `xml:"title-info>lang"`
SrcLang *string `xml:"title-info>src-lang"`
Translators *[]Person `xml:"title-info>translator"`
Sequence Sequences `xml:"title-info>sequence"`
Year *int `xml:"publish-info>year"`
ISBN *string `xml:"publish-info>isbn"`
Publisher *string `xml:"publish-info>publisher"`
Cover struct {
Id *string `xml:"href,attr"`
} `xml:"title-info>coverpage>image"`
Annotation struct {
Html string `xml:",innerxml"`
} `xml:"title-info>annotation"`
}
type FB2 struct {
SrcFile string
Bookcase *string
Title string
Genres []string
Authors []Person
HasCover bool
Lang string
SrcLang *string
Translators *[]Person
Sequence Sequences
Year *int
ISBN *string
Publisher *string
Annotation *string
SymbolsCount int
Size uint64
Hash *string
}
+150
View File
@@ -0,0 +1,150 @@
package reaper
import (
"encoding/xml"
"io"
"golang.org/x/net/html/charset"
"gorm.io/gorm"
)
func Parse(filereader io.Reader) *FB2Read {
// legacy, but normal algo
bookXML := new(FB2Read)
decoder := xml.NewDecoder(filereader)
decoder.CharsetReader = charset.NewReaderLabel
for t, _ := decoder.Token(); t != nil; t, _ = decoder.Token() {
if se, ok := t.(xml.StartElement); ok {
if se.Name.Local == "description" {
decoder.DecodeElement(&bookXML, &se)
break
}
}
}
if bookXML.Title == "" {
return nil
}
return bookXML
}
func RawToFB2(
reaped FB2Read,
filename string,
bookcase *string,
size uint64,
hash *string,
) FB2 {
return FB2{
SrcFile: filename,
Bookcase: bookcase,
Title: reaped.Title,
Genres: reaped.Genres,
Authors: reaped.Authors,
HasCover: reaped.Cover.Id != nil,
Lang: reaped.Lang,
SrcLang: reaped.SrcLang,
Translators: reaped.Translators,
Sequence: reaped.Sequence,
Year: reaped.Year,
ISBN: reaped.ISBN,
Publisher: reaped.Publisher,
Annotation: &reaped.Annotation.Html,
SymbolsCount: 0,
Size: size,
Hash: hash,
}
}
// omg legacy here ->
func nilCheck(nilString *string) string {
if nilString == nil {
return ""
}
return *nilString
}
func FB2toDB(tx *gorm.DB, book FB2) uint64 {
var genres *[]Genre
//book.Genres = nil
if book.Genres != nil {
var genresNN []Genre
for _, genre := range book.Genres {
genresNN = append(genresNN, Genre{
RawTag: genre,
})
}
genres = &genresNN
}
var authors []Author
for _, author := range book.Authors {
var dbAuthor Author
tx.FirstOrCreate(&dbAuthor, Author{
Key: author.FirstName + nilCheck(author.MiddleName) + nilCheck(author.LastName),
FirstName: author.FirstName,
MiddleName: author.MiddleName,
LastName: author.LastName,
})
authors = append(authors, dbAuthor)
}
var translators []Translator
if book.Translators != nil {
for _, translator := range *book.Translators {
var dbTranslator Translator
tx.FirstOrCreate(&dbTranslator, Translator{
Key: translator.FirstName + nilCheck(translator.MiddleName) + nilCheck(translator.LastName),
FirstName: translator.FirstName,
MiddleName: translator.MiddleName,
LastName: translator.LastName,
})
translators = append(translators, dbTranslator)
}
}
var sequence *Sequence
if book.Sequence.Name != nil {
sequence = &Sequence{
Name: *book.Sequence.Name,
}
}
var publisher *Publisher
if book.Publisher != nil {
publisher = &Publisher{
Name: *book.Publisher,
}
}
// var filetype schemas.Filetype
// tx.FirstOrCreate(&filetype, schemas.Filetype{
// Filetype: "fb2",
// Name: "FB2",
// })
//fmt.Println(book.Lang)
dbBook := Book{
Title: book.Title,
Authors: authors,
Language: Language{
Code: book.Lang,
},
Genre: genres,
Description: book.Annotation,
HasCover: book.HasCover,
SequenceID: sequence,
SequenceBook: book.Sequence.Number,
IsTranslated: book.Translators != nil,
Translators: &translators,
SrcLanguage: Language{
Code: book.Lang,
},
Year: book.Year,
Isbn: book.ISBN,
PublisherID: publisher,
SymbolsCount: &book.SymbolsCount,
Size: int(book.Size),
Hash: book.Hash,
Bookcase: book.Bookcase,
Filename: book.SrcFile,
Filetype: "fb2",
}
tx.Create(&dbBook)
return dbBook.ID
}
+122
View File
@@ -0,0 +1,122 @@
package reaper
import "time"
type Language struct {
ID uint `gorm:",unique;autoIncrement:true"`
Code string `gorm:"primaryKey;index:,unique"`
ISO *string
}
type Genre struct {
ID uint `gorm:",unique;autoIncrement:true"`
RawTag string `gorm:"index:,unique"`
Tag *string
Name *string
}
type Author struct {
ID uint `gorm:"primaryKey;index:,unique;autoIncrement:true"`
Key string `gorm:",unique"`
FirstName string
MiddleName *string
LastName *string
IsBanned *bool
BanReason *string
Books []Book `gorm:"many2many:BookAuthor"`
}
type User struct {
ID uint `gorm:"primaryKey"`
Username string `gorm:"autoIncrement:false;index:,unique"`
Avatar *string
Name *string
Admin bool
Lang *string
Language *Language `gorm:"foreignKey:Lang"`
LastSeen time.Time
Created time.Time
Password string
OTPToken *string
//Favorites []Book `gorm:"many2many:FavoriteBook;ForeignKey:username"`
BookShelf []ReaderBook
}
type Translator struct {
ID uint `gorm:"primaryKey;index:,unique;autoIncrement:true"`
Key string `gorm:",unique"`
FirstName string
MiddleName *string
LastName *string
}
type Sequence struct {
ID uint `gorm:",unique;autoIncrement:true"`
Name string `gorm:"primaryKey;index:,unique"`
}
type Publisher struct {
ID uint `gorm:",unique;autoIncrement:true"`
Name string `gorm:"primaryKey;index:,unique"`
}
type Book struct {
ID uint64 `gorm:"primaryKey"`
Title string
Authors []Author `gorm:"many2many:BookAuthor;References:ID"`
Lang *string
Language Language `gorm:"foreignKey:Lang"`
Genre *[]Genre `gorm:"many2many:BookGenre;References:RawTag"`
Description *string `gorm:"type:text"`
HasCover bool
IsTranslated bool
Translators *[]Translator `gorm:"many2many:BookTranslator;References:ID"`
SrcLang *string
SrcLanguage Language `gorm:"foreignKey:SrcLang"`
Sequence *string
SequenceID *Sequence `gorm:"foreignKey:Sequence"`
SequenceBook *int
Year *int
Publisher *string
PublisherID *Publisher `gorm:"foreignKey:Publisher"`
Isbn *string
Downloads int `gorm:"default:0"`
Views int `gorm:"default:0"`
SymbolsCount *int
PagesCount *int // pdf
Size int
Hash *string
Bookcase *string
Filename string
// FiletypeID uint
Filetype string
UploadedByID *uint
UploadedBy *User
UploadedAt *time.Time
Collections []Collection `gorm:"many2many:CollectionBook;"`
ExternalCover *string
}
// type Filetype struct {
// ID uint `gorm:"primaryKey"`
// Filetype string `gorm:"uniqueIndex"`
// Name string
// }
type ReaderBook struct {
ID uint `gorm:"primaryKey"`
UserID uint
BookID uint
Book Book `gorm:"foreignKey:BookID"`
Progress float64
LastRead time.Time
}
type Collection struct {
ID uint `gorm:"primaryKey"`
Link string `gorm:"uniqueIndex"`
Name string
UserID uint
Creator User `gorm:"foreignKey:UserID"`
Books []Book `gorm:"many2many:CollectionBook;"`
Created time.Time
Modified time.Time
}