Download everything with retries on failures
This commit is contained in:
parent
3f6bf86f2a
commit
cecdd22d99
|
@ -15,3 +15,5 @@
|
||||||
# Dependency directories (remove the comment below to include it)
|
# Dependency directories (remove the comment below to include it)
|
||||||
# vendor/
|
# vendor/
|
||||||
|
|
||||||
|
# Test outputs
|
||||||
|
Output
|
||||||
|
|
132
pkg/cli/cli.go
132
pkg/cli/cli.go
|
@ -15,6 +15,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"code.fmartingr.dev/fmartingr/go-mangadex"
|
"code.fmartingr.dev/fmartingr/go-mangadex"
|
||||||
|
"code.fmartingr.dev/fmartingr/mangadex2cbr/pkg/tasks"
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -23,6 +24,7 @@ type CliOPtions struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
const defaultLogLevel string = "INFO"
|
const defaultLogLevel string = "INFO"
|
||||||
|
const defaultLanguage string = "gb"
|
||||||
|
|
||||||
func fileExists(filename string) bool {
|
func fileExists(filename string) bool {
|
||||||
info, err := os.Stat(filename)
|
info, err := os.Stat(filename)
|
||||||
|
@ -40,7 +42,7 @@ func DownloadFile(url string, destinationPath string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
if response.StatusCode != 200 {
|
if response.StatusCode != 200 {
|
||||||
logrus.Errorf("Status code not OK: %d", response.StatusCode)
|
//logrus.Errorf("Status code not OK: %d", response.StatusCode)
|
||||||
return errors.New("Status code not 200")
|
return errors.New("Status code not 200")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -62,7 +64,10 @@ func DownloadFile(url string, destinationPath string) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
file.Write(body)
|
_, errWrite := file.Write(body)
|
||||||
|
if errWrite != nil {
|
||||||
|
logrus.Errorf("Error writting image: %s", errWrite)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -71,6 +76,7 @@ func Start() {
|
||||||
logLevelFlag := flag.String("log-level", defaultLogLevel, "Log level")
|
logLevelFlag := flag.String("log-level", defaultLogLevel, "Log level")
|
||||||
mangaIDFlag := flag.Int("manga-id", 0, "Manga ID to convert")
|
mangaIDFlag := flag.Int("manga-id", 0, "Manga ID to convert")
|
||||||
noCacheFlag := flag.Bool("no-cache", false, "Cache requests to mangadex")
|
noCacheFlag := flag.Bool("no-cache", false, "Cache requests to mangadex")
|
||||||
|
languageFlag := flag.String("language", defaultLanguage, "Language to fetch chapters")
|
||||||
outputPath := "Output"
|
outputPath := "Output"
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
|
@ -108,18 +114,27 @@ func Start() {
|
||||||
var mangaChapters []mangadex.MangaChapterList
|
var mangaChapters []mangadex.MangaChapterList
|
||||||
mangaGroups := map[int]mangadex.MangaGroup{}
|
mangaGroups := map[int]mangadex.MangaGroup{}
|
||||||
chapterParams := mangadex.NewGetChaptersParams()
|
chapterParams := mangadex.NewGetChaptersParams()
|
||||||
currentPage := 1
|
|
||||||
|
|
||||||
// TODO: Select language
|
selectedLanguage := *languageFlag
|
||||||
selectedLanguage := "gb"
|
var fetchChaptersTasks []tasks.Task
|
||||||
|
|
||||||
|
fetchChaptersTasks = tasks.TaskPush(fetchChaptersTasks, tasks.Task{Arguments: map[string]string{"page": "1"}})
|
||||||
|
|
||||||
|
for len(fetchChaptersTasks) > 0 {
|
||||||
|
var task tasks.Task
|
||||||
|
task, fetchChaptersTasks = tasks.TaskPop(fetchChaptersTasks)
|
||||||
|
logrus.Infof("Fetching chapters (page %s)", task.Arguments["page"])
|
||||||
|
page, errPage := strconv.Atoi(task.Arguments["page"])
|
||||||
|
if errPage != nil {
|
||||||
|
logrus.Error("Can't convert page to int: %s: %s", task.Arguments["page"], errPage)
|
||||||
|
}
|
||||||
|
chapterParams.Page = page
|
||||||
|
|
||||||
for currentPage != chapterParams.Page {
|
|
||||||
chapterParams.Page = currentPage
|
|
||||||
logrus.Infof("Downloading chapters page %d", currentPage)
|
|
||||||
chapters, groups, errChapters := manga.GetChapters(chapterParams)
|
chapters, groups, errChapters := manga.GetChapters(chapterParams)
|
||||||
if errChapters != nil {
|
if errChapters != nil {
|
||||||
logrus.Errorf("Error retrieving manga chapters: %s", errChapters)
|
logrus.Errorf("Error retrieving manga chapters: %s", errChapters)
|
||||||
}
|
}
|
||||||
|
|
||||||
for chapter := range chapters {
|
for chapter := range chapters {
|
||||||
if chapters[chapter].Language == selectedLanguage {
|
if chapters[chapter].Language == selectedLanguage {
|
||||||
mangaChapters = append(mangaChapters, chapters[chapter])
|
mangaChapters = append(mangaChapters, chapters[chapter])
|
||||||
|
@ -135,7 +150,7 @@ func Start() {
|
||||||
|
|
||||||
// If we have the total number of items we try the next page
|
// If we have the total number of items we try the next page
|
||||||
if len(chapters) == chapterParams.Limit {
|
if len(chapters) == chapterParams.Limit {
|
||||||
currentPage++
|
fetchChaptersTasks = tasks.TaskPush(fetchChaptersTasks, tasks.Task{Arguments: map[string]string{"page": strconv.Itoa(page + 1)}})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -144,8 +159,7 @@ func Start() {
|
||||||
logrus.Printf(" %6d: %s", mangaGroups[group].ID, mangaGroups[group].Name)
|
logrus.Printf(" %6d: %s", mangaGroups[group].ID, mangaGroups[group].Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Select groups for digitalization
|
// TODO: Select groups for digitalization, Using all for testing
|
||||||
// TODO: Using all for testing
|
|
||||||
selectedGroups := make([]int, 0, len(mangaGroups))
|
selectedGroups := make([]int, 0, len(mangaGroups))
|
||||||
for k := range mangaGroups {
|
for k := range mangaGroups {
|
||||||
selectedGroups = append(selectedGroups, k)
|
selectedGroups = append(selectedGroups, k)
|
||||||
|
@ -157,19 +171,38 @@ func Start() {
|
||||||
|
|
||||||
mangaVolumeChapter := map[string]mangadex.MangaChapterList{}
|
mangaVolumeChapter := map[string]mangadex.MangaChapterList{}
|
||||||
var mangaVolumeChapterKeys []string
|
var mangaVolumeChapterKeys []string
|
||||||
//testTest := map[string]map[string]mangadex.MangaChapter{}
|
|
||||||
|
|
||||||
|
// Using keys to get track of which chapters we already have in store, and for sorting
|
||||||
|
// through chatpers and volumes as well. Also keys are useful as page prefixes.
|
||||||
|
|
||||||
|
var downloadChaptersTasks []tasks.Task
|
||||||
for chapter := range mangaChapters {
|
for chapter := range mangaChapters {
|
||||||
|
// Sorting fix for chapters that do not contain "decimals"
|
||||||
|
// Usually chapters are just 1, 2, 3... but in some ocassions the chapters are sorted like
|
||||||
|
// 1, 1.1, 1.2, 2, ... and that make a mess when sorting. Making the "non-decimal" chapters
|
||||||
|
// fake "decimals" solves this issue easily for us.
|
||||||
|
// TODO Delete me
|
||||||
|
if mangaChapters[chapter].Volume != "1" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
if !strings.Contains(mangaChapters[chapter].Chapter, ".") {
|
if !strings.Contains(mangaChapters[chapter].Chapter, ".") {
|
||||||
mangaChapters[chapter].Chapter += ".0"
|
mangaChapters[chapter].Chapter += ".0"
|
||||||
}
|
}
|
||||||
volumeChapterKey := fmt.Sprintf("%04s_%08s", mangaChapters[chapter].Volume, mangaChapters[chapter].Chapter)
|
volumeChapterKey := fmt.Sprintf("%04s_%08s", mangaChapters[chapter].Volume, mangaChapters[chapter].Chapter)
|
||||||
_, exists := mangaVolumeChapter[volumeChapterKey]
|
_, exists := mangaVolumeChapter[volumeChapterKey]
|
||||||
if !exists {
|
if !exists {
|
||||||
//testTest[fmt.Sprintf("%04s", mangaChapters[chapter].Volume)][fmt.Sprintf("%08s", mangaChapters[chapter].Chapter)] = mangaChapters[chapter]
|
|
||||||
logrus.Debugf("Collecting volume %4s chapter %4s from group %7d", mangaChapters[chapter].Volume, mangaChapters[chapter].Chapter, mangaChapters[chapter].Groups)
|
logrus.Debugf("Collecting volume %4s chapter %4s from group %7d", mangaChapters[chapter].Volume, mangaChapters[chapter].Chapter, mangaChapters[chapter].Groups)
|
||||||
mangaVolumeChapter[volumeChapterKey] = mangaChapters[chapter]
|
mangaVolumeChapter[volumeChapterKey] = mangaChapters[chapter]
|
||||||
mangaVolumeChapterKeys = append(mangaVolumeChapterKeys, volumeChapterKey)
|
mangaVolumeChapterKeys = append(mangaVolumeChapterKeys, volumeChapterKey)
|
||||||
|
downloadChaptersTasks = tasks.TaskPush(
|
||||||
|
downloadChaptersTasks,
|
||||||
|
tasks.Task{
|
||||||
|
Arguments: map[string]string{
|
||||||
|
"chapterID": strconv.Itoa(mangaChapters[chapter].ID),
|
||||||
|
"volume": mangaChapters[chapter].Volume,
|
||||||
|
"key": volumeChapterKey,
|
||||||
|
}})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -177,42 +210,79 @@ func Start() {
|
||||||
sort.Strings(mangaVolumeChapterKeys)
|
sort.Strings(mangaVolumeChapterKeys)
|
||||||
|
|
||||||
mangaOutputPath := filepath.Join(outputPath, manga.Title)
|
mangaOutputPath := filepath.Join(outputPath, manga.Title)
|
||||||
for i := range mangaVolumeChapterKeys {
|
|
||||||
chapter := mangaVolumeChapter[mangaVolumeChapterKeys[i]]
|
logrus.Infof("Downloading chapters and calculating pages ")
|
||||||
if chapter.Volume == "1" {
|
|
||||||
|
var downloadPagesTasks []tasks.Task
|
||||||
|
|
||||||
|
for len(downloadChaptersTasks) > 0 {
|
||||||
|
var task tasks.Task
|
||||||
|
task, downloadChaptersTasks = tasks.TaskPop(downloadChaptersTasks)
|
||||||
|
|
||||||
|
// TODO: remove me
|
||||||
|
if task.Arguments["volume"] != "1" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
chapter, errChapterDetail := manga.GetChapter(task.Arguments["chapterID"])
|
||||||
|
if errChapterDetail != nil {
|
||||||
|
logrus.Errorf("Failed getting chapter detail: %s", errChapterDetail)
|
||||||
|
}
|
||||||
|
|
||||||
volumeOutputPath := filepath.Join(mangaOutputPath, fmt.Sprintf("%s - Volume %s", manga.Title, chapter.Volume))
|
volumeOutputPath := filepath.Join(mangaOutputPath, fmt.Sprintf("%s - Volume %s", manga.Title, chapter.Volume))
|
||||||
|
|
||||||
logrus.Infof("Processing Volume %s Chapter %s", chapter.Volume, chapter.Chapter)
|
logrus.Infof("Processing Volume %s Chapter %s", chapter.Volume, chapter.Chapter)
|
||||||
logrus.Tracef("Processing %s", mangaVolumeChapterKeys[i])
|
logrus.Tracef("Processing %s", task.Arguments["key"])
|
||||||
|
|
||||||
errMkdir := os.MkdirAll(volumeOutputPath, 0766)
|
errMkdir := os.MkdirAll(volumeOutputPath, 0766)
|
||||||
if errMkdir != nil {
|
if errMkdir != nil {
|
||||||
logrus.Fatalf("Error creating output directory: %s", errMkdir)
|
logrus.Fatalf("Error creating output directory: %s", errMkdir)
|
||||||
}
|
}
|
||||||
|
|
||||||
chapterDetail, errChapterDetail := manga.GetChapter(strconv.Itoa(chapter.ID))
|
for page := range chapter.Pages {
|
||||||
if errChapterDetail != nil {
|
extension := path.Ext(chapter.Pages[page])
|
||||||
logrus.Errorf("Failed getting chapter detail: %s", errChapterDetail)
|
pageFilename := fmt.Sprintf("%s_%03d%s", task.Arguments["key"], page, extension)
|
||||||
|
pageDestinationPath := filepath.Join(volumeOutputPath, pageFilename)
|
||||||
|
downloadPagesTasks = tasks.TaskPush(downloadPagesTasks, tasks.Task{
|
||||||
|
Arguments: map[string]string{
|
||||||
|
"filename": pageFilename,
|
||||||
|
"destinationPath": pageDestinationPath,
|
||||||
|
"url": chapter.Server + path.Join(chapter.Hash, chapter.Pages[page]),
|
||||||
|
"urlFallback": chapter.ServerFallback + path.Join(chapter.Hash, chapter.Pages[page]),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for page := range chapterDetail.Pages {
|
logrus.Infof("Downloading %d pages", len(downloadPagesTasks))
|
||||||
logrus.Infof("Processing page %s", chapterDetail.Pages[page])
|
|
||||||
extension := path.Ext(chapterDetail.Pages[page])
|
for len(downloadPagesTasks) > 0 {
|
||||||
pageDestinationPath := filepath.Join(volumeOutputPath, fmt.Sprintf("%s_%03d%s", mangaVolumeChapterKeys[i], page, extension))
|
var task tasks.Task
|
||||||
if !fileExists(pageDestinationPath) {
|
task, downloadPagesTasks = tasks.TaskPop(downloadPagesTasks)
|
||||||
logrus.Tracef("Downloading page from %s", chapterDetail.Server+path.Join(chapterDetail.Hash, chapterDetail.Pages[page]))
|
destinationPath := task.Arguments["destinationPath"]
|
||||||
logrus.Tracef("Downlading page to %s", pageDestinationPath)
|
if !fileExists(task.Arguments["destinationPath"]) {
|
||||||
errDownload := DownloadFile(chapterDetail.Server+path.Join(chapterDetail.Hash, chapterDetail.Pages[page]), pageDestinationPath)
|
logrus.Tracef("Downloading page from %s", task.Arguments["url"])
|
||||||
|
logrus.Tracef("Downlading page to %s", destinationPath)
|
||||||
|
errDownload := DownloadFile(task.Arguments["url"], destinationPath)
|
||||||
if errDownload != nil {
|
if errDownload != nil {
|
||||||
|
_, exists := task.Arguments["urlFallback"]
|
||||||
|
if exists {
|
||||||
|
logrus.Warnf("Using fallback for page %s", task.Arguments["filename"])
|
||||||
|
downloadPagesTasks = tasks.TaskPush(downloadPagesTasks, tasks.Task{
|
||||||
|
Arguments: map[string]string{
|
||||||
|
"destinationPath": task.Arguments["destinationPath"],
|
||||||
|
"url": task.Arguments["urlFallback"],
|
||||||
|
"key": task.Arguments["key"],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
} else {
|
||||||
logrus.Errorf("Error downloading page: %s", errDownload)
|
logrus.Errorf("Error downloading page: %s", errDownload)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
time.Sleep(100 * time.Millisecond)
|
time.Sleep(100 * time.Millisecond)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
covers, errCovers := manga.GetCovers()
|
covers, errCovers := manga.GetCovers()
|
||||||
if errCovers != nil {
|
if errCovers != nil {
|
||||||
logrus.Errorf("Error getting cover information: %s", errCovers)
|
logrus.Errorf("Error getting cover information: %s", errCovers)
|
||||||
|
|
|
@ -0,0 +1,25 @@
|
||||||
|
package tasks
|
||||||
|
|
||||||
|
const (
|
||||||
|
TaskTypeDownload = iota
|
||||||
|
TaskTypeFetch = iota
|
||||||
|
TaskTypeMkDir = iota
|
||||||
|
)
|
||||||
|
|
||||||
|
type Task struct {
|
||||||
|
Type int8
|
||||||
|
Name string
|
||||||
|
Failed bool
|
||||||
|
Arguments map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func TaskPop(taskList []Task) (Task, []Task) {
|
||||||
|
var result Task
|
||||||
|
// https://github.com/golang/go/wiki/SliceTricks
|
||||||
|
result, taskList = taskList[0], taskList[1:]
|
||||||
|
return result, taskList
|
||||||
|
}
|
||||||
|
|
||||||
|
func TaskPush(taskList []Task, task Task) []Task {
|
||||||
|
return append(taskList, task)
|
||||||
|
}
|
Loading…
Reference in New Issue