2021-02-06 22:50:43 +00:00
package cli
import (
2021-02-10 16:12:48 +00:00
"archive/zip"
2021-02-07 16:53:57 +00:00
"errors"
2021-02-06 22:50:43 +00:00
"flag"
"fmt"
2021-02-10 16:12:48 +00:00
"io"
2021-02-07 16:53:57 +00:00
"io/ioutil"
"net/http"
2021-02-06 22:50:43 +00:00
"os"
2021-02-07 16:53:57 +00:00
"path"
2021-02-06 22:50:43 +00:00
"path/filepath"
"sort"
2021-02-07 16:53:57 +00:00
"strconv"
2021-02-06 22:50:43 +00:00
"strings"
2021-02-07 16:53:57 +00:00
"time"
2021-02-06 22:50:43 +00:00
2021-02-13 16:28:57 +00:00
"github.com/fmartingr/go-mangadex"
2021-02-21 21:09:37 +00:00
"github.com/fmartingr/mangadex2cbz/pkg/tasks"
2021-02-06 22:50:43 +00:00
"github.com/sirupsen/logrus"
)
type CliOPtions struct {
MangaID string
}
const defaultLogLevel string = "INFO"
2021-02-09 22:38:18 +00:00
const defaultLanguage string = "gb"
2021-02-06 22:50:43 +00:00
2021-02-07 16:53:57 +00:00
func fileExists ( filename string ) bool {
info , err := os . Stat ( filename )
if os . IsNotExist ( err ) {
return false
}
return ! info . IsDir ( )
}
func DownloadFile ( url string , destinationPath string ) error {
response , err := http . Get ( url )
if err != nil {
logrus . Errorf ( "Error downloading page: %s" , err )
return err
}
if response . StatusCode != 200 {
2021-02-09 22:38:18 +00:00
//logrus.Errorf("Status code not OK: %d", response.StatusCode)
2021-02-07 16:53:57 +00:00
return errors . New ( "Status code not 200" )
}
if response . Body != nil {
defer response . Body . Close ( )
}
file , errOpen := os . Create ( destinationPath )
if err != nil {
logrus . Errorf ( "Error opening file: %s" , errOpen )
return errOpen
}
defer file . Close ( )
body , errBody := ioutil . ReadAll ( response . Body )
if errBody != nil {
logrus . Errorf ( "Failing read body from response: %s" , errBody )
return err
}
2021-02-09 22:38:18 +00:00
_ , errWrite := file . Write ( body )
if errWrite != nil {
2021-02-10 16:12:48 +00:00
return errWrite
2021-02-09 22:38:18 +00:00
}
2021-02-07 16:53:57 +00:00
return nil
}
2021-02-06 22:50:43 +00:00
func Start ( ) {
logLevelFlag := flag . String ( "log-level" , defaultLogLevel , "Log level" )
mangaIDFlag := flag . Int ( "manga-id" , 0 , "Manga ID to convert" )
noCacheFlag := flag . Bool ( "no-cache" , false , "Cache requests to mangadex" )
2021-02-09 22:38:18 +00:00
languageFlag := flag . String ( "language" , defaultLanguage , "Language to fetch chapters" )
2021-02-06 22:50:43 +00:00
outputPath := "Output"
flag . Parse ( )
cwd , errCwd := os . Getwd ( )
if errCwd != nil {
logrus . Fatalf ( "Error retrieving current working directory: %s" , errCwd )
}
outputPath = filepath . Join ( cwd , outputPath )
if ! * noCacheFlag {
mangadex . EnableCache ( )
}
logLevel , errLogLevel := logrus . ParseLevel ( * logLevelFlag )
if errLogLevel != nil {
logrus . Warnf ( "Incorrect loglevel %s, using default %s" , * logLevelFlag , defaultLogLevel )
} else {
logrus . SetLevel ( logLevel )
}
if * mangaIDFlag == 0 {
logrus . Error ( "You should set -manga-id" )
logrus . Exit ( 1 )
}
logrus . Infof ( "Getting information for Manga ID: %d" , * mangaIDFlag )
manga , err := mangadex . GetManga ( * mangaIDFlag )
if err != nil {
panic ( err )
}
2021-02-10 16:12:48 +00:00
// TODO: Allow selecting a name from AlternativeNames
2021-02-06 22:50:43 +00:00
logrus . Infof ( "Found! %s" , manga . Title )
logrus . Infof ( "Getting chapter information..." )
2021-02-07 16:53:57 +00:00
var mangaChapters [ ] mangadex . MangaChapterList
2021-02-06 22:50:43 +00:00
mangaGroups := map [ int ] mangadex . MangaGroup { }
2021-02-13 16:28:57 +00:00
chapterParams := mangadex . NewChaptersParams ( )
2021-02-06 22:50:43 +00:00
2021-02-09 22:38:18 +00:00
selectedLanguage := * languageFlag
var fetchChaptersTasks [ ] tasks . Task
fetchChaptersTasks = tasks . TaskPush ( fetchChaptersTasks , tasks . Task { Arguments : map [ string ] string { "page" : "1" } } )
for len ( fetchChaptersTasks ) > 0 {
var task tasks . Task
task , fetchChaptersTasks = tasks . TaskPop ( fetchChaptersTasks )
logrus . Infof ( "Fetching chapters (page %s)" , task . Arguments [ "page" ] )
page , errPage := strconv . Atoi ( task . Arguments [ "page" ] )
if errPage != nil {
logrus . Error ( "Can't convert page to int: %s: %s" , task . Arguments [ "page" ] , errPage )
}
chapterParams . Page = page
2021-02-06 22:50:43 +00:00
chapters , groups , errChapters := manga . GetChapters ( chapterParams )
if errChapters != nil {
logrus . Errorf ( "Error retrieving manga chapters: %s" , errChapters )
}
2021-02-09 22:38:18 +00:00
2021-02-06 22:50:43 +00:00
for chapter := range chapters {
if chapters [ chapter ] . Language == selectedLanguage {
mangaChapters = append ( mangaChapters , chapters [ chapter ] )
}
}
for group := range groups {
_ , exists := mangaGroups [ groups [ group ] . ID ]
if ! exists {
mangaGroups [ groups [ group ] . ID ] = groups [ group ]
}
}
// If we have the total number of items we try the next page
if len ( chapters ) == chapterParams . Limit {
2021-02-09 22:38:18 +00:00
fetchChaptersTasks = tasks . TaskPush ( fetchChaptersTasks , tasks . Task { Arguments : map [ string ] string { "page" : strconv . Itoa ( page + 1 ) } } )
2021-02-06 22:50:43 +00:00
}
}
logrus . Printf ( "Found following groups: " )
for group := range mangaGroups {
logrus . Printf ( " %6d: %s" , mangaGroups [ group ] . ID , mangaGroups [ group ] . Name )
}
2021-02-09 22:38:18 +00:00
// TODO: Select groups for digitalization, Using all for testing
2021-02-06 22:50:43 +00:00
selectedGroups := make ( [ ] int , 0 , len ( mangaGroups ) )
for k := range mangaGroups {
selectedGroups = append ( selectedGroups , k )
}
logrus . Infof ( "Selected groups: %d" , selectedGroups )
logrus . Infof ( "Calculating Volumes..." )
2021-02-07 16:53:57 +00:00
mangaVolumeChapter := map [ string ] mangadex . MangaChapterList { }
2021-02-06 22:50:43 +00:00
var mangaVolumeChapterKeys [ ] string
2021-02-09 22:38:18 +00:00
// Using keys to get track of which chapters we already have in store, and for sorting
// through chatpers and volumes as well. Also keys are useful as page prefixes.
var downloadChaptersTasks [ ] tasks . Task
2021-02-06 22:50:43 +00:00
for chapter := range mangaChapters {
2021-02-09 22:38:18 +00:00
// Sorting fix for chapters that do not contain "decimals"
// Usually chapters are just 1, 2, 3... but in some ocassions the chapters are sorted like
// 1, 1.1, 1.2, 2, ... and that make a mess when sorting. Making the "non-decimal" chapters
// fake "decimals" solves this issue easily for us.
2021-02-10 16:12:48 +00:00
2021-02-09 22:38:18 +00:00
// TODO Delete me
2021-02-10 16:12:48 +00:00
// if mangaChapters[chapter].Volume != "1" {
// continue
// }
2021-02-09 22:38:18 +00:00
2021-02-06 22:50:43 +00:00
if ! strings . Contains ( mangaChapters [ chapter ] . Chapter , "." ) {
mangaChapters [ chapter ] . Chapter += ".0"
}
volumeChapterKey := fmt . Sprintf ( "%04s_%08s" , mangaChapters [ chapter ] . Volume , mangaChapters [ chapter ] . Chapter )
_ , exists := mangaVolumeChapter [ volumeChapterKey ]
if ! exists {
logrus . Debugf ( "Collecting volume %4s chapter %4s from group %7d" , mangaChapters [ chapter ] . Volume , mangaChapters [ chapter ] . Chapter , mangaChapters [ chapter ] . Groups )
mangaVolumeChapter [ volumeChapterKey ] = mangaChapters [ chapter ]
mangaVolumeChapterKeys = append ( mangaVolumeChapterKeys , volumeChapterKey )
2021-02-09 22:38:18 +00:00
downloadChaptersTasks = tasks . TaskPush (
downloadChaptersTasks ,
tasks . Task {
Arguments : map [ string ] string {
"chapterID" : strconv . Itoa ( mangaChapters [ chapter ] . ID ) ,
"volume" : mangaChapters [ chapter ] . Volume ,
"key" : volumeChapterKey ,
} } )
2021-02-06 22:50:43 +00:00
}
}
logrus . Debugf ( "Sorting by volume and chapter" )
sort . Strings ( mangaVolumeChapterKeys )
2021-02-07 16:53:57 +00:00
mangaOutputPath := filepath . Join ( outputPath , manga . Title )
2021-02-09 22:38:18 +00:00
logrus . Infof ( "Downloading chapters and calculating pages " )
2021-02-07 16:53:57 +00:00
2021-02-09 22:38:18 +00:00
var downloadPagesTasks [ ] tasks . Task
2021-02-07 16:53:57 +00:00
2021-02-09 22:38:18 +00:00
for len ( downloadChaptersTasks ) > 0 {
var task tasks . Task
task , downloadChaptersTasks = tasks . TaskPop ( downloadChaptersTasks )
2021-02-10 16:12:48 +00:00
// TODO: Delete me
// if task.Arguments["volume"] != "1" {
// continue
// }
2021-02-09 22:38:18 +00:00
chapter , errChapterDetail := manga . GetChapter ( task . Arguments [ "chapterID" ] )
if errChapterDetail != nil {
logrus . Errorf ( "Failed getting chapter detail: %s" , errChapterDetail )
}
2021-02-07 16:53:57 +00:00
2021-02-09 22:38:18 +00:00
volumeOutputPath := filepath . Join ( mangaOutputPath , fmt . Sprintf ( "%s - Volume %s" , manga . Title , chapter . Volume ) )
logrus . Infof ( "Processing Volume %s Chapter %s" , chapter . Volume , chapter . Chapter )
logrus . Tracef ( "Processing %s" , task . Arguments [ "key" ] )
errMkdir := os . MkdirAll ( volumeOutputPath , 0766 )
if errMkdir != nil {
logrus . Fatalf ( "Error creating output directory: %s" , errMkdir )
}
for page := range chapter . Pages {
extension := path . Ext ( chapter . Pages [ page ] )
pageFilename := fmt . Sprintf ( "%s_%03d%s" , task . Arguments [ "key" ] , page , extension )
pageDestinationPath := filepath . Join ( volumeOutputPath , pageFilename )
downloadPagesTasks = tasks . TaskPush ( downloadPagesTasks , tasks . Task {
Arguments : map [ string ] string {
"filename" : pageFilename ,
"destinationPath" : pageDestinationPath ,
"url" : chapter . Server + path . Join ( chapter . Hash , chapter . Pages [ page ] ) ,
"urlFallback" : chapter . ServerFallback + path . Join ( chapter . Hash , chapter . Pages [ page ] ) ,
} ,
} )
}
}
logrus . Infof ( "Downloading %d pages" , len ( downloadPagesTasks ) )
for len ( downloadPagesTasks ) > 0 {
var task tasks . Task
task , downloadPagesTasks = tasks . TaskPop ( downloadPagesTasks )
destinationPath := task . Arguments [ "destinationPath" ]
if ! fileExists ( task . Arguments [ "destinationPath" ] ) {
logrus . Tracef ( "Downloading page from %s" , task . Arguments [ "url" ] )
logrus . Tracef ( "Downlading page to %s" , destinationPath )
errDownload := DownloadFile ( task . Arguments [ "url" ] , destinationPath )
if errDownload != nil {
2021-02-10 16:12:48 +00:00
// TODO: Distinguish between 420 errors (GONE -> Use fallback) and other errors to keep retrying on the
// original (potentially MangaDex@HOME) urls.
2021-02-09 22:38:18 +00:00
_ , exists := task . Arguments [ "urlFallback" ]
if exists {
logrus . Warnf ( "Using fallback for page %s" , task . Arguments [ "filename" ] )
downloadPagesTasks = tasks . TaskPush ( downloadPagesTasks , tasks . Task {
Arguments : map [ string ] string {
"destinationPath" : task . Arguments [ "destinationPath" ] ,
"url" : task . Arguments [ "urlFallback" ] ,
"key" : task . Arguments [ "key" ] ,
} ,
} )
} else {
logrus . Errorf ( "Error downloading page: %s" , errDownload )
2021-02-07 16:53:57 +00:00
}
}
2021-02-10 16:12:48 +00:00
time . Sleep ( 100 * time . Millisecond )
2021-02-07 16:53:57 +00:00
}
}
2021-02-10 16:12:48 +00:00
logrus . Info ( "Downloading covers" )
2021-02-07 16:53:57 +00:00
covers , errCovers := manga . GetCovers ( )
if errCovers != nil {
logrus . Errorf ( "Error getting cover information: %s" , errCovers )
}
for cover := range covers {
2021-02-10 16:12:48 +00:00
extension := path . Ext ( covers [ cover ] . URL )
coverOutputPath := filepath . Join ( mangaOutputPath , fmt . Sprintf ( "%s - Volume %s" , manga . Title , covers [ cover ] . Volume ) , fmt . Sprintf ( "0000%s" , extension ) )
if ! fileExists ( coverOutputPath ) {
logrus . Tracef ( "Downloading cover from %s" , covers [ cover ] . URL )
logrus . Tracef ( "Writting cover to: %s" , coverOutputPath )
errDownload := DownloadFile ( covers [ cover ] . URL , coverOutputPath )
if errDownload != nil {
logrus . Errorf ( "Error downloading cover: %s" , errDownload )
2021-02-07 16:53:57 +00:00
}
time . Sleep ( 100 * time . Millisecond )
}
2021-02-06 22:50:43 +00:00
}
2021-02-10 16:12:48 +00:00
logrus . Info ( "Compressing volumes" )
files , errReadVolumes := ioutil . ReadDir ( mangaOutputPath )
if errReadVolumes != nil {
logrus . Errorf ( "Error reading path with volume files: %s" , errReadVolumes )
}
for file := range files {
if files [ file ] . IsDir ( ) {
2021-02-21 21:09:37 +00:00
destinationPath := filepath . Join ( mangaOutputPath , files [ file ] . Name ( ) ) + ".cbz"
2021-02-10 16:12:48 +00:00
if ! fileExists ( destinationPath ) {
_ , errZip := zipVolume ( destinationPath )
if errZip != nil {
2021-02-21 21:09:37 +00:00
logrus . Errorf ( "Error packing CB file: %s" , errZip )
2021-02-10 16:12:48 +00:00
}
}
}
}
}
func zipVolume ( destinationPath string ) ( string , error ) {
logrus . Tracef ( "Packing volume %s" , path . Base ( destinationPath ) )
pagesPath := strings . Replace ( destinationPath , path . Ext ( destinationPath ) , "" , 1 )
newZipFile , errCreate := os . Create ( destinationPath )
if errCreate != nil {
return "" , errCreate
}
defer newZipFile . Close ( )
zipWriter := zip . NewWriter ( newZipFile )
defer zipWriter . Close ( )
files , errReadFiles := ioutil . ReadDir ( pagesPath )
if errReadFiles != nil {
return "" , errReadFiles
}
for _ , file := range files {
if errAddFile := AddFileToZip ( zipWriter , filepath . Join ( pagesPath , file . Name ( ) ) ) ; errAddFile != nil {
return "" , errAddFile
}
}
return "" , nil
}
func AddFileToZip ( zipWriter * zip . Writer , filename string ) error {
fileToZip , err := os . Open ( filename )
if err != nil {
return err
}
defer fileToZip . Close ( )
// Get the file information
info , err := fileToZip . Stat ( )
if err != nil {
return err
}
header , err := zip . FileInfoHeader ( info )
if err != nil {
return err
}
// Using FileInfoHeader() above only uses the basename of the file. If we want
// to preserve the folder structure we can overwrite this with the full path.
header . Name = filename
// Change to deflate to gain better compression
// see http://golang.org/pkg/archive/zip/#pkg-constants
header . Method = zip . Deflate
writer , err := zipWriter . CreateHeader ( header )
if err != nil {
return err
}
_ , err = io . Copy ( writer , fileToZip )
return err
2021-02-06 22:50:43 +00:00
}