From 657b8e1f44ea6117092ee7026d0da19c8abfc916 Mon Sep 17 00:00:00 2001 From: Schemen Date: Sun, 19 Nov 2017 23:35:36 +0100 Subject: [PATCH 01/28] Update commands & README --- README.md | 33 ++++++++++++++++++++------------- m2em.py | 35 ++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index cf6a87e..0343673 100644 --- a/README.md +++ b/README.md @@ -55,27 +55,32 @@ With the sourceparser you can add support of a Webhost. ### Help output: ``` -usage: m2em.py [-h] [-r RSS_FEED] [-u] [-l] [-L] [--list-feeds] [--list-users] - [-cd] [-s SWITCH_SEND] [-S SWITCH_CHAPTER] [-dc DELETE_CHAPTER] - [-du DELETE_USER] [-df DELETE_FEED] [--daemon] [-d] +usage: m2em.py [-h] [-af ADD_FEED] [-au] [-lc] [-Lc] [-lf] [-lu] [-cd] [-a] + [-s] [-ss SWITCH_SEND] [-sc SWITCH_CHAPTER] + [-dc DELETE_CHAPTER] [-du DELETE_USER] [-df DELETE_FEED] + [--daemon] [-d] Manga to eManga - m2em optional arguments: -h, --help show this help message and exit - -r RSS_FEED, --rss-feed RSS_FEED + -af ADD_FEED, --add-feed ADD_FEED Add RSS Feed of Manga. Only Mangastream & MangaFox are supported - -u, --add-user Adds new user - -l, --list-chapters Lists the last 10 Chapters - -L, --list-chapters-all + -au, --add-user Adds new user + -lc, --list-chapters Lists the last 10 Chapters + -Lc, --list-chapters-all Lists all Chapters - --list-feeds Lists all feeds - --list-users Lists all Users + -lf, --list-feeds Lists all feeds + -lu, --list-users Lists all Users -cd, --create-db Creates DB. Uses Configfile for Naming - -s SWITCH_SEND, --switch-send SWITCH_SEND + -a, --action Start action. Options are: rss (collecting feed data), + downloader, converter or sender + -s, --start Start a single loop. If no arguments are passed, a + single loop will also start + -ss SWITCH_SEND, --switch-send SWITCH_SEND Pass ID of User. Switches said user Send eBook status - -S SWITCH_CHAPTER, --switch-chapter SWITCH_CHAPTER + -sc SWITCH_CHAPTER, --switch-chapter SWITCH_CHAPTER Pass ID of Chapter. Switches said Chapter Sent status -dc DELETE_CHAPTER, --delete-chapter DELETE_CHAPTER Pass ID of Chapter. Deletes said Chapter @@ -91,7 +96,7 @@ optional arguments: To have a working environment you need to add some initial data and create the database ```x-sh ./m2em.py --create-db # Create a DB -./m2em.py --rss-feed # Add an RSS Feed you want to pull +./m2em.py --add-feed # Add an RSS Feed you want to pull # Please note that you should set the sending AFTER a complete run for now ./m2em.py --add-user # Add a user @@ -130,7 +135,9 @@ To start a single run through the workers, you can simply execute the main progr ``` If you wish to run the program as a daemon, start it with the option "--daemon". It will re-run at the config "Sleep" in second. - +``` +./m2em.py --daemon +``` ### A complete run with nothing happening: ``` diff --git a/m2em.py b/m2em.py index b689fc0..dcd6a02 100755 --- a/m2em.py +++ b/m2em.py @@ -34,6 +34,7 @@ def __init__(self): self.config = None if not self.config: self.read_config() + # Check if Database exists, else create if not os.path.isfile(self.config["Database"]): helper.createDB(self.config) @@ -43,21 +44,25 @@ def read_arguments(self): # Get user Input parser = argparse.ArgumentParser(description='Manga to eManga - m2em') - parser.add_argument("-r", "--rss-feed", help="Add RSS Feed of Manga. Only Mangastream & MangaFox are supported") - parser.add_argument("-u", "--add-user", help="Adds new user", + parser.add_argument("-af", "--add-feed", help="Add RSS Feed of Manga. Only Mangastream & MangaFox are supported") + parser.add_argument("-au", "--add-user", help="Adds new user", action="store_true") - parser.add_argument("-l", "--list-chapters", help="Lists the last 10 Chapters", + parser.add_argument("-lc", "--list-chapters", help="Lists the last 10 Chapters", action="store_true") - parser.add_argument("-L", "--list-chapters-all", help="Lists all Chapters", + parser.add_argument("-Lc", "--list-chapters-all", help="Lists all Chapters", action="store_true") - parser.add_argument("--list-feeds", help="Lists all feeds", + parser.add_argument("-lf", "--list-feeds", help="Lists all feeds", action="store_true") - parser.add_argument("--list-users", help="Lists all Users", + parser.add_argument("-lu", "--list-users", help="Lists all Users", action="store_true") parser.add_argument("-cd", "--create-db", help="Creates DB. Uses Configfile for Naming", action="store_true") - parser.add_argument("-s", "--switch-send", help="Pass ID of User. Switches said user Send eBook status") - parser.add_argument("-S", "--switch-chapter", help="Pass ID of Chapter. Switches said Chapter Sent status") + parser.add_argument("-a", "--action", help="Start action. Options are: rss (collecting feed data), downloader, converter or sender ", + action="store_true") + parser.add_argument("-s", "--start", help="Start a single loop. If no arguments are passed, a single loop will also start", + action="store_true") + parser.add_argument("-ss", "--switch-send", help="Pass ID of User. Switches said user Send eBook status") + parser.add_argument("-sc", "--switch-chapter", help="Pass ID of Chapter. Switches said Chapter Sent status") parser.add_argument("-dc", "--delete-chapter", help="Pass ID of Chapter. Deletes said Chapter") parser.add_argument("-du", "--delete-user", help="Pass ID of User. Deletes said User") parser.add_argument("-df", "--delete-feed", help="Pass ID of Feed. Deletes said Feed") @@ -94,12 +99,12 @@ def read_config(self): ''' - Catch -r/--add-rss + Catch -af/--add-feed ''' def save_feed_to_db(self): - logging.debug("Entered URL: %s" % self.args.rss_feed) - if validators.url(self.args.rss_feed): - helper.writeFeed(self.args.rss_feed, self.config) + logging.debug("Entered URL: %s" % self.args.add_feed) + if validators.url(self.args.add_feed): + helper.writeFeed(self.args.add_feed, self.config) else: logging.error("You need to enter an URL!") @@ -191,7 +196,7 @@ def create_db(self): This are the worker, one round ''' # Worker to get and parse rss feeds - def parse_rss_feeds(self): + def parse_add_feeds(self): mparser.RssParser(self.config) # Worker to fetch all images @@ -213,7 +218,7 @@ def send_ebooks(self): ''' def run(self): - if self.args.rss_feed: + if self.args.add_feed: self.save_feed_to_db() return @@ -275,7 +280,7 @@ def run(self): logging.info("Starting Loop at %s" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) logging.info("Starting RSS Data Fetcher!") - self.parse_rss_feeds() + self.parse_add_feeds() logging.info("Finished Loading RSS Data") logging.info("Starting all outstanding Chapter Downloads!") From 837071e71413ca539abef1ff16f8b21672260229 Mon Sep 17 00:00:00 2001 From: Schemen Date: Sun, 19 Nov 2017 23:36:24 +0100 Subject: [PATCH 02/28] Start rework on independet worker & loop Downloader LOOP now only downloads chapters that are younger than 1 day --- bin/m2emDownloader.py | 146 ++++++++++++++++++++++-------------------- bin/m2emHelper.py | 19 ++++++ 2 files changed, 96 insertions(+), 69 deletions(-) diff --git a/bin/m2emDownloader.py b/bin/m2emDownloader.py index 8b52803..0e51ff3 100644 --- a/bin/m2emDownloader.py +++ b/bin/m2emDownloader.py @@ -30,100 +30,108 @@ def ChapterDownloader(config): mangapages = chapter[9] mangatitle = chapter[2] manganame = chapter[11] + chapterdate = chapter[3] - # check if mangatitle contains ":" characters that OS can't handle as folders - mangatitle = helper.sanetizeName(mangatitle) - # check if manganame contains ":" characters that OS can't handle as folders - manganame = helper.sanetizeName(manganame) + # Only start run if chapter is younger than 24h + if helper.checkTime(chapterdate): - # Old Download folder from v0.1.0 - oldlocation = str(saveloc + mangatitle) - newlocation = str(saveloc + manganame) - # Define Download location - downloadfolder = str(saveloc + manganame + "/" + mangatitle + "/images") + # check if mangatitle contains ":" characters that OS can't handle as folders + mangatitle = helper.sanetizeName(mangatitle) + # check if manganame contains ":" characters that OS can't handle as folders + manganame = helper.sanetizeName(manganame) - # Check if the old DL location is being used - if os.path.isdir(oldlocation): - logging.info("Moving old DL location to new one") - helper.createFolder(newlocation) - move(oldlocation, newlocation) + # Old Download folder from v0.1.0 + oldlocation = str(saveloc + mangatitle) + newlocation = str(saveloc + manganame) + # Define Download location + downloadfolder = str(saveloc + manganame + "/" + mangatitle + "/images") - if os.path.isdir(downloadfolder): - logging.debug("Manga %s downloaded already!" % mangatitle) - else: - logging.info("Starting download of %s..." % mangatitle) + # Check if the old DL location is being used + if os.path.isdir(oldlocation): + logging.info("Moving old DL location to new one") + helper.createFolder(newlocation) + move(oldlocation, newlocation) - # get Origin of manga - origin = helper.getSourceURL(mangastarturl) + if os.path.isdir(downloadfolder): + logging.debug("Manga %s downloaded already!" % mangatitle) + else: + logging.info("Starting download of %s..." % mangatitle) - # Get image urls! - if origin == "mangastream.com": - urllist = msparser.getPagesUrl(mangastarturl,mangapages) + # get Origin of manga + origin = helper.getSourceURL(mangastarturl) - # Turn Manga pages into Image links! - imageurls=[] - for i in urllist: - imageurls.append(msparser.getImageUrl(i)) - logging.debug("List of all Images for %s" % mangatitle) - logging.debug(imageurls) - - # Mangafox Parser - elif origin == "mangafox.me": - urllist = mxparser.getPagesUrl(mangastarturl,mangapages) + # Get image urls! + # Mangastream Parser + if origin == "mangastream.com": + urllist = msparser.getPagesUrl(mangastarturl,mangapages) - # Turn Manga pages into Image links! - imageurls=[] - for i in urllist: - imageurls.append(mxparser.getImageUrl(i)) - logging.debug("List of all Images for %s" % mangatitle) - logging.debug(imageurls) + # Turn Manga pages into Image links! + imageurls=[] + for i in urllist: + imageurls.append(msparser.getImageUrl(i)) + logging.debug("List of all Images for %s" % mangatitle) + logging.debug(imageurls) - else: - pass + + # Mangafox Parser + elif origin == "mangafox.me": + urllist = mxparser.getPagesUrl(mangastarturl,mangapages) + + + # Turn Manga pages into Image links! + imageurls=[] + for i in urllist: + imageurls.append(mxparser.getImageUrl(i)) + logging.debug("List of all Images for %s" % mangatitle) + logging.debug(imageurls) + + else: + pass - # Download & save images! - # check if we have images to download - if not len(imageurls) == 0: - helper.createFolder(downloadfolder) - counter = 0 - for image in imageurls: - counter = counter + 1 + # Download & save images! + # check if we have images to download + if not len(imageurls) == 0: + helper.createFolder(downloadfolder) + counter = 0 + for image in imageurls: + counter = counter + 1 - imagepath = downloadfolder + "/" + str("{0:0=3d}".format(counter)) + ".png" + imagepath = downloadfolder + "/" + str("{0:0=3d}".format(counter)) + ".png" - f = open(imagepath, 'wb') - f.write(requests.get(image).content) - f.close + f = open(imagepath, 'wb') + f.write(requests.get(image).content) + f.close - # Cleanse image, remove footer - # - # I have borrowed this code from the kmanga project. - # https://github.com/aplanas/kmanga/blob/master/mobi/mobi.py#L416 - # Thanks a lot to Alberto Planas for coming up with it! - # - if origin == "mangafox.me": - logging.debug("Cleaning Mangafox") - img = Image.open(imagepath) - _img = ImageOps.invert(img.convert(mode='L')) - _img = _img.point(lambda x: x and 255) - _img = _img.filter(ImageFilter.MinFilter(size=3)) - _img = _img.filter(ImageFilter.GaussianBlur(radius=5)) - _img = _img.point(lambda x: (x >= 48) and x) + # Cleanse image, remove footer + # + # I have borrowed this code from the kmanga project. + # https://github.com/aplanas/kmanga/blob/master/mobi/mobi.py#L416 + # Thanks a lot to Alberto Planas for coming up with it! + # + if origin == "mangafox.me": + logging.debug("Cleaning Mangafox Footer") + img = Image.open(imagepath) + _img = ImageOps.invert(img.convert(mode='L')) + _img = _img.point(lambda x: x and 255) + _img = _img.filter(ImageFilter.MinFilter(size=3)) + _img = _img.filter(ImageFilter.GaussianBlur(radius=5)) + _img = _img.point(lambda x: (x >= 48) and x) - cleaned = img.crop(_img.getbbox()) if _img.getbbox() else img - cleaned.save(imagepath) + cleaned = img.crop(_img.getbbox()) if _img.getbbox() else img + cleaned.save(imagepath) - logging.info("Finished download!") \ No newline at end of file + logging.info("Finished download!") + logging.debug("Chapter %s is older than 1 day. Skipping in the loop..." % mangatitle) \ No newline at end of file diff --git a/bin/m2emHelper.py b/bin/m2emHelper.py index faefd19..b8c2c0e 100644 --- a/bin/m2emHelper.py +++ b/bin/m2emHelper.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import logging +import datetime import os import sqlite3 import texttable @@ -794,3 +795,21 @@ def sanetizeName(name): return name else: return name + + + + +''' +Check if time is older than 24h +Returns: true or false +''' +def checkTime(time): + objecttime = datetime.datetime.strptime(time, "%a, %d %b %Y %H:%M:%S %z") + now = datetime.datetime.now(datetime.timezone.utc) + + delta = now - objecttime + + if delta.days == 0: + return True + else: + return False \ No newline at end of file From 67b1a3421999d7d7793893b750e3ebee55fe08a9 Mon Sep 17 00:00:00 2001 From: Schemen Date: Mon, 20 Nov 2017 17:39:19 +0100 Subject: [PATCH 03/28] extend logging to be able to log to stdout and file Add different logging methods to file, to stdout --- .gitignore | 3 ++- bin/m2emHelper.py | 44 +++++++++++++++++++++++++++++++++++++++++++- m2em.py | 6 ++++-- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 8b27c39..2a0988d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ comic/* kindlegen.exe .idea venv -main.db \ No newline at end of file +main.db +log/* \ No newline at end of file diff --git a/bin/m2emHelper.py b/bin/m2emHelper.py index b8c2c0e..afd9004 100644 --- a/bin/m2emHelper.py +++ b/bin/m2emHelper.py @@ -812,4 +812,46 @@ def checkTime(time): if delta.days == 0: return True else: - return False \ No newline at end of file + return False + + +''' + +Init Logging! + +''' +def initialize_logger(output_dir, outputlevel): + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + # create console handler and set level to info + handler = logging.StreamHandler() + if outputlevel == "debug": + handler.setLevel(logging.DEBUG) + formatter = logging.Formatter("%(levelname)s: %(message)s") + else: + handler.setLevel(logging.INFO) + formatter = logging.Formatter("%(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) + + # create error file handler and set level to error + handler = logging.FileHandler(os.path.join(output_dir, "error.log"), encoding=None, delay="true") + handler.setLevel(logging.ERROR) + formatter = logging.Formatter("%(asctime)s; %(levelname)s - %(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) + + # create debug file handler and set level to debug + handler = logging.FileHandler(os.path.join(output_dir, "all.log")) + handler.setLevel(logging.DEBUG) + formatter = logging.Formatter("%(asctime)s; %(levelname)s - %(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) + + # create debug file handler and set level to debug - per run 1 file + handler = logging.FileHandler(os.path.join(output_dir, "run.log"), "w") + handler.setLevel(logging.DEBUG) + formatter = logging.Formatter("%(asctime)s; %(levelname)s - %(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) \ No newline at end of file diff --git a/m2em.py b/m2em.py index dcd6a02..21b8d16 100755 --- a/m2em.py +++ b/m2em.py @@ -74,9 +74,11 @@ def read_arguments(self): # Logging if self.args.debug: - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG) + outputlevel = "debug" else: - logging.basicConfig(format='%(message)s', level=logging.INFO) + outputlevel = "info" + helper.initialize_logger("log/", outputlevel) + #Read Config From 5b6ace882b4c60fbccacab4b049edd97e4f162cf Mon Sep 17 00:00:00 2001 From: Schemen Date: Mon, 20 Nov 2017 20:55:28 +0100 Subject: [PATCH 04/28] Create log folder if it doesn't exist --- bin/m2emHelper.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/m2emHelper.py b/bin/m2emHelper.py index afd9004..78fd6d4 100644 --- a/bin/m2emHelper.py +++ b/bin/m2emHelper.py @@ -834,7 +834,11 @@ def initialize_logger(output_dir, outputlevel): formatter = logging.Formatter("%(message)s") handler.setFormatter(formatter) logger.addHandler(handler) - + + if not os.path.isdir(output_dir): + createFolder(output_dir) + + # create error file handler and set level to error handler = logging.FileHandler(os.path.join(output_dir, "error.log"), encoding=None, delay="true") handler.setLevel(logging.ERROR) From 73a3b6798236a5dab554bb7121dc8117f1eccb44 Mon Sep 17 00:00:00 2001 From: Schemen Date: Mon, 20 Nov 2017 21:13:18 +0100 Subject: [PATCH 05/28] Fix closure of file. Changed how files are downloaded Instead of directly downloading it will download into a temporary file and then move it to the correct name once it is done --- bin/m2emDownloader.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/m2emDownloader.py b/bin/m2emDownloader.py index 0e51ff3..a2d0f00 100644 --- a/bin/m2emDownloader.py +++ b/bin/m2emDownloader.py @@ -107,10 +107,13 @@ def ChapterDownloader(config): counter = counter + 1 imagepath = downloadfolder + "/" + str("{0:0=3d}".format(counter)) + ".png" + tempdl = downloadfolder + "/" + str("{0:0=3d}".format(counter)) + ".tmp" - f = open(imagepath, 'wb') + f = open(tempdl, 'wb') f.write(requests.get(image).content) - f.close + f.close() + + os.rename(tempdl, imagepath) # Cleanse image, remove footer From 913cb94d217d72a19e2cbe506ea0761393f240ad Mon Sep 17 00:00:00 2001 From: Schemen Date: Mon, 20 Nov 2017 22:32:56 +0100 Subject: [PATCH 06/28] Remove -a argument and clean up config output --- m2em.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/m2em.py b/m2em.py index 21b8d16..32efa23 100755 --- a/m2em.py +++ b/m2em.py @@ -57,10 +57,7 @@ def read_arguments(self): action="store_true") parser.add_argument("-cd", "--create-db", help="Creates DB. Uses Configfile for Naming", action="store_true") - parser.add_argument("-a", "--action", help="Start action. Options are: rss (collecting feed data), downloader, converter or sender ", - action="store_true") - parser.add_argument("-s", "--start", help="Start a single loop. If no arguments are passed, a single loop will also start", - action="store_true") + parser.add_argument("-a", "--action", help="Start action. Options are: rss (collecting feed data), downloader, converter or sender ") parser.add_argument("-ss", "--switch-send", help="Pass ID of User. Switches said user Send eBook status") parser.add_argument("-sc", "--switch-chapter", help="Pass ID of Chapter. Switches said Chapter Sent status") parser.add_argument("-dc", "--delete-chapter", help="Pass ID of Chapter. Deletes said Chapter") @@ -89,15 +86,9 @@ def read_config(self): config_reader.read("config.ini") self.config = config_reader["CONFIG"] - # Load Config Variables - if self.config["SaveLocation"]: - logging.debug("Succesfully loaded SaveLocation: %s ", self.config["SaveLocation"]) - if self.config["EbookFormat"]: - logging.debug("Succesfully loaded EbookFormat: %s ", self.config["EbookFormat"]) - if self.config["Database"]: - logging.debug("Succesfully loaded Database: %s ", self.config["Database"]) - if self.config["Sleep"]: - logging.debug("Succesfully loaded Database: %s ", self.config["Sleep"]) + logging.debug("Loaded Config:") + logging.debug(self.config) + ''' From 7bbbe2662f964134ccd4ab7ea24c108b7aa135e1 Mon Sep 17 00:00:00 2001 From: Schemen Date: Mon, 20 Nov 2017 22:33:28 +0100 Subject: [PATCH 07/28] rename ChapterDownloader to LoopDownloader --- bin/m2emDownloader.py | 7 ++++++- m2em.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/m2emDownloader.py b/bin/m2emDownloader.py index a2d0f00..4a07960 100644 --- a/bin/m2emDownloader.py +++ b/bin/m2emDownloader.py @@ -9,7 +9,12 @@ from PIL import ImageOps from PIL import ImageFilter -def ChapterDownloader(config): + + +''' +Loop downloader +''' +def LoopDownloader(config): # Load configs required here database = config["Database"] diff --git a/m2em.py b/m2em.py index 32efa23..78719fa 100755 --- a/m2em.py +++ b/m2em.py @@ -194,7 +194,7 @@ def parse_add_feeds(self): # Worker to fetch all images def images_fetcher(self): - mdownloader.ChapterDownloader(self.config) + mdownloader.LoopDownloader(self.config) # Worker to convert all downloaded chapters into ebooks def image_converter(self): From e6d2d896541ba6f9bc181aa9ee371f19ccf922c5 Mon Sep 17 00:00:00 2001 From: Schemen Date: Mon, 20 Nov 2017 22:34:16 +0100 Subject: [PATCH 08/28] Check if mobi exists before sending it --- bin/m2emSender.py | 133 ++++++++++++++++++++++++---------------------- 1 file changed, 69 insertions(+), 64 deletions(-) diff --git a/bin/m2emSender.py b/bin/m2emSender.py index 729479b..d3c01cb 100644 --- a/bin/m2emSender.py +++ b/bin/m2emSender.py @@ -46,72 +46,77 @@ def sendEbook(config): mangatitle = chapter[2] mangaid = int(chapter[0]) issent = int(chapter[8]) - manganame = chapter[11] - eblocation = str(saveloc + manganame + "/" + mangatitle + "/" + mangatitle + "." + ebformat.lower()) + manganame = chapter[11] + eblocation = str(saveloc + manganame + "/" + mangatitle + "/" + mangatitle + "." + ebformat.lower()) - if issent != 0: - logging.debug("%s has been sent already!" % mangatitle) + # Check if ebook has been converted yet, else skip + if not os.path.exists(eblocation): + logging.debug("Manga %s has not been converted yet." % mangatitle) else: - logging.info("Sending %s..."% mangatitle) - - - for user in users: - kindle_mail = user[3] - shouldsend = user[4] - - # Check if user wants Mails - if shouldsend == "True": - - logging.debug("Compiling Email for %s" % user[1]) - - # Compile Email - msg = MIMEMultipart() - msg['Subject'] = 'Ebook Delivery of %s' % mangatitle - msg['Date'] = formatdate(localtime=True) - msg['From'] = emailadress - msg['To'] = kindle_mail - msg['Message-ID'] = make_msgid() - - text = "Automatic Ebook delivery by m2em." - msg.attach(MIMEText(text)) - - - # Add Ebook as attachment - ebfile = open(eblocation, 'rb') - - attachment = MIMEBase('application', 'octet-stream', name=os.path.basename(eblocation)) - attachment.set_payload(ebfile.read()) - ebfile.close() - encoders.encode_base64(attachment) - attachment.add_header('Content-Disposition', 'attachment', - filename=os.path.basename(eblocation)) - - msg.attach(attachment) - - # Convert message to string - sio = StringIO() - gen = Generator(sio, mangle_from_=False) - gen.flatten(msg) - msg = sio.getvalue() - - # Send Email Off! - try: - server = smtplib.SMTP(smtpserver,serverport,) - if starttls: - server.starttls() - server.ehlo() - server.login(emailadress,password) - #server.sendmail(emailadress, kindle_mail, msg.as_string()) - server.sendmail(emailadress, kindle_mail, msg) - server.close() - logging.debug("Sent email to %s "% kindle_mail) - except smtplib.SMTPException as e: - logging.debug("Could not send email! %s" % e) - - - # Set Email as Sent - helper.setIsSent(mangaid,database) - logging.info("Sent %s to all requested users."% mangatitle) + + if issent != 0: + logging.debug("%s has been sent already!" % mangatitle) + else: + logging.info("Sending %s..."% mangatitle) + + + for user in users: + kindle_mail = user[3] + shouldsend = user[4] + + # Check if user wants Mails + if shouldsend == "True": + + logging.debug("Compiling Email for %s" % user[1]) + + # Compile Email + msg = MIMEMultipart() + msg['Subject'] = 'Ebook Delivery of %s' % mangatitle + msg['Date'] = formatdate(localtime=True) + msg['From'] = emailadress + msg['To'] = kindle_mail + msg['Message-ID'] = make_msgid() + + text = "Automatic Ebook delivery by m2em." + msg.attach(MIMEText(text)) + + + # Add Ebook as attachment + ebfile = open(eblocation, 'rb') + + attachment = MIMEBase('application', 'octet-stream', name=os.path.basename(eblocation)) + attachment.set_payload(ebfile.read()) + ebfile.close() + encoders.encode_base64(attachment) + attachment.add_header('Content-Disposition', 'attachment', + filename=os.path.basename(eblocation)) + + msg.attach(attachment) + + # Convert message to string + sio = StringIO() + gen = Generator(sio, mangle_from_=False) + gen.flatten(msg) + msg = sio.getvalue() + + # Send Email Off! + try: + server = smtplib.SMTP(smtpserver,serverport,) + if starttls: + server.starttls() + server.ehlo() + server.login(emailadress,password) + #server.sendmail(emailadress, kindle_mail, msg.as_string()) + server.sendmail(emailadress, kindle_mail, msg) + server.close() + logging.debug("Sent email to %s "% kindle_mail) + except smtplib.SMTPException as e: + logging.debug("Could not send email! %s" % e) + + + # Set Email as Sent + helper.setIsSent(mangaid,database) + logging.info("Sent %s to all requested users."% mangatitle) From 2238ef377223db1015b15b549230a00b5f2a2c09 Mon Sep 17 00:00:00 2001 From: Schemen Date: Mon, 20 Nov 2017 22:35:29 +0100 Subject: [PATCH 09/28] create verifyDownload function Function will verify if a download is done correctly. If corruption is detected, it will remove all files concerning one chapter to start cleanly in the next run --- bin/m2emHelper.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/bin/m2emHelper.py b/bin/m2emHelper.py index 78fd6d4..ff305df 100644 --- a/bin/m2emHelper.py +++ b/bin/m2emHelper.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import logging +import shutil import datetime import os import sqlite3 @@ -719,7 +720,6 @@ def getMangaData(url,entry): # Get source of to decide which parser to use origin = getSourceURL(url) - print(origin) # Mangastream Parser if origin == "mangastream.com": @@ -815,6 +815,42 @@ def checkTime(time): return False + +''' +Verify if chapter has been downloaded +Returns: true or false +''' +def verifyDownload(config, chapter): + + saveloc = config["SaveLocation"] + mangapages = chapter[9] + mangatitle = chapter[2] + manganame = chapter[11] + + downloadfolder = str(saveloc + manganame + "/" + mangatitle + "/images") + + if not os.path.exists(downloadfolder): + return False + else: + # First check checks if there is the right amount of files in the folder + if len(os.listdir(downloadfolder)) != int(mangapages): + shutil.rmtree(downloadfolder) + return False + else: + # second check checks if there is an unfinished download + for item in os.listdir(downloadfolder): + + if item.endswith(".tmp"): + logging.debug("%s seems to be corrupted, removing all images for redownload"% mangatitle) + shutil.rmtree(downloadfolder) + return False + return True + + + + + + ''' Init Logging! From eba3bac3d073e5a0301f95374a43395a8e4a68b0 Mon Sep 17 00:00:00 2001 From: Schemen Date: Mon, 20 Nov 2017 22:36:56 +0100 Subject: [PATCH 10/28] Code cleaning & verifyDownload usage to make everything more robust --- bin/m2emConverter.py | 60 +++++++++++++++++++++++-------------------- bin/m2emDownloader.py | 13 +++++----- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/bin/m2emConverter.py b/bin/m2emConverter.py index d330b58..37804e2 100644 --- a/bin/m2emConverter.py +++ b/bin/m2emConverter.py @@ -24,51 +24,55 @@ def RecursiveConverter(config): mangatitle = chapter[2] manganame = chapter[11] - # check if mangatitle contains ":" characters that OS can't handle as folders + # check if mangatitle or manganame contains ":" characters that OS can't handle as folders mangatitle = helper.sanetizeName(mangatitle) - - # check if manganame contains ":" characters that OS can't handle as folders manganame = helper.sanetizeName(manganame) + + # create folder variables imagefolder = str(saveloc + manganame + "/"+ mangatitle + "/images/") eblocation = str(saveloc + manganame + "/"+ mangatitle + "/" + mangatitle + "." + ebformat.lower()) cbzlocation = str(saveloc + manganame + "/"+ mangatitle + "/" + mangatitle + ".cbz") - - # Create CBZ to make creation easier - if os.path.exists(cbzlocation): - logging.debug("Manga %s converted to CBZ already!" % mangatitle) + # Verify if chapter has been downloaded already + if not helper.verifyDownload(config, chapter): + logging.debug("Manga %s has not been downloaded!" % mangatitle) else: - logging.info("Starting conversion to CBZ of %s..." % mangatitle) + # Create CBZ to make creation easier + if os.path.exists(cbzlocation): + logging.debug("Manga %s converted to CBZ already!" % mangatitle) + else: + logging.info("Starting conversion to CBZ of %s..." % mangatitle) - logging.debug("Opening CBZ archive...") - try: - zf = zipfile.ZipFile(cbzlocation, "w") - except Exception as e: - logging.warning("Failed opening archive! %s" % e) + logging.debug("Opening CBZ archive...") + try: + zf = zipfile.ZipFile(cbzlocation, "w") + except Exception as e: + logging.warning("Failed opening archive! %s" % e) - logging.debug("Writing Images into CBZ") - for img in sorted(os.listdir(imagefolder)): - image = imagefolder + img - logging.debug("Writing %s" % image) - zf.write(image,img) - zf.close() + logging.debug("Writing Images into CBZ") + for img in sorted(os.listdir(imagefolder)): + image = imagefolder + img + logging.debug("Writing %s" % image) + zf.write(image,img) + zf.close() - # Start conversion to Ebook format! - if os.path.exists(eblocation): - logging.debug("Manga %s converted to Ebook already!" % mangatitle) - else: - logging.info("Starting conversion to Ebook of %s..." % mangatitle) - try: - subprocess.call(["kcc-c2e", "-p", ebprofile, "-f", ebformat, "-m", "-q", "-r", "2", "-u", "-s", cbzlocation]) - except Exception as e: - logging.debug("Failed to convert epub %s" % e) + # Start conversion to Ebook format! + if os.path.exists(eblocation): + logging.debug("Manga %s converted to Ebook already!" % mangatitle) + else: + logging.info("Starting conversion to Ebook of %s..." % mangatitle) + + try: + subprocess.call(["kcc-c2e", "-p", ebprofile, "-f", ebformat, "-m", "-q", "-r", "2", "-u", "-s", cbzlocation]) + except Exception as e: + logging.debug("Failed to convert epub %s" % e) def ChapterConverter(imagelocation, config): pass \ No newline at end of file diff --git a/bin/m2emDownloader.py b/bin/m2emDownloader.py index 4a07960..2a7f340 100644 --- a/bin/m2emDownloader.py +++ b/bin/m2emDownloader.py @@ -42,10 +42,8 @@ def LoopDownloader(config): if helper.checkTime(chapterdate): - # check if mangatitle contains ":" characters that OS can't handle as folders + # check if mangatitle or manganame contains ":" characters that OS can't handle as folders mangatitle = helper.sanetizeName(mangatitle) - - # check if manganame contains ":" characters that OS can't handle as folders manganame = helper.sanetizeName(manganame) # Old Download folder from v0.1.0 @@ -56,19 +54,19 @@ def LoopDownloader(config): downloadfolder = str(saveloc + manganame + "/" + mangatitle + "/images") - # Check if the old DL location is being used + # Check if the old DL location is being used and fix it! if os.path.isdir(oldlocation): logging.info("Moving old DL location to new one") helper.createFolder(newlocation) move(oldlocation, newlocation) - if os.path.isdir(downloadfolder): + # Verify if chapter has been downloaded already, or if it may require redownload due to corruption + if helper.verifyDownload(config, chapter): logging.debug("Manga %s downloaded already!" % mangatitle) else: logging.info("Starting download of %s..." % mangatitle) - # get Origin of manga origin = helper.getSourceURL(mangastarturl) @@ -142,4 +140,5 @@ def LoopDownloader(config): logging.info("Finished download!") - logging.debug("Chapter %s is older than 1 day. Skipping in the loop..." % mangatitle) \ No newline at end of file + else: + logging.debug("Chapter %s is older than 1 day. Skipping in the loop..." % mangatitle) \ No newline at end of file From 5628ea0b96c93591e7ebb208c93484cdd5671f82 Mon Sep 17 00:00:00 2001 From: Schemen Date: Mon, 20 Nov 2017 22:40:23 +0100 Subject: [PATCH 11/28] create m2em.log --- bin/m2emHelper.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/m2emHelper.py b/bin/m2emHelper.py index ff305df..5242211 100644 --- a/bin/m2emHelper.py +++ b/bin/m2emHelper.py @@ -883,7 +883,7 @@ def initialize_logger(output_dir, outputlevel): logger.addHandler(handler) # create debug file handler and set level to debug - handler = logging.FileHandler(os.path.join(output_dir, "all.log")) + handler = logging.FileHandler(os.path.join(output_dir, "debug.log")) handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s; %(levelname)s - %(message)s") handler.setFormatter(formatter) @@ -894,4 +894,11 @@ def initialize_logger(output_dir, outputlevel): handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s; %(levelname)s - %(message)s") handler.setFormatter(formatter) + logger.addHandler(handler) + + # create debug file handler and set level to info + handler = logging.FileHandler(os.path.join(output_dir, "m2em.log")) + handler.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s; %(levelname)s - %(message)s") + handler.setFormatter(formatter) logger.addHandler(handler) \ No newline at end of file From d8398dcbea9281e492139beb7c25fbb28d386de7 Mon Sep 17 00:00:00 2001 From: Schemen Date: Mon, 20 Nov 2017 23:18:01 +0100 Subject: [PATCH 12/28] Move timecondition to a later point and remove unnessesary log output --- bin/m2emDownloader.py | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/bin/m2emDownloader.py b/bin/m2emDownloader.py index 2a7f340..5797058 100644 --- a/bin/m2emDownloader.py +++ b/bin/m2emDownloader.py @@ -38,33 +38,36 @@ def LoopDownloader(config): chapterdate = chapter[3] - # Only start run if chapter is younger than 24h - if helper.checkTime(chapterdate): - # check if mangatitle or manganame contains ":" characters that OS can't handle as folders - mangatitle = helper.sanetizeName(mangatitle) - manganame = helper.sanetizeName(manganame) + # check if mangatitle or manganame contains ":" characters that OS can't handle as folders + mangatitle = helper.sanetizeName(mangatitle) + manganame = helper.sanetizeName(manganame) - # Old Download folder from v0.1.0 - oldlocation = str(saveloc + mangatitle) - newlocation = str(saveloc + manganame) + # Old Download folder from v0.1.0 + oldlocation = str(saveloc + mangatitle) + newlocation = str(saveloc + manganame) - # Define Download location - downloadfolder = str(saveloc + manganame + "/" + mangatitle + "/images") + # Define Download location + downloadfolder = str(saveloc + manganame + "/" + mangatitle + "/images") - # Check if the old DL location is being used and fix it! - if os.path.isdir(oldlocation): - logging.info("Moving old DL location to new one") - helper.createFolder(newlocation) - move(oldlocation, newlocation) + # Check if the old DL location is being used and fix it! + if os.path.isdir(oldlocation): + logging.info("Moving old DL location to new one") + helper.createFolder(newlocation) + move(oldlocation, newlocation) - # Verify if chapter has been downloaded already, or if it may require redownload due to corruption - if helper.verifyDownload(config, chapter): - logging.debug("Manga %s downloaded already!" % mangatitle) - else: + # Verify if chapter has been downloaded already, or if it may require redownload due to corruption + if helper.verifyDownload(config, chapter): + logging.debug("Manga %s downloaded already!" % mangatitle) + else: + + # Only start run if chapter is younger than 24h + logging.debug("The loop will only consider Chapters younger than 24h!") + if helper.checkTime(chapterdate): + logging.info("Starting download of %s..." % mangatitle) # get Origin of manga @@ -140,5 +143,3 @@ def LoopDownloader(config): logging.info("Finished download!") - else: - logging.debug("Chapter %s is older than 1 day. Skipping in the loop..." % mangatitle) \ No newline at end of file From 67a2df4154365a2b462594d7fbfcb63d4c9e0bea Mon Sep 17 00:00:00 2001 From: Schemen Date: Tue, 21 Nov 2017 17:20:41 +0100 Subject: [PATCH 13/28] Add Sanitization to verifyDownload as well --- bin/m2emHelper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/m2emHelper.py b/bin/m2emHelper.py index 5242211..9a93922 100644 --- a/bin/m2emHelper.py +++ b/bin/m2emHelper.py @@ -827,6 +827,10 @@ def verifyDownload(config, chapter): mangatitle = chapter[2] manganame = chapter[11] + # check if mangatitle or manganame contains ":" characters that OS can't handle as folders + mangatitle = sanetizeName(mangatitle) + manganame = sanetizeName(manganame) + downloadfolder = str(saveloc + manganame + "/" + mangatitle + "/images") if not os.path.exists(downloadfolder): From 32a2c5936a35c23b9e6de0650d71ca0dd0fb5b75 Mon Sep 17 00:00:00 2001 From: Schemen Date: Tue, 21 Nov 2017 17:22:43 +0100 Subject: [PATCH 14/28] Create argument verification and add --start option to start one loop --- m2em.py | 107 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 79 insertions(+), 28 deletions(-) diff --git a/m2em.py b/m2em.py index 78719fa..c8d4ac4 100755 --- a/m2em.py +++ b/m2em.py @@ -10,7 +10,7 @@ # Start of the fun! import bin.m2emHelper as helper import bin.m2emRssParser as mparser -import bin.m2emDownloader as mdownloader +import bin.m2emDownloaderHandler as mdownloader import bin.m2emConverter as mconverter import bin.m2emSender as msender @@ -30,6 +30,7 @@ def __init__(self): if not self.args: self.read_arguments() + # Load config right at the start self.config = None if not self.config: @@ -57,6 +58,8 @@ def read_arguments(self): action="store_true") parser.add_argument("-cd", "--create-db", help="Creates DB. Uses Configfile for Naming", action="store_true") + parser.add_argument("-s", "--start", help="Starts one loop", + action="store_true") parser.add_argument("-a", "--action", help="Start action. Options are: rss (collecting feed data), downloader, converter or sender ") parser.add_argument("-ss", "--switch-send", help="Pass ID of User. Switches said user Send eBook status") parser.add_argument("-sc", "--switch-chapter", help="Pass ID of Chapter. Switches said Chapter Sent status") @@ -77,6 +80,24 @@ def read_arguments(self): helper.initialize_logger("log/", outputlevel) + # Check if Arguments are passed or not. At least one is required + if self.args.action is None \ + and self.args.add_feed is None \ + and self.args.delete_chapter is None \ + and self.args.delete_feed is None \ + and self.args.delete_user is None \ + and self.args.switch_chapter is None \ + and self.args.switch_send is None \ + and self.args.add_user is False \ + and not any([self.args.add_user, + self.args.create_db, + self.args.daemon, + self.args.list_chapters, + self.args.list_chapters_all, + self.args.list_feeds, + self.args.list_users, + self.args.start,]): + logging.error("At least one argument is required!") #Read Config def read_config(self): @@ -182,6 +203,31 @@ def create_db(self): helper.createDB(self.config) pass + ''' + Catch -a / --action + ''' + def start_action(self): + + # Start downloader + if self.args.action == "downloader": + logging.info("Starting downloader to get all outstanding chapters") + self.images_fetcher() + logging.info("Finished downloading all outstanding chapters.") + + + elif self.args.action == "rssparser": + logging.info("Action '%s' is not yet implemented." % self.args.action) + + elif self.args.action == "converter": + logging.info("Action '%s' is not yet implemented." % self.args.action) + + elif self.args.action == "sender": + logging.info("Action '%s' is not yet implemented." % self.args.action) + + else: + logging.info("%s is not a valid action. Choose between 'rssparser', 'downloader', 'converter' or 'sender'"% self.args.action) + pass + @@ -194,7 +240,7 @@ def parse_add_feeds(self): # Worker to fetch all images def images_fetcher(self): - mdownloader.LoopDownloader(self.config) + mdownloader.downloader(self.config, self.args) # Worker to convert all downloaded chapters into ebooks def image_converter(self): @@ -263,34 +309,39 @@ def run(self): self.create_db() return + if self.args.action: + self.start_action() + return # Mainloop - loop = True - while loop: - if not self.args.daemon: - loop = False - - logging.info("Starting Loop at %s" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - - logging.info("Starting RSS Data Fetcher!") - self.parse_add_feeds() - logging.info("Finished Loading RSS Data") - - logging.info("Starting all outstanding Chapter Downloads!") - self.images_fetcher() - logging.info("Finished all outstanding Chapter Downloads") - - logging.info("Starting recursive image conversion!") - self.image_converter() - logging.info("Finished recursive image conversion!") - - logging.info("Starting to send all ebooks!") - self.send_ebooks() - logging.info("Finished sending ebooks!") - - if loop: - logging.info("Sleeping for %s seconds...\n" % (self.config["Sleep"])) - time.sleep(int(self.config["Sleep"])) + if self.args.start: + daemon = True + while daemon: + if not self.args.daemon: + daemon = False + logging.info("#########################") + logging.info("Starting Loop at %s" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + logging.info("Don't forget that the loop only handles data younger than 24h Hours!") + + logging.info("Starting RSS Data Fetcher!") + self.parse_add_feeds() + logging.info("Finished Loading RSS Data") + + logging.info("Starting all outstanding Chapter Downloads!") + self.images_fetcher() + logging.info("Finished all outstanding Chapter Downloads") + + logging.info("Starting recursive image conversion!") + self.image_converter() + logging.info("Finished recursive image conversion!") + + logging.info("Starting to send all ebooks!") + self.send_ebooks() + logging.info("Finished sending ebooks!") + + if daemon: + logging.info("Sleeping for %s seconds...\n" % (self.config["Sleep"])) + time.sleep(int(self.config["Sleep"])) # Execute Main if __name__ == '__main__': From 75434e2fa36d3af6f610e2332df641d9d454e7e4 Mon Sep 17 00:00:00 2001 From: Schemen Date: Tue, 21 Nov 2017 17:22:59 +0100 Subject: [PATCH 15/28] Code cleanup --- bin/models/m2emManga.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/bin/models/m2emManga.py b/bin/models/m2emManga.py index ae0eb2d..8dae7db 100644 --- a/bin/models/m2emManga.py +++ b/bin/models/m2emManga.py @@ -21,9 +21,6 @@ def __init__(self): self.duplicated = None - #def __repr__(self): - # return '' % (self.manga_name, self.chapter) - def load_from_feed(self, entry, parent_feed): self.chapter_link = entry.link From 557beb6a02e5bf6a82d2196e9382f90460150122 Mon Sep 17 00:00:00 2001 From: Schemen Date: Tue, 21 Nov 2017 17:23:41 +0100 Subject: [PATCH 16/28] Rewrite downloader to a downloadHandler and download class --- bin/m2emDownloader.py | 187 +++++++++++++++++------------------ bin/m2emDownloaderHandler.py | 65 ++++++++++++ 2 files changed, 158 insertions(+), 94 deletions(-) create mode 100644 bin/m2emDownloaderHandler.py diff --git a/bin/m2emDownloader.py b/bin/m2emDownloader.py index 5797058..494db87 100644 --- a/bin/m2emDownloader.py +++ b/bin/m2emDownloader.py @@ -10,136 +10,135 @@ from PIL import ImageFilter +class Downloader: -''' -Loop downloader -''' -def LoopDownloader(config): - - # Load configs required here - database = config["Database"] - saveloc = config["SaveLocation"] + def __init__(self): + self.database = None + self.saveloc = None + self.mangastarturl = None + self.mangapages = None + self.mangatitle = None + self.manganame = None + self.chapterdate = None + self.downloadfolder = None + self.origin = None + self.imageurls = None - # Load Chapters from Database - chapters = helper.getChapters(database) - logging.debug("Loaded Chapers:") - for i in chapters: - logging.debug(i) - # Start Download loop! - for chapter in chapters: - # get relevant data of this Manga - mangastarturl = chapter[4] - mangapages = chapter[9] - mangatitle = chapter[2] - manganame = chapter[11] - chapterdate = chapter[3] + def data_collector(self, config, chapter): + + # Load configs required here + self.database = config["Database"] + self.saveloc = config["SaveLocation"] + # get relevant data of this Chapter + self.mangastarturl = chapter[4] + self.mangapages = chapter[9] + self.mangatitle = chapter[2] + self.manganame = chapter[11] + self.chapterdate = chapter[3] + # check if mangatitle or manganame contains ":" characters that OS can't handle as folders + self.mangatitle = helper.sanetizeName(self.mangatitle) + self.manganame = helper.sanetizeName(self.manganame) + # Define Download location + self.downloadfolder = str(self.saveloc + self.manganame + "/" + self.mangatitle + "/images") - # check if mangatitle or manganame contains ":" characters that OS can't handle as folders - mangatitle = helper.sanetizeName(mangatitle) - manganame = helper.sanetizeName(manganame) + # get Origin of manga (Which mangawebsite) + self.origin = helper.getSourceURL(self.mangastarturl) + + # Initiate URL list + self.imageurls=[] - # Old Download folder from v0.1.0 - oldlocation = str(saveloc + mangatitle) - newlocation = str(saveloc + manganame) - # Define Download location - downloadfolder = str(saveloc + manganame + "/" + mangatitle + "/images") + def data_processor(self): + + logging.debug("Proccesing data for %s"% self.mangatitle) # Check if the old DL location is being used and fix it! + oldlocation = str(self.saveloc + self.mangatitle) + newlocation = str(self.saveloc + self.manganame) if os.path.isdir(oldlocation): - logging.info("Moving old DL location to new one") + logging.info("Moving %s from old DL location to new one..." % self.mangatitle) helper.createFolder(newlocation) move(oldlocation, newlocation) - # Verify if chapter has been downloaded already, or if it may require redownload due to corruption - if helper.verifyDownload(config, chapter): - logging.debug("Manga %s downloaded already!" % mangatitle) - else: - - # Only start run if chapter is younger than 24h - logging.debug("The loop will only consider Chapters younger than 24h!") - if helper.checkTime(chapterdate): - - logging.info("Starting download of %s..." % mangatitle) - - # get Origin of manga - origin = helper.getSourceURL(mangastarturl) + # Get image urls! + # Mangastream Parser + if self.origin == "mangastream.com": + urllist = msparser.getPagesUrl(self.mangastarturl,self.mangapages) - # Get image urls! - # Mangastream Parser - if origin == "mangastream.com": - urllist = msparser.getPagesUrl(mangastarturl,mangapages) + # Turn Manga pages into Image links! + for i in urllist: + self.imageurls.append(msparser.getImageUrl(i)) + logging.debug("List of all Images for %s" % self.mangatitle) + logging.debug(self.imageurls) - # Turn Manga pages into Image links! - imageurls=[] - for i in urllist: - imageurls.append(msparser.getImageUrl(i)) - logging.debug("List of all Images for %s" % mangatitle) - logging.debug(imageurls) + # Mangafox Parser + elif self.origin == "mangafox.me": + urllist = mxparser.getPagesUrl(self.mangastarturl,self.mangapages) - # Mangafox Parser - elif origin == "mangafox.me": - urllist = mxparser.getPagesUrl(mangastarturl,mangapages) + # Turn Manga pages into Image links! + for i in urllist: + self.imageurls.append(mxparser.getImageUrl(i)) + logging.debug("List of all Images for %s" % self.mangatitle) + logging.debug(self.imageurls) - # Turn Manga pages into Image links! - imageurls=[] - for i in urllist: - imageurls.append(mxparser.getImageUrl(i)) - logging.debug("List of all Images for %s" % mangatitle) - logging.debug(imageurls) - else: - pass - # Download & save images! - # check if we have images to download - if not len(imageurls) == 0: - helper.createFolder(downloadfolder) - counter = 0 - for image in imageurls: - counter = counter + 1 + def downloader(self): + logging.info("Starting download of %s..." % self.mangatitle) + # Download & save images! + # check if we have images to download + if not len(self.imageurls) == 0: - imagepath = downloadfolder + "/" + str("{0:0=3d}".format(counter)) + ".png" - tempdl = downloadfolder + "/" + str("{0:0=3d}".format(counter)) + ".tmp" + # Check if we have the Download folder + helper.createFolder(self.downloadfolder) - f = open(tempdl, 'wb') - f.write(requests.get(image).content) - f.close() + # Start download Task + counter = 0 + for image in self.imageurls: + counter = counter + 1 - os.rename(tempdl, imagepath) + imagepath = self.downloadfolder + "/" + str("{0:0=3d}".format(counter)) + ".png" + tempdl = self.downloadfolder + "/" + str("{0:0=3d}".format(counter)) + ".tmp" + # Download the image! + f = open(tempdl, 'wb') + f.write(requests.get(image).content) + f.close() - # Cleanse image, remove footer - # - # I have borrowed this code from the kmanga project. - # https://github.com/aplanas/kmanga/blob/master/mobi/mobi.py#L416 - # Thanks a lot to Alberto Planas for coming up with it! - # - if origin == "mangafox.me": - logging.debug("Cleaning Mangafox Footer") - img = Image.open(imagepath) - _img = ImageOps.invert(img.convert(mode='L')) - _img = _img.point(lambda x: x and 255) - _img = _img.filter(ImageFilter.MinFilter(size=3)) - _img = _img.filter(ImageFilter.GaussianBlur(radius=5)) - _img = _img.point(lambda x: (x >= 48) and x) + # If everything is alright, write image to final name + os.rename(tempdl, imagepath) - cleaned = img.crop(_img.getbbox()) if _img.getbbox() else img - cleaned.save(imagepath) + # Cleanse image, remove footer + # + # I have borrowed this code from the kmanga project. + # https://github.com/aplanas/kmanga/blob/master/mobi/mobi.py#L416 + # Thanks a lot to Alberto Planas for coming up with it! + # + if self.origin == "mangafox.me": + logging.debug("Cleaning Mangafox Footer") + img = Image.open(imagepath) + _img = ImageOps.invert(img.convert(mode='L')) + _img = _img.point(lambda x: x and 255) + _img = _img.filter(ImageFilter.MinFilter(size=3)) + _img = _img.filter(ImageFilter.GaussianBlur(radius=5)) + _img = _img.point(lambda x: (x >= 48) and x) + cleaned = img.crop(_img.getbbox()) if _img.getbbox() else img + cleaned.save(imagepath) - logging.info("Finished download!") + # Finish :) + logging.info("Finished download of %s!"% self.mangatitle) \ No newline at end of file diff --git a/bin/m2emDownloaderHandler.py b/bin/m2emDownloaderHandler.py new file mode 100644 index 0000000..e2f601e --- /dev/null +++ b/bin/m2emDownloaderHandler.py @@ -0,0 +1,65 @@ +import logging +import os +import requests +from shutil import move +import bin.m2emHelper as helper +import bin.sourceparser.m2emMangastream as msparser +import bin.sourceparser.m2emMangafox as mxparser +from bin.m2emDownloader import Downloader +from PIL import Image +from PIL import ImageOps +from PIL import ImageFilter + + + +''' +Loop downloader +''' +def downloader(config, args): + + # Load configs required here + database = config["Database"] + + # Load Chapters from Database + chapters = helper.getChapters(database) + logging.debug("Loaded Chapers:") + for i in chapters: + logging.debug(i) + + if args.start: + logging.debug("The loop will only consider Chapters younger than 24h!") + + # Start Download loop! + for chapter in chapters: + + + current_chapter = Downloader() + current_chapter.data_collector(config,chapter) + + # Check if the old DL location is being used and fix it! + oldlocation = str(current_chapter.saveloc + current_chapter.mangatitle) + newlocation = str(current_chapter.saveloc + current_chapter.manganame) + if os.path.isdir(oldlocation): + logging.info("Moving %s from old DL location to new one..." % current_chapter.mangatitle) + helper.createFolder(newlocation) + move(oldlocation, newlocation) + + logging.debug("Processing %s..."% current_chapter.mangatitle) + # Check if chapter needs to be downloaded + if helper.verifyDownload(config, chapter): + logging.debug("Manga %s downloaded already!" % current_chapter.mangatitle) + else: + + # Start Tasks! + # Check if Download loop & Download task is selected + if not args.start: + current_chapter.data_processor() + current_chapter.downloader() + else: + + # Only start run if chapter is younger than 24h + if helper.checkTime(current_chapter.chapterdate): + current_chapter.data_processor() + current_chapter.downloader() + else: + logging.debug("%s is older than 24h, will not be processed by daemon." % current_chapter.mangatitle) From 3a46053b58601227f2504d05e2d1456ac4503629 Mon Sep 17 00:00:00 2001 From: Schemen Date: Tue, 21 Nov 2017 17:25:26 +0100 Subject: [PATCH 17/28] Remove old dl location check from downloader to handler --- bin/m2emDownloader.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/bin/m2emDownloader.py b/bin/m2emDownloader.py index 494db87..cd31fb4 100644 --- a/bin/m2emDownloader.py +++ b/bin/m2emDownloader.py @@ -59,13 +59,6 @@ def data_collector(self, config, chapter): def data_processor(self): logging.debug("Proccesing data for %s"% self.mangatitle) - # Check if the old DL location is being used and fix it! - oldlocation = str(self.saveloc + self.mangatitle) - newlocation = str(self.saveloc + self.manganame) - if os.path.isdir(oldlocation): - logging.info("Moving %s from old DL location to new one..." % self.mangatitle) - helper.createFolder(newlocation) - move(oldlocation, newlocation) # Get image urls! From 2a12bf87bd75c6db3f765a2d98779f0c2aa53661 Mon Sep 17 00:00:00 2001 From: Schemen Date: Tue, 21 Nov 2017 17:34:20 +0100 Subject: [PATCH 18/28] Clean up downloadhandler --- bin/m2emDownloaderHandler.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/bin/m2emDownloaderHandler.py b/bin/m2emDownloaderHandler.py index e2f601e..bad6275 100644 --- a/bin/m2emDownloaderHandler.py +++ b/bin/m2emDownloaderHandler.py @@ -1,25 +1,20 @@ import logging import os -import requests from shutil import move import bin.m2emHelper as helper -import bin.sourceparser.m2emMangastream as msparser -import bin.sourceparser.m2emMangafox as mxparser from bin.m2emDownloader import Downloader -from PIL import Image -from PIL import ImageOps -from PIL import ImageFilter - ''' -Loop downloader +downloadHandler ''' def downloader(config, args): # Load configs required here database = config["Database"] + + # Load Chapters from Database chapters = helper.getChapters(database) logging.debug("Loaded Chapers:") @@ -29,13 +24,16 @@ def downloader(config, args): if args.start: logging.debug("The loop will only consider Chapters younger than 24h!") + + # Start Download loop! for chapter in chapters: - + # Initialize Downloader class & load basic params current_chapter = Downloader() current_chapter.data_collector(config,chapter) + # Check if the old DL location is being used and fix it! oldlocation = str(current_chapter.saveloc + current_chapter.mangatitle) newlocation = str(current_chapter.saveloc + current_chapter.manganame) @@ -44,19 +42,22 @@ def downloader(config, args): helper.createFolder(newlocation) move(oldlocation, newlocation) + logging.debug("Processing %s..."% current_chapter.mangatitle) + + # Check if chapter needs to be downloaded if helper.verifyDownload(config, chapter): logging.debug("Manga %s downloaded already!" % current_chapter.mangatitle) else: - - # Start Tasks! + + # Check if Download loop & Download task is selected if not args.start: current_chapter.data_processor() current_chapter.downloader() else: - + # Only start run if chapter is younger than 24h if helper.checkTime(current_chapter.chapterdate): current_chapter.data_processor() From ecb29bece352d683da71795c34ae882f6b88a705 Mon Sep 17 00:00:00 2001 From: Schemen Date: Tue, 21 Nov 2017 20:10:23 +0100 Subject: [PATCH 19/28] Reduce spam, improve information --- bin/m2emDownloader.py | 2 +- bin/m2emDownloaderHandler.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bin/m2emDownloader.py b/bin/m2emDownloader.py index cd31fb4..5549e63 100644 --- a/bin/m2emDownloader.py +++ b/bin/m2emDownloader.py @@ -58,7 +58,7 @@ def data_collector(self, config, chapter): def data_processor(self): - logging.debug("Proccesing data for %s"% self.mangatitle) + logging.info("Proccesing data for %s"% self.mangatitle) # Get image urls! diff --git a/bin/m2emDownloaderHandler.py b/bin/m2emDownloaderHandler.py index bad6275..06ff8ea 100644 --- a/bin/m2emDownloaderHandler.py +++ b/bin/m2emDownloaderHandler.py @@ -43,15 +43,12 @@ def downloader(config, args): move(oldlocation, newlocation) - logging.debug("Processing %s..."% current_chapter.mangatitle) - # Check if chapter needs to be downloaded if helper.verifyDownload(config, chapter): logging.debug("Manga %s downloaded already!" % current_chapter.mangatitle) else: - # Check if Download loop & Download task is selected if not args.start: current_chapter.data_processor() From d09e0101eb05fec8fed8b111e5bd300c03dcfbb9 Mon Sep 17 00:00:00 2001 From: Schemen Date: Tue, 21 Nov 2017 21:18:58 +0100 Subject: [PATCH 20/28] Convert Converter into Class and Handler, add possibility to DL by id --- bin/m2emConverter.py | 103 +++++++++++++++++++---------------- bin/m2emConverterHandler.py | 46 ++++++++++++++++ bin/m2emDownloaderHandler.py | 13 +++-- m2em.py | 15 +++-- 4 files changed, 120 insertions(+), 57 deletions(-) create mode 100644 bin/m2emConverterHandler.py diff --git a/bin/m2emConverter.py b/bin/m2emConverter.py index 37804e2..894c5eb 100644 --- a/bin/m2emConverter.py +++ b/bin/m2emConverter.py @@ -5,74 +5,83 @@ import bin.m2emHelper as helper -def RecursiveConverter(config): +class Converter: - # Load configs required here - database = config["Database"] - saveloc = config["SaveLocation"] - ebformat = config["EbookFormat"] - ebprofile = config["EbookProfile"] + def __init__(self): + self.saveloc = None + self.ebformat = None + self.ebprofile = None + self.mangatitle = None + self.manganame = None + self.imagefolder = None + self.eblocation = None + self.cbzlocation = None + self.chapterdate = None - # Load Chapters from Database - chapters = helper.getChapters(database) - # Start conversion loop! - for chapter in chapters: + def data_collector(self, config, chapter): + + # Load configs required here + self.saveloc = config["SaveLocation"] + self.ebformat = config["EbookFormat"] + self.ebprofile = config["EbookProfile"] + # get relevant data of this Manga - mangatitle = chapter[2] - manganame = chapter[11] + self.mangatitle = chapter[2] + self.manganame = chapter[11] + self.chapterdate = chapter[3] # check if mangatitle or manganame contains ":" characters that OS can't handle as folders - mangatitle = helper.sanetizeName(mangatitle) - manganame = helper.sanetizeName(manganame) + self.mangatitle = helper.sanetizeName(self.mangatitle) + self.manganame = helper.sanetizeName(self.manganame) # create folder variables - imagefolder = str(saveloc + manganame + "/"+ mangatitle + "/images/") - eblocation = str(saveloc + manganame + "/"+ mangatitle + "/" + mangatitle + "." + ebformat.lower()) - cbzlocation = str(saveloc + manganame + "/"+ mangatitle + "/" + mangatitle + ".cbz") + self.imagefolder = str(self.saveloc + self.manganame + "/"+ self.mangatitle + "/images/") + self.eblocation = str(self.saveloc + self.manganame + "/"+ self.mangatitle + "/" + self.mangatitle + "." + self.ebformat.lower()) + self.cbzlocation = str(self.saveloc + self.manganame + "/"+ self.mangatitle + "/" + self.mangatitle + ".cbz") + + - # Verify if chapter has been downloaded already - if not helper.verifyDownload(config, chapter): - logging.debug("Manga %s has not been downloaded!" % mangatitle) + + def cbz_creator(self): + + # Create CBZ to make creation easier + if os.path.exists(self.cbzlocation): + logging.debug("Manga %s converted to CBZ already!" % self.mangatitle) else: + logging.info("Starting conversion to CBZ of %s..." % self.mangatitle) - # Create CBZ to make creation easier - if os.path.exists(cbzlocation): - logging.debug("Manga %s converted to CBZ already!" % mangatitle) - else: - logging.info("Starting conversion to CBZ of %s..." % mangatitle) + logging.debug("Opening CBZ archive...") + try: + zf = zipfile.ZipFile(self.cbzlocation, "w") + except Exception as e: + logging.warning("Failed opening archive! %s" % e) - logging.debug("Opening CBZ archive...") - try: - zf = zipfile.ZipFile(cbzlocation, "w") - except Exception as e: - logging.warning("Failed opening archive! %s" % e) + logging.debug("Writing Images into CBZ") + for img in sorted(os.listdir(self.imagefolder)): + image = self.imagefolder + img + logging.debug("Writing %s" % image) + zf.write(image,img) - logging.debug("Writing Images into CBZ") - for img in sorted(os.listdir(imagefolder)): - image = imagefolder + img - logging.debug("Writing %s" % image) - zf.write(image,img) + zf.close() - zf.close() - # Start conversion to Ebook format! - if os.path.exists(eblocation): - logging.debug("Manga %s converted to Ebook already!" % mangatitle) - else: - logging.info("Starting conversion to Ebook of %s..." % mangatitle) + def eb_creator(self): - try: - subprocess.call(["kcc-c2e", "-p", ebprofile, "-f", ebformat, "-m", "-q", "-r", "2", "-u", "-s", cbzlocation]) - except Exception as e: - logging.debug("Failed to convert epub %s" % e) + # Start conversion to Ebook format! + if os.path.exists(self.eblocation): + logging.debug("Manga %s converted to Ebook already!" % self.mangatitle) + else: + logging.info("Starting conversion to Ebook of %s..." % self.mangatitle) -def ChapterConverter(imagelocation, config): - pass \ No newline at end of file + try: + subprocess.call(["kcc-c2e", "-p", self.ebprofile, "-f", self.ebformat, "-m", "-q", "-r", "2", "-u", "-s", self.cbzlocation]) + except Exception as e: + logging.debug("Failed to convert epub %s" % e) diff --git a/bin/m2emConverterHandler.py b/bin/m2emConverterHandler.py new file mode 100644 index 0000000..549ae12 --- /dev/null +++ b/bin/m2emConverterHandler.py @@ -0,0 +1,46 @@ +import logging +import bin.m2emHelper as helper +from bin.m2emConverter import Converter + + +def ConverterHandler(config, args, chapterids=[]): + + # Load configs required here + database = config["Database"] + + if not chapterids: + # Load Chapters from Database + chapters = helper.getChapters(database) + else: + # TODO Create helper function to extract chapters out of chapter IDs + pass + + + # Start conversion loop! + for chapter in chapters: + + + + # Verify if chapter has been downloaded already + if not helper.verifyDownload(config, chapter): + logging.debug("Manga %s has not been downloaded!" % chapter[2]) + else: + + + # Spawn an Converter Object & get basic data from database & config + current_conversation = Converter() + current_conversation.data_collector(config,chapter) + + # Check if Download loop & Download task is selected + if not args.start: + current_conversation.cbz_creator() + current_conversation.eb_creator() + else: + + # Only start run if chapter is younger than 24h + if helper.checkTime(current_conversation.chapterdate): + current_conversation.cbz_creator() + current_conversation.eb_creator() + else: + logging.debug("%s is older than 24h, will not be processed by daemon." % current_conversation.mangatitle) + diff --git a/bin/m2emDownloaderHandler.py b/bin/m2emDownloaderHandler.py index 06ff8ea..aa9f06a 100644 --- a/bin/m2emDownloaderHandler.py +++ b/bin/m2emDownloaderHandler.py @@ -8,18 +8,19 @@ ''' downloadHandler ''' -def downloader(config, args): +def downloader(config, args, chapterids=[]): # Load configs required here database = config["Database"] - # Load Chapters from Database - chapters = helper.getChapters(database) - logging.debug("Loaded Chapers:") - for i in chapters: - logging.debug(i) + if not chapterids: + # Load Chapters from Database + chapters = helper.getChapters(database) + else: + # TODO Create helper function to extract chapters out of chapter IDs + pass if args.start: logging.debug("The loop will only consider Chapters younger than 24h!") diff --git a/m2em.py b/m2em.py index c8d4ac4..9961167 100755 --- a/m2em.py +++ b/m2em.py @@ -11,7 +11,7 @@ import bin.m2emHelper as helper import bin.m2emRssParser as mparser import bin.m2emDownloaderHandler as mdownloader -import bin.m2emConverter as mconverter +import bin.m2emConverterHandler as mconverter import bin.m2emSender as msender #logging.basicConfig(format='%(message)s', level=logging.DEBUG) @@ -203,6 +203,7 @@ def create_db(self): helper.createDB(self.config) pass + ''' Catch -a / --action ''' @@ -215,15 +216,21 @@ def start_action(self): logging.info("Finished downloading all outstanding chapters.") + elif self.args.action == "rssparser": logging.info("Action '%s' is not yet implemented." % self.args.action) + elif self.args.action == "converter": - logging.info("Action '%s' is not yet implemented." % self.args.action) + logging.info("Starting converter to convert all outstanding chapters") + self.image_converter() + logging.info("Finished converting all remaining chapters!") + elif self.args.action == "sender": logging.info("Action '%s' is not yet implemented." % self.args.action) + else: logging.info("%s is not a valid action. Choose between 'rssparser', 'downloader', 'converter' or 'sender'"% self.args.action) pass @@ -240,11 +247,11 @@ def parse_add_feeds(self): # Worker to fetch all images def images_fetcher(self): - mdownloader.downloader(self.config, self.args) + mdownloader.downloader(self.config, self.args, chapterids=[]) # Worker to convert all downloaded chapters into ebooks def image_converter(self): - mconverter.RecursiveConverter(self.config) + mconverter.ConverterHandler(self.config, self.args, chapterids=[]) # Worker to convert all downloaded chapters into ebooks def send_ebooks(self): From 15289752c32f8b6b437c98957606283df81c7d74 Mon Sep 17 00:00:00 2001 From: Schemen Date: Tue, 21 Nov 2017 21:19:14 +0100 Subject: [PATCH 21/28] Improve Output --- m2em.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/m2em.py b/m2em.py index 9961167..d4a375c 100755 --- a/m2em.py +++ b/m2em.py @@ -324,11 +324,14 @@ def run(self): if self.args.start: daemon = True while daemon: - if not self.args.daemon: + if self.args.daemon: + logging.info("Don't forget that the daemon only handles data younger than 24h Hours!") + else: daemon = False + logging.info("#########################") logging.info("Starting Loop at %s" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - logging.info("Don't forget that the loop only handles data younger than 24h Hours!") + logging.info("Starting RSS Data Fetcher!") self.parse_add_feeds() From 284bf5472c04af18d161a8862eeebc3bc559bfb0 Mon Sep 17 00:00:00 2001 From: Schemen Date: Tue, 21 Nov 2017 22:23:44 +0100 Subject: [PATCH 22/28] Switch sender to Class & handler, add ability to call seperately --- bin/m2emSender.py | 211 +++++++++++++++++++++------------------ bin/m2emSenderHandler.py | 62 ++++++++++++ m2em.py | 17 ++-- 3 files changed, 185 insertions(+), 105 deletions(-) create mode 100644 bin/m2emSenderHandler.py diff --git a/bin/m2emSender.py b/bin/m2emSender.py index d3c01cb..2c375d4 100644 --- a/bin/m2emSender.py +++ b/bin/m2emSender.py @@ -2,10 +2,9 @@ import os import smtplib from email.mime.text import MIMEText -from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.base import MIMEBase -from email.utils import COMMASPACE, formatdate, make_msgid +from email.utils import formatdate, make_msgid from email.generator import Generator from email import encoders import bin.m2emHelper as helper @@ -15,108 +14,126 @@ except ImportError: from io import StringIO -def sendEbook(config): - # Load configs required here - database = config["Database"] - saveloc = config["SaveLocation"] - ebformat = config["EbookFormat"] - smtpserver = config["SMTPServer"] - serverport = config["ServerPort"] - emailadress = config["EmailAdress"] - password = config["EmailAdressPw"] - starttls = config["ServerStartSSL"] - # Debug Server Data - logging.debug("Server: %s" % smtpserver) - logging.debug("Port: %s" % serverport) +class Sender: - # Load Data from Database - chapters = helper.getChapters(database) - users = helper.getUsers(database) + def __init__(self): + self.saveloc = None + self.ebformat = None + self.mangatitle = None + self.manganame = None + self.eblocation = None + self.chapterdate = None + self.smtpserver = None + self.serverport = None + self.emailadress = None + self.password = None + self.starttls = None + self.mangaid = None + self.issent = None + self.chapterdate = None - # Debug Users: - logging.debug("Userlist:") - logging.debug(users) + # Will be defined by handler + self.users = None + self.database = None - # Start conversion loop! - for chapter in chapters: + + def data_collector(self, config, chapter): + # Load configs required here + self.saveloc = config["SaveLocation"] + self.ebformat = config["EbookFormat"] + self.smtpserver = config["SMTPServer"] + self.serverport = config["ServerPort"] + self.emailadress = config["EmailAdress"] + self.password = config["EmailAdressPw"] + self.starttls = config["ServerStartSSL"] + # get relevant data of this Manga - mangatitle = chapter[2] - mangaid = int(chapter[0]) - issent = int(chapter[8]) - manganame = chapter[11] - eblocation = str(saveloc + manganame + "/" + mangatitle + "/" + mangatitle + "." + ebformat.lower()) - - # Check if ebook has been converted yet, else skip - if not os.path.exists(eblocation): - logging.debug("Manga %s has not been converted yet." % mangatitle) - else: - - if issent != 0: - logging.debug("%s has been sent already!" % mangatitle) - else: - logging.info("Sending %s..."% mangatitle) - - - for user in users: - kindle_mail = user[3] - shouldsend = user[4] - - # Check if user wants Mails - if shouldsend == "True": - - logging.debug("Compiling Email for %s" % user[1]) - - # Compile Email - msg = MIMEMultipart() - msg['Subject'] = 'Ebook Delivery of %s' % mangatitle - msg['Date'] = formatdate(localtime=True) - msg['From'] = emailadress - msg['To'] = kindle_mail - msg['Message-ID'] = make_msgid() - - text = "Automatic Ebook delivery by m2em." - msg.attach(MIMEText(text)) - - - # Add Ebook as attachment - ebfile = open(eblocation, 'rb') - - attachment = MIMEBase('application', 'octet-stream', name=os.path.basename(eblocation)) - attachment.set_payload(ebfile.read()) - ebfile.close() - encoders.encode_base64(attachment) - attachment.add_header('Content-Disposition', 'attachment', - filename=os.path.basename(eblocation)) - - msg.attach(attachment) - - # Convert message to string - sio = StringIO() - gen = Generator(sio, mangle_from_=False) - gen.flatten(msg) - msg = sio.getvalue() - - # Send Email Off! - try: - server = smtplib.SMTP(smtpserver,serverport,) - if starttls: - server.starttls() - server.ehlo() - server.login(emailadress,password) - #server.sendmail(emailadress, kindle_mail, msg.as_string()) - server.sendmail(emailadress, kindle_mail, msg) - server.close() - logging.debug("Sent email to %s "% kindle_mail) - except smtplib.SMTPException as e: - logging.debug("Could not send email! %s" % e) - - - # Set Email as Sent - helper.setIsSent(mangaid,database) - logging.info("Sent %s to all requested users."% mangatitle) + self.mangatitle = chapter[2] + self.chapterdate = chapter[3] + self.mangaid = int(chapter[0]) + self.issent = int(chapter[8]) + self.manganame = chapter[11] + + # check if mangatitle or manganame contains ":" characters that OS can't handle as folders + self.mangatitle = helper.sanetizeName(self.mangatitle) + self.manganame = helper.sanetizeName(self.manganame) + + + self.eblocation = str(self.saveloc + self.manganame + "/" + self.mangatitle + "/" + self.mangatitle + "." + self.ebformat.lower()) + + + + + + + def send_eb(self): + + # Iterate through user + for user in self.users: + kindle_mail = user[3] + shouldsend = user[4] + + # Check if user wants Mails + if shouldsend == "True": + + logging.debug("Compiling Email for %s" % user[1]) + + + # Compile Email + msg = MIMEMultipart() + msg['Subject'] = 'Ebook Delivery of %s' % self.mangatitle + msg['Date'] = formatdate(localtime=True) + msg['From'] = self.emailadress + msg['To'] = kindle_mail + msg['Message-ID'] = make_msgid() + + text = "Automatic Ebook delivery by m2em." + msg.attach(MIMEText(text)) + + + # Add Ebook as attachment + ebfile = open(self.eblocation, 'rb') + + attachment = MIMEBase('application', 'octet-stream', name=os.path.basename(self.eblocation)) + attachment.set_payload(ebfile.read()) + ebfile.close() + encoders.encode_base64(attachment) + attachment.add_header('Content-Disposition', 'attachment', + filename=os.path.basename(self.eblocation)) + + msg.attach(attachment) + + # Convert message to string + sio = StringIO() + gen = Generator(sio, mangle_from_=False) + gen.flatten(msg) + msg = sio.getvalue() + + # Send Email Off! + # Debug Server Data + logging.debug("Server: %s" % self.smtpserver) + logging.debug("Port: %s" % self.serverport) + + try: + server = smtplib.SMTP(self.smtpserver,self.serverport,) + if self.starttls: + server.starttls() + server.ehlo() + server.login(self.emailadress,self.password) + #server.sendmail(emailadress, kindle_mail, msg.as_string()) + server.sendmail(self.emailadress, kindle_mail, msg) + server.close() + logging.debug("Sent email to %s "% kindle_mail) + except smtplib.SMTPException as e: + logging.debug("Could not send email! %s" % e) + + + # Set Email as Sent + helper.setIsSent(self.mangaid,self.database) + logging.info("Sent %s to all requested users."% self.mangatitle) diff --git a/bin/m2emSenderHandler.py b/bin/m2emSenderHandler.py new file mode 100644 index 0000000..feb2bd4 --- /dev/null +++ b/bin/m2emSenderHandler.py @@ -0,0 +1,62 @@ +import logging +import os +import bin.m2emHelper as helper +from bin.m2emSender import Sender + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +def SenderHandler(config, args, chapterids=[]): + # Load configs required here + database = config["Database"] + + + if not chapterids: + # Load Chapters from Database + chapters = helper.getChapters(database) + else: + # TODO Create helper function to extract chapters out of chapter IDs + pass + + # Load Users + users = helper.getUsers(database) + + # Debug Users: + logging.debug("Userlist:") + logging.debug(users) + + + + # Start conversion loop! + for chapter in chapters: + + # Initiate Sender class and fill it with data + current_sender = Sender() + current_sender.data_collector(config, chapter) + current_sender.database = database + current_sender.users = users + + # Check if ebook has been converted yet, else skip + if not os.path.exists(current_sender.eblocation): + logging.debug("Manga %s has not been converted yet." % current_sender.mangatitle) + else: + + # Check if Chapter has been sent already + if current_sender.issent != 0: + logging.debug("%s has been sent already!" % current_sender.mangatitle) + else: + + # Check if Sender loop or Sender task is selected + if not args.start: + logging.info("Sending %s..." % current_sender.mangatitle) + current_sender.send_eb() + else: + + # Only start run if chapter is younger than 24h + if helper.checkTime(current_sender.chapterdate): + logging.info("Sending %s..." % current_sender.mangatitle) + current_sender.send_eb() + else: + logging.debug("%s is older than 24h, will not be processed by daemon." % current_sender.mangatitle) diff --git a/m2em.py b/m2em.py index d4a375c..d5710ff 100755 --- a/m2em.py +++ b/m2em.py @@ -12,7 +12,7 @@ import bin.m2emRssParser as mparser import bin.m2emDownloaderHandler as mdownloader import bin.m2emConverterHandler as mconverter -import bin.m2emSender as msender +import bin.m2emSenderHandler as msender #logging.basicConfig(format='%(message)s', level=logging.DEBUG) @@ -211,9 +211,9 @@ def start_action(self): # Start downloader if self.args.action == "downloader": - logging.info("Starting downloader to get all outstanding chapters") + logging.info("Starting downloader to get all outstanding/selected chapters") self.images_fetcher() - logging.info("Finished downloading all outstanding chapters.") + logging.info("Finished downloading all chapters.") @@ -222,14 +222,15 @@ def start_action(self): elif self.args.action == "converter": - logging.info("Starting converter to convert all outstanding chapters") + logging.info("Starting converter to convert all outstanding/selected chapters") self.image_converter() - logging.info("Finished converting all remaining chapters!") + logging.info("Finished converting all chapters!") elif self.args.action == "sender": - logging.info("Action '%s' is not yet implemented." % self.args.action) - + logging.info("Starting sender to send all outstanding/selected chapters") + self.send_ebooks() + logging.info("Finished sending all chapters!") else: logging.info("%s is not a valid action. Choose between 'rssparser', 'downloader', 'converter' or 'sender'"% self.args.action) @@ -255,7 +256,7 @@ def image_converter(self): # Worker to convert all downloaded chapters into ebooks def send_ebooks(self): - msender.sendEbook(self.config) + msender.SenderHandler(self.config, self.args, chapterids=[]) From 0a26b972eff8f064ab0529d478fbf989197e4cb4 Mon Sep 17 00:00:00 2001 From: Schemen Date: Fri, 24 Nov 2017 13:01:55 +0100 Subject: [PATCH 23/28] define mangadata before it gets returned empty --- bin/m2emHelper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/m2emHelper.py b/bin/m2emHelper.py index 9a93922..709a6da 100644 --- a/bin/m2emHelper.py +++ b/bin/m2emHelper.py @@ -720,6 +720,7 @@ def getMangaData(url,entry): # Get source of to decide which parser to use origin = getSourceURL(url) + mangadata=[] # Mangastream Parser if origin == "mangastream.com": @@ -767,7 +768,7 @@ def getMangaData(url,entry): else: - pass + logging.error("Not supportet origin!") # Return mangadata return mangadata From 2f989ce22f6dc07dfaf2bc728a69d75e0aad37cb Mon Sep 17 00:00:00 2001 From: Schemen Date: Fri, 24 Nov 2017 13:28:28 +0100 Subject: [PATCH 24/28] Mangastream had a DNS Change --- bin/m2emDownloader.py | 2 +- bin/m2emHelper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/m2emDownloader.py b/bin/m2emDownloader.py index 5549e63..90c4508 100644 --- a/bin/m2emDownloader.py +++ b/bin/m2emDownloader.py @@ -63,7 +63,7 @@ def data_processor(self): # Get image urls! # Mangastream Parser - if self.origin == "mangastream.com": + if self.origin == "mangastream.com" or self.origin == "readms.net": urllist = msparser.getPagesUrl(self.mangastarturl,self.mangapages) diff --git a/bin/m2emHelper.py b/bin/m2emHelper.py index 709a6da..56cf855 100644 --- a/bin/m2emHelper.py +++ b/bin/m2emHelper.py @@ -722,7 +722,7 @@ def getMangaData(url,entry): mangadata=[] # Mangastream Parser - if origin == "mangastream.com": + if origin == "mangastream.com" or origin == "readms.net": logging.debug("Getting Mangadata from Mangastream.com for %s" % url) From 4a6fb68302d44e54c6f5b9d1f6eed2fdac1c0d4f Mon Sep 17 00:00:00 2001 From: Schemen Date: Fri, 24 Nov 2017 19:26:22 +0100 Subject: [PATCH 25/28] Implement direct Actions --download, --convert and --send, you can pass a single or multiple chapter IDs to it --- bin/m2emConverterHandler.py | 54 +++++++++++++++++++++++++++++----- bin/m2emDownloaderHandler.py | 56 ++++++++++++++++++++++++++++++------ bin/m2emHelper.py | 32 +++++++++++++++++++++ bin/m2emSenderHandler.py | 48 ++++++++++++++++++++++++++----- m2em.py | 43 +++++++++++++++++++++++++-- 5 files changed, 208 insertions(+), 25 deletions(-) diff --git a/bin/m2emConverterHandler.py b/bin/m2emConverterHandler.py index 549ae12..95b107f 100644 --- a/bin/m2emConverterHandler.py +++ b/bin/m2emConverterHandler.py @@ -1,19 +1,16 @@ import logging +import os import bin.m2emHelper as helper from bin.m2emConverter import Converter -def ConverterHandler(config, args, chapterids=[]): +def ConverterHandler(config, args): # Load configs required here database = config["Database"] - if not chapterids: - # Load Chapters from Database - chapters = helper.getChapters(database) - else: - # TODO Create helper function to extract chapters out of chapter IDs - pass + # Load Chapters! + chapters = helper.getChapters(database) # Start conversion loop! @@ -44,3 +41,46 @@ def ConverterHandler(config, args, chapterids=[]): else: logging.debug("%s is older than 24h, will not be processed by daemon." % current_conversation.mangatitle) + + + +def directConverter(config, chapterids=[]): + + logging.debug("Following Chapters are directly converted:") + logging.debug(chapterids) + + # Load configs required here + database = config["Database"] + + + chapters = helper.getChaptersFromID(database, chapterids) + + + if not chapters: + logging.error("No Chapters found with said ID!") + else: + # Start conversion loop! + for chapter in chapters: + + # Verify if chapter has been downloaded already + if not helper.verifyDownload(config, chapter): + logging.info("Manga %s has not been downloaded!" % chapter[2]) + else: + + + # Spawn an Converter Object & get basic data from database & config + current_conversation = Converter() + current_conversation.data_collector(config,chapter) + + if os.path.exists(current_conversation.cbzlocation): + logging.info("Manga %s converted to CBZ already!" % current_conversation.mangatitle) + else: + logging.info("Starting conversion to CBZ of %s..." % current_conversation.mangatitle) + current_conversation.cbz_creator() + + # Start conversion to Ebook format! + if os.path.exists(current_conversation.eblocation): + logging.info("Manga %s converted to Ebook already!" % current_conversation.mangatitle) + else: + logging.info("Starting conversion to Ebook of %s..." % current_conversation.mangatitle) + current_conversation.eb_creator() diff --git a/bin/m2emDownloaderHandler.py b/bin/m2emDownloaderHandler.py index aa9f06a..2ae6cca 100644 --- a/bin/m2emDownloaderHandler.py +++ b/bin/m2emDownloaderHandler.py @@ -8,19 +8,13 @@ ''' downloadHandler ''' -def downloader(config, args, chapterids=[]): +def downloader(config, args): # Load configs required here database = config["Database"] - - if not chapterids: - # Load Chapters from Database - chapters = helper.getChapters(database) - else: - # TODO Create helper function to extract chapters out of chapter IDs - pass + chapters = helper.getChapters(database) if args.start: logging.debug("The loop will only consider Chapters younger than 24h!") @@ -62,3 +56,49 @@ def downloader(config, args, chapterids=[]): current_chapter.downloader() else: logging.debug("%s is older than 24h, will not be processed by daemon." % current_chapter.mangatitle) + + +def directDownloader(config, chapterids=[]): + + logging.debug("Following Chapters are directly converted:") + logging.debug(chapterids) + + # Load configs required here + database = config["Database"] + + + chapters = helper.getChaptersFromID(database, chapterids) + + # Load Users + users = helper.getUsers(database) + + # Debug Users: + logging.debug("Userlist:") + logging.debug(users) + + + if not chapters: + logging.error("No Chapters found with said ID!") + else: + # Start conversion loop! + for chapter in chapters: + + # Initialize Downloader class & load basic params + current_chapter = Downloader() + current_chapter.data_collector(config, chapter) + + # Check if the old DL location is being used and fix it! + oldlocation = str(current_chapter.saveloc + current_chapter.mangatitle) + newlocation = str(current_chapter.saveloc + current_chapter.manganame) + if os.path.isdir(oldlocation): + logging.info("Moving %s from old DL location to new one..." % current_chapter.mangatitle) + helper.createFolder(newlocation) + move(oldlocation, newlocation) + + # Check if chapter needs to be downloaded + if helper.verifyDownload(config, chapter): + logging.info("Manga %s downloaded already!" % current_chapter.mangatitle) + else: + + current_chapter.data_processor() + current_chapter.downloader() diff --git a/bin/m2emHelper.py b/bin/m2emHelper.py index 56cf855..2cebd02 100644 --- a/bin/m2emHelper.py +++ b/bin/m2emHelper.py @@ -668,6 +668,38 @@ def getChapters(database): +''' +Function that gets chapters from IDs and returns it +Returns: __chapterdata +''' +def getChaptersFromID(database, chapterids): + + # Open Database + try: + conn = sqlite3.connect(database) + except Exception as e: + print("Could not connect to DB %s" % e) + + c = conn.cursor() + logging.debug("Succesfully Connected to DB %s" % database) + + __chapterdata = [] + + for i in chapterids: + # Get Data + __data = c.execute("SELECT * FROM chapter where chapterid=(?)", (i,)) + __chapterdata = __chapterdata + __data.fetchall() + + logging.debug("Passed chapters:") + logging.debug(__chapterdata) + return __chapterdata + + + + + + + ''' diff --git a/bin/m2emSenderHandler.py b/bin/m2emSenderHandler.py index feb2bd4..d894a70 100644 --- a/bin/m2emSenderHandler.py +++ b/bin/m2emSenderHandler.py @@ -8,17 +8,12 @@ except ImportError: from io import StringIO -def SenderHandler(config, args, chapterids=[]): +def SenderHandler(config, args,): # Load configs required here database = config["Database"] - if not chapterids: - # Load Chapters from Database - chapters = helper.getChapters(database) - else: - # TODO Create helper function to extract chapters out of chapter IDs - pass + chapters = helper.getChapters(database) # Load Users users = helper.getUsers(database) @@ -60,3 +55,42 @@ def SenderHandler(config, args, chapterids=[]): current_sender.send_eb() else: logging.debug("%s is older than 24h, will not be processed by daemon." % current_sender.mangatitle) + + +def directSender(config, chapterids=[]): + + logging.debug("Following Chapters are directly sent:") + logging.debug(chapterids) + + # Load configs required here + database = config["Database"] + + + chapters = helper.getChaptersFromID(database, chapterids) + + # Load Users + users = helper.getUsers(database) + + # Debug Users: + logging.debug("Userlist:") + logging.debug(users) + + + if not chapters: + logging.error("No Chapters found with said ID!") + else: + # Start conversion loop! + for chapter in chapters: + + # Initiate Sender class and fill it with data + current_sender = Sender() + current_sender.data_collector(config, chapter) + current_sender.database = database + current_sender.users = users + + # Check if ebook has been converted yet, else skip + if not os.path.exists(current_sender.eblocation): + logging.debug("Manga %s has not been converted yet." % current_sender.mangatitle) + else: + logging.info("Sending %s..." % current_sender.mangatitle) + current_sender.send_eb() diff --git a/m2em.py b/m2em.py index d5710ff..29507a0 100755 --- a/m2em.py +++ b/m2em.py @@ -60,6 +60,9 @@ def read_arguments(self): action="store_true") parser.add_argument("-s", "--start", help="Starts one loop", action="store_true") + parser.add_argument("--send", help="Sends Chapter directly by chapter ID. Multiple IDs can be given", nargs = '*',) + parser.add_argument("--convert", help="Converts Chapter directly by chapter ID. Multiple IDs can be given", default=[], nargs = '*',) + parser.add_argument("--download", help="Downloads Chapter directly by chapter ID. Multiple IDs can be given", default=[], nargs = '*',) parser.add_argument("-a", "--action", help="Start action. Options are: rss (collecting feed data), downloader, converter or sender ") parser.add_argument("-ss", "--switch-send", help="Pass ID of User. Switches said user Send eBook status") parser.add_argument("-sc", "--switch-chapter", help="Pass ID of Chapter. Switches said Chapter Sent status") @@ -88,6 +91,9 @@ def read_arguments(self): and self.args.delete_user is None \ and self.args.switch_chapter is None \ and self.args.switch_send is None \ + and self.args.send is None \ + and self.args.download is None \ + and self.args.convert is None \ and self.args.add_user is False \ and not any([self.args.add_user, self.args.create_db, @@ -238,6 +244,24 @@ def start_action(self): + ''' + direct callers + ''' + def send_chapter(self): + msender.directSender(self.config, self.args.send) + pass + + + def convert_chapter(self): + mconverter.directConverter(self.config, self.args.convert) + pass + + + def download_chapter(self): + mdownloader.directDownloader(self.config, self.args.download) + pass + + ''' This are the worker, one round @@ -248,15 +272,15 @@ def parse_add_feeds(self): # Worker to fetch all images def images_fetcher(self): - mdownloader.downloader(self.config, self.args, chapterids=[]) + mdownloader.downloader(self.config, self.args) # Worker to convert all downloaded chapters into ebooks def image_converter(self): - mconverter.ConverterHandler(self.config, self.args, chapterids=[]) + mconverter.ConverterHandler(self.config, self.args) # Worker to convert all downloaded chapters into ebooks def send_ebooks(self): - msender.SenderHandler(self.config, self.args, chapterids=[]) + msender.SenderHandler(self.config, self.args) @@ -321,6 +345,19 @@ def run(self): self.start_action() return + if self.args.send: + self.send_chapter() + return + + if self.args.download: + self.download_chapter() + return + + if self.args.convert: + self.convert_chapter() + return + + # Mainloop if self.args.start: daemon = True From 0fcc98e48ba9e0fd4c47e86fd399389abc261a88 Mon Sep 17 00:00:00 2001 From: Schemen Date: Fri, 24 Nov 2017 19:45:22 +0100 Subject: [PATCH 26/28] add version function --- bin/_version.py | 1 + m2em.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 bin/_version.py diff --git a/bin/_version.py b/bin/_version.py new file mode 100644 index 0000000..37ac431 --- /dev/null +++ b/bin/_version.py @@ -0,0 +1 @@ +__version__ = "v0.3.0" \ No newline at end of file diff --git a/m2em.py b/m2em.py index 29507a0..b47a19c 100755 --- a/m2em.py +++ b/m2em.py @@ -7,6 +7,7 @@ import configparser import datetime import validators +from bin._version import __version__ # Start of the fun! import bin.m2emHelper as helper import bin.m2emRssParser as mparser @@ -63,7 +64,7 @@ def read_arguments(self): parser.add_argument("--send", help="Sends Chapter directly by chapter ID. Multiple IDs can be given", nargs = '*',) parser.add_argument("--convert", help="Converts Chapter directly by chapter ID. Multiple IDs can be given", default=[], nargs = '*',) parser.add_argument("--download", help="Downloads Chapter directly by chapter ID. Multiple IDs can be given", default=[], nargs = '*',) - parser.add_argument("-a", "--action", help="Start action. Options are: rss (collecting feed data), downloader, converter or sender ") + parser.add_argument("-a", "--action", help="Start action. Options are: rssparser (collecting feed data), downloader, converter or sender ") parser.add_argument("-ss", "--switch-send", help="Pass ID of User. Switches said user Send eBook status") parser.add_argument("-sc", "--switch-chapter", help="Pass ID of Chapter. Switches said Chapter Sent status") parser.add_argument("-dc", "--delete-chapter", help="Pass ID of Chapter. Deletes said Chapter") @@ -73,6 +74,8 @@ def read_arguments(self): action="store_true") parser.add_argument("-d", "--debug", help="Debug Mode", action="store_true") + parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__) + self.args = parser.parse_args() # Logging From cdbe1c0e266e421dd58d16e14c6625fd205fbf4b Mon Sep 17 00:00:00 2001 From: Schemen Date: Fri, 24 Nov 2017 20:09:15 +0100 Subject: [PATCH 27/28] Fix argparse --- m2em.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/m2em.py b/m2em.py index b47a19c..1d875bc 100755 --- a/m2em.py +++ b/m2em.py @@ -61,7 +61,7 @@ def read_arguments(self): action="store_true") parser.add_argument("-s", "--start", help="Starts one loop", action="store_true") - parser.add_argument("--send", help="Sends Chapter directly by chapter ID. Multiple IDs can be given", nargs = '*',) + parser.add_argument("--send", help="Sends Chapter directly by chapter ID. Multiple IDs can be given", default=[], nargs = '*',) parser.add_argument("--convert", help="Converts Chapter directly by chapter ID. Multiple IDs can be given", default=[], nargs = '*',) parser.add_argument("--download", help="Downloads Chapter directly by chapter ID. Multiple IDs can be given", default=[], nargs = '*',) parser.add_argument("-a", "--action", help="Start action. Options are: rssparser (collecting feed data), downloader, converter or sender ") @@ -94,9 +94,6 @@ def read_arguments(self): and self.args.delete_user is None \ and self.args.switch_chapter is None \ and self.args.switch_send is None \ - and self.args.send is None \ - and self.args.download is None \ - and self.args.convert is None \ and self.args.add_user is False \ and not any([self.args.add_user, self.args.create_db, @@ -105,6 +102,9 @@ def read_arguments(self): self.args.list_chapters_all, self.args.list_feeds, self.args.list_users, + self.args.download, + self.args.convert, + self.args.send, self.args.start,]): logging.error("At least one argument is required!") From 1d5fd0e58e8f26789d6aaad57d5cef5af1a3f0a9 Mon Sep 17 00:00:00 2001 From: Schemen Date: Fri, 24 Nov 2017 20:09:32 +0100 Subject: [PATCH 28/28] Update Readme to v0.3.0 --- README.md | 66 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 60729e2..d2900f0 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,29 @@ As a concept, M2em has different workers that run in a loop. All Chapter/user da * Converter - Converting images into ebooks * Sender - Compiling & Sending Emails to users and marking them as SENT -With the sourceparser you can add support of a Webhost. + +### The Loop Run & Daemon Loop Run +If you start m2em in loop mode (with or without --daemon) it will only consider any action with elements that that are +younger than 24h hours. + +The use of that is having it running on the server 24/7, waiting for updates from the feeds and ONLY handling said updates. + +### Direct action +You can start a part of the loop without the restriction of 24h. Use the -a (--action) command with either element you wish to start. + +Example: if you wish to download all chapters you have saved in your database, you start the download action. +``` +./m2em.py --action downloader +``` +### Chapter action +You can directly apply an action to one chapter with the options --download, --convert or --send. You need to pass +the ID of said chapter, you can find that out with "-Lc" or "-lc". +You can pass multiple IDs. +Example: +``` +./m2em.py --download 100 #Downloads chapter with ID 100 +``` + ## Supported Websites * Mangastream @@ -56,10 +78,11 @@ With the sourceparser you can add support of a Webhost. ### Help output: ``` -usage: m2em.py [-h] [-af ADD_FEED] [-au] [-lc] [-Lc] [-lf] [-lu] [-cd] [-a] - [-s] [-ss SWITCH_SEND] [-sc SWITCH_CHAPTER] - [-dc DELETE_CHAPTER] [-du DELETE_USER] [-df DELETE_FEED] - [--daemon] [-d] +usage: m2em.py [-h] [-af ADD_FEED] [-au] [-lc] [-Lc] [-lf] [-lu] [-cd] [-s] + [--send [SEND [SEND ...]]] [--convert [CONVERT [CONVERT ...]]] + [--download [DOWNLOAD [DOWNLOAD ...]]] [-a ACTION] + [-ss SWITCH_SEND] [-sc SWITCH_CHAPTER] [-dc DELETE_CHAPTER] + [-du DELETE_USER] [-df DELETE_FEED] [--daemon] [-d] [-v] Manga to eManga - m2em @@ -75,10 +98,19 @@ optional arguments: -lf, --list-feeds Lists all feeds -lu, --list-users Lists all Users -cd, --create-db Creates DB. Uses Configfile for Naming - -a, --action Start action. Options are: rss (collecting feed data), - downloader, converter or sender - -s, --start Start a single loop. If no arguments are passed, a - single loop will also start + -s, --start Starts one loop + --send [SEND [SEND ...]] + Sends Chapter directly by chapter ID. Multiple IDs can + be given + --convert [CONVERT [CONVERT ...]] + Converts Chapter directly by chapter ID. Multiple IDs + can be given + --download [DOWNLOAD [DOWNLOAD ...]] + Downloads Chapter directly by chapter ID. Multiple IDs + can be given + -a ACTION, --action ACTION + Start action. Options are: rssparser (collecting feed + data), downloader, converter or sender -ss SWITCH_SEND, --switch-send SWITCH_SEND Pass ID of User. Switches said user Send eBook status -sc SWITCH_CHAPTER, --switch-chapter SWITCH_CHAPTER @@ -91,6 +123,8 @@ optional arguments: Pass ID of Feed. Deletes said Feed --daemon Run as daemon -d, --debug Debug Mode + -v, --version show program's version number and exit + ``` ## Initial Data @@ -132,13 +166,20 @@ ServerStartSSL = True To start a single run through the workers, you can simply execute the main program: ``` -./m2em.py +./m2em.py -s ``` -If you wish to run the program as a daemon, start it with the option "--daemon". It will re-run at the config "Sleep" in second. +If you wish to run the program as a daemon, start it with the option "--daemon" as well. It will re-run at the config "Sleep" in second. ``` -./m2em.py --daemon +./m2em.py -s --daemon +``` + +If you wish to disable/enable sending status of a user, use the -ss command ``` +./m2em.py -ss +``` + + ### A complete run with nothing happening: ``` @@ -159,7 +200,6 @@ Finished sending ebooks! Everything else should be self-explanatory with the "-h" option. ## Known Issues -* There is a huge data load in the beginning. It is recommended to only activate sending of Emails after one complete run * MangaFox has issues with SSL Verification on some systems. For now, Simply add the http feed. Please Open an issue if you find anything!