From fed6c12231f82d240b5c915c08701337d3a0ed3e Mon Sep 17 00:00:00 2001 From: PommeDroid Date: Tue, 27 Aug 2019 18:12:28 +0200 Subject: [PATCH] Initial Daemon Support * add "dreampower deamon" command * simple deamon watching a tree directory patah when .start file is create in one of directory start processing this folder. --- argv/__init__.py | 8 +++++- argv/daemon.py | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++ argv/run/__init__.py | 2 +- daemon.py | 38 ++++++++++++++++++-------- main.py | 3 --- processing/folder.py | 4 +-- processing/image.py | 6 ++--- processing/utils.py | 6 ++--- utils.py | 2 ++ 9 files changed, 120 insertions(+), 24 deletions(-) diff --git a/argv/__init__.py b/argv/__init__.py index 65f95ec..60342cb 100644 --- a/argv/__init__.py +++ b/argv/__init__.py @@ -6,7 +6,7 @@ import sys from config import Config as Conf from argv.checkpoints import init_checkpoints_sub_parser, check_args_checkpoints_parser, set_args_checkpoints_parser from argv.common import arg_help, arg_debug, arg_version -from argv.daemon import init_daemon_sub_parser +from argv.daemon import init_daemon_sub_parser, check_args_daemon_parser, set_args_daemon_parser from argv.gpu_info import init_gpu_info_sub_parser from argv.run import init_run_parser from argv.run.config import set_args_run_parser, check_args_run_parser @@ -73,6 +73,9 @@ def check_args_parser(parser, args): if args.mode == "checkpoints": check_args_checkpoints_parser(parser, args) + if args.mode == "daemon": + check_args_daemon_parser(parser, args) + return parser @@ -83,6 +86,9 @@ def set_args_parser(parser, args): if args.mode == "checkpoints": set_args_checkpoints_parser(args) + if args.mode == "daemon": + set_args_daemon_parser(args) + return parser diff --git a/argv/daemon.py b/argv/daemon.py index 673b567..263ba10 100644 --- a/argv/daemon.py +++ b/argv/daemon.py @@ -1,5 +1,12 @@ +import os + import daemon +from argv.checkpoints import arg_checkpoints, set_arg_checkpoints, check_arg_checkpoints from argv.common import arg_debug, arg_help, arg_version +from argv.run import arg_json_folder_name, arg_json_args, arg_n_core, arg_gpu, arg_cpu, arg_preferences, \ + arg_color_transfer, arg_ignore_size, arg_auto_resize_crop, arg_auto_resize, \ + arg_auto_rescale +from argv.run.config import set_arg_preference, set_gpu_ids def init_daemon_sub_parser(subparsers): @@ -11,9 +18,77 @@ def init_daemon_sub_parser(subparsers): ) daemon_parser.set_defaults(func=daemon.main) + # conflicts handler + processing_mod = daemon_parser.add_mutually_exclusive_group() + scale_mod = daemon_parser.add_mutually_exclusive_group() + # add daemon arguments + arg_input(daemon_parser) + arg_output(daemon_parser) + + arg_auto_rescale(scale_mod) + arg_auto_resize(scale_mod) + arg_auto_resize_crop(scale_mod) + arg_ignore_size(daemon_parser) + + arg_color_transfer(daemon_parser) + + arg_preferences(daemon_parser) + + arg_cpu(processing_mod) + arg_gpu(processing_mod) + arg_checkpoints(daemon_parser) + arg_n_core(daemon_parser) + + arg_json_args(daemon_parser) + arg_json_folder_name(daemon_parser) + arg_help(daemon_parser) arg_debug(daemon_parser) arg_version(daemon_parser) return daemon_parser + + +def set_args_daemon_parser(args): + set_arg_checkpoints(args) + set_arg_preference(args) + set_gpu_ids(args) + + +def check_args_daemon_parser(parser, args): + check_arg_input(parser, args) + check_arg_output(parser, args) + check_arg_checkpoints(parser, args) + + +def arg_input(parser): + parser.add_argument( + "-i", + "--input", + help="Path directory to watching.", + required=True + ) + + +def arg_output(parser): + parser.add_argument( + "-o", + "--output", + help="Path of directory where the transformed photo(s) will be saved.", + required=True + ) + + +def check_arg_input(parser, args): + if not args.input: + parser.error("-i, --input INPUT is required.") + if not os.path.isdir(args.input): + parser.error("Input {} directory doesn't exist.".format(args.input)) + + +def check_arg_output(parser, args): + if not args.output: + parser.error("-o, --output OUTPUT is required.") + if not os.path.isdir(args.output): + parser.error("Output {} directory doesn't exist.".format(args.output)) diff --git a/argv/run/__init__.py b/argv/run/__init__.py index 426bab5..f49045d 100644 --- a/argv/run/__init__.py +++ b/argv/run/__init__.py @@ -1,7 +1,7 @@ import main from argv.checkpoints import arg_checkpoints from argv.common import arg_debug, arg_help, arg_version -from argv.run.argument import arg_altered, arg_auto_rescale, arg_auto_resize, arg_auto_resize_crop, arg_color_transfer, \ +from argv.run.argument import arg_altered, arg_auto_rescale, arg_auto_resize, arg_auto_resize_crop, arg_color_transfer,\ arg_cpu, arg_gpu, arg_ignore_size, arg_input, arg_json_args, arg_json_folder_name, arg_n_core, arg_n_run, \ arg_output, arg_overlay, arg_preferences, arg_step diff --git a/daemon.py b/daemon.py index bae743f..10d0a32 100644 --- a/daemon.py +++ b/daemon.py @@ -1,4 +1,5 @@ """daemon logic.""" +import copy import os import sys import time @@ -7,6 +8,7 @@ from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer from config import Config as Conf +from processing.folder import FolderImageProcessing from transform.gan.mask import CorrectToMask, MaskrefToMaskdet, MaskfinToNude from transform.opencv.correct import DressToCorrect from transform.opencv.mask import MaskToMaskref, MaskdetToMaskfin @@ -26,13 +28,11 @@ class Watcher: self.__watching_dir = watching_dir self.__out_dir = out_dir - if not os.path.isdir(self.__watching_dir): - Conf.log.error("{} Watching Dir Doesn't Exit.".format(self.__watching_dir)) - sys.exit(0) - - if not os.path.isdir(self.__out_dir): - Conf.log.error("{} Output Dir Doesn't Exit.".format(self.__watching_dir)) - sys.exit(0) + for k, v in {"Watched": self.__watching_dir, "Output": self.__out_dir}.items(): + Conf.log.info("{} Directory Is {}".format(k, v)) + if not os.path.isdir(self.__watching_dir): + Conf.log.error("{} Directory {} Doesn't Exit.".format(k, v)) + sys.exit(0) def run(self): """ @@ -77,9 +77,24 @@ class Handler(FileSystemEventHandler): :param event: trigger event :return: None """ - if event.is_directory: - Conf.log.debug("Received directory created event {}.".format(event.src_path)) - # TODO Implements this + if not event.is_directory: + Conf.log.debug("Received file created event {}.".format(event.src_path)) + if os.path.basename(event.src_path) == ".start": + os.remove(event.src_path) + + start = time.time() + Conf.log.info("Execution Of {} Folder.".format(os.path.dirname(event.src_path))) + args = copy.deepcopy(Conf.args) + args.update({ + "input": os.path.dirname(event.src_path), + "output": self.__out_dir, + }) + + FolderImageProcessing(args=args).run() + + Conf.log.info("Execution of {} Folder Done in {}.".format( + os.path.dirname(event.src_path), round(time.time() - start, 2) + )) def main(_): @@ -89,4 +104,5 @@ def main(_): :param _: None :return: None """ - Watcher("test_dir", "out_dir").run() + Conf.log.info("Welcome to Dreampower Daemon") + Watcher(Conf.args['input'], Conf.args['output']).run() diff --git a/main.py b/main.py index 35e2c03..b5dc022 100644 --- a/main.py +++ b/main.py @@ -4,8 +4,6 @@ import sys import time from multiprocessing import freeze_support -import colorama - import argv from config import Config as Conf from processing import SimpleProcessing @@ -80,6 +78,5 @@ def processing_image_folder(): if __name__ == "__main__": - colorama.init() freeze_support() argv.run() diff --git a/processing/folder.py b/processing/folder.py index 84af1ba..4d8d048 100644 --- a/processing/folder.py +++ b/processing/folder.py @@ -45,7 +45,7 @@ class FolderImageProcessing(MultipleImageProcessing): os.path.join( Conf.args['output'], pathlib.Path(*pathlib.Path(r).parts[1:]), - os.path.basename(x.path) + os.path.basename(x) ) for x in args['input'] ] @@ -59,7 +59,7 @@ class FolderImageProcessing(MultipleImageProcessing): @staticmethod def __get_folder_args(args, folder_path): def add_folder_altered(args): - if args['altered']: + if args.get('altered'): args['folder_altered'] = os.path.join(args['altered'], pathlib.Path(*pathlib.Path(folder_path).parts[1:])) return args diff --git a/processing/image.py b/processing/image.py index 213eaa3..3c29a49 100644 --- a/processing/image.py +++ b/processing/image.py @@ -21,9 +21,9 @@ class ImageProcessing(Processing): self.__phases = select_phases(self._args) self.__input_path = self._args['input'] self.__output_path = self._args['output'] - self.__altered_path = self._args['altered'] - self.__starting_step = self._args['steps'][0] if self._args['steps'] else 0 - self.__ending_step = self._args['steps'][1] if self._args['steps'] else None + self.__altered_path = self._args.get('altered') + self.__starting_step = self._args['steps'][0] if self._args.get('steps') else 0 + self.__ending_step = self._args['steps'][1] if self._args.get('steps') else None Conf.log.debug("All Phases : {}".format(self.__phases)) Conf.log.debug("To Be Executed Phases : {}".format(self.__phases[self.__starting_step:self.__ending_step])) diff --git a/processing/utils.py b/processing/utils.py index d33a69a..ca42b9c 100644 --- a/processing/utils.py +++ b/processing/utils.py @@ -18,12 +18,12 @@ def shift_step(args, shift_start_add=0, shift_end_add=0): def shift_starting(args): - if args['steps'] and args['steps'][0] != 0: + if args.get('steps') and args['steps'][0] != 0: shift_step(args, shift_start_add=1) def shift_ending(args, p): - if args['steps'] and args['steps'][1] == len(p) - 1: + if args.get('steps') and args['steps'][1] == len(p) - 1: shift_step(args, shift_end_add=1) @@ -68,7 +68,7 @@ def is_file(args): def scale_mod(args, p): for mod in (overlay, auto_resize, auto_resize_crop, auto_rescale): - if args[mod.__name__]: + if args.get(mod.__name__): return mod(args, p) if os.path.isfile(Conf.args["input"]): is_file(args) diff --git a/utils.py b/utils.py index a60f10e..0be4632 100644 --- a/utils.py +++ b/utils.py @@ -6,6 +6,7 @@ import sys import zipfile from re import finditer +import colorama import coloredlogs import cv2 import imageio @@ -100,6 +101,7 @@ def setup_log(log_lvl=logging.INFO): :param log_lvl: level of the log :return: a logger """ + colorama.init() log = logging.getLogger(__name__) coloredlogs.install(level=log_lvl, fmt='[%(levelname)s] %(message)s') return log