Merge pull request #61 from private-dreamnet/multiprocess

Enable multiprocessing on multiple computation (folder,gif)

.gitignore

@@ -65,3 +65,7 @@ venv.bak/
 # flake
 .flake8
 requirements_flake8.txt
+
+# Test
+requirements_dev.txt
+tests

argv/daemon.py

@@ -3,9 +3,9 @@ import os
 import daemon
 from argv.checkpoints import arg_checkpoints, set_arg_checkpoints, check_arg_checkpoints
 from argv.common import arg_debug, arg_help, arg_version
-from argv.run import arg_json_folder_name, arg_json_args, arg_n_core, arg_gpu, arg_cpu, arg_preferences, \
+from argv.run import arg_json_folder_name, arg_json_args, arg_gpu, arg_cpu, arg_preferences, \
     arg_color_transfer, arg_ignore_size, arg_auto_resize_crop, arg_auto_resize, \
-    arg_auto_rescale
+    arg_auto_rescale, arg_n_core
 from argv.run.config import set_arg_preference, set_gpu_ids
 
 

argv/run/__init__.py

@@ -3,7 +3,7 @@ from argv.checkpoints import arg_checkpoints
 from argv.common import arg_debug, arg_help, arg_version
 from argv.run.argument import arg_altered, arg_auto_rescale, arg_auto_resize, arg_auto_resize_crop, arg_color_transfer,\
     arg_cpu, arg_gpu, arg_ignore_size, arg_input, arg_json_args, arg_json_folder_name, arg_n_run, \
-    arg_output, arg_overlay, arg_preferences, arg_step, arg_gan_persistent
+    arg_output, arg_overlay, arg_preferences, arg_step, arg_gan_persistent, arg_n_core
 
 
 def init_run_parser(subparsers):
@@ -39,7 +39,7 @@ def init_run_parser(subparsers):
     arg_cpu(processing_mod)
     arg_gpu(processing_mod)
     arg_checkpoints(run_parser)
-    # arg_n_core(run_parser) TODO Broken Fix this
+    arg_n_core(run_parser)
     arg_gan_persistent(run_parser)
 
     arg_json_args(run_parser)

argv/run/argument.py

@@ -120,8 +120,8 @@ def arg_n_core(parser):
     parser.add_argument(
         "--n-cores",
         type=int,
-        default=1,
-        help="Number of cpu cores to use. Default : 1",
+        default=4,
+        help="Number of cpu cores to use. Default : 4",
     )
 
 

config.py

@@ -60,5 +60,13 @@ class Config:
 
         :return: <boolean> True is multiprocessing can be use
         """
-        # return Config.args['gpu_ids'] is None and Config.args['n_cores'] > 1
-        return False  # TODO Fix multiprocessing
+        return Config.args['n_cores'] > 1
+
+    @staticmethod
+    def cuda_multiprocessing():
+        """
+        Return multiprocessing status.
+
+        :return: <boolean> True is multiprocessing can be use
+        """
+        return Config.args['gpu_ids'] is None and Config.args['n_cores'] > 1

processing/__init__.py

@@ -15,13 +15,15 @@ class Processing:
         :param args: <dict> settings for the transformation
         :return: <RGB> image
         """
-        self.__start = time.time()
+        self.running = True
+        self._start = time.time()
         self._args = Conf.args.copy() if config is None else config.copy()
         self._info_start_run()
         self._setup(*args)
         r = self._execute(*args)
         self._clean(*args)
         self._info_end_run()
+        self.running = False
         return r
 
     def _info_start_run(self):

processing/image.py

@@ -5,7 +5,7 @@ import sys
 from config import Config as Conf
 from processing import Processing
 from processing.utils import select_phases
-from processing.worker import get_worker
+from processing.worker import run_worker
 from utils import camel_case_to_str, write_image
 from loader import Loader
 
@@ -56,8 +56,8 @@ class ImageProcessing(Processing):
 
         :return: None
         """
-        for p in (get_worker(x) for x in self.__phases[self.__starting_step:self.__ending_step]):
-            r = p.run(*[self.__image_steps[i] for i in p.input_index], config=self._args)
+        for p in (x for x in self.__phases[self.__starting_step:self.__ending_step]):
+            r = run_worker(p, self.__image_steps, config=self._args)
             self.__image_steps.append(r)
 
             if self.__altered_path:

processing/utils.py

@@ -2,6 +2,10 @@ import os
 
 from config import Config as Conf
 from utils import check_shape
+from transform.gan.mask import CorrectToMask, MaskrefToMaskdet, MaskfinToNude
+from transform.opencv.correct import DressToCorrect, ColorTransfer
+from transform.opencv.mask import MaskToMaskref, MaskdetToMaskfin
+from transform.opencv.resize import ImageToResized, ImageToCrop, ImageToOverlay, ImageToResizedCrop, ImageToRescale
 
 
 def shift_step(args, shift_start_add=0, shift_end_add=0):
@@ -37,22 +41,22 @@ def add_head(args, p, add):
 
 
 def overlay(args, p):
-    p = add_tail(args, p, "ImageToResized")
-    p = add_tail(args, p, "ImageToCrop")
-    p = add_head(args, p, "ImageToOverlay")
+    p = add_tail(args, p, ImageToResized)
+    p = add_tail(args, p, ImageToCrop)
+    p = add_head(args, p, ImageToOverlay)
     return p
 
 
 def auto_resize(args, p):
-    return add_tail(args, p, "ImageToResized")
+    return add_tail(args, p, ImageToResized)
 
 
 def auto_resize_crop(args, p):
-    return add_tail(args, p, "ImageToResizedCrop")
+    return add_tail(args, p, ImageToResizedCrop)
 
 
 def auto_rescale(args, p):
-    return add_tail(args, p, "ImageToRescale")
+    return add_tail(args, p, ImageToRescale)
 
 
 def is_file(args):
@@ -78,12 +82,12 @@ def select_phases(args):
     :return: <ImageTransform[]> list of image transformation
     """
 
-    phases = ["DressToCorrect", "CorrectToMask", "MaskToMaskref",
-              "MaskrefToMaskdet", "MaskdetToMaskfin", "MaskfinToNude"]
+    phases = [DressToCorrect, CorrectToMask, MaskToMaskref,
+              MaskrefToMaskdet, MaskdetToMaskfin, MaskfinToNude]
 
     phases = scale_mod(args, phases)
 
     if args['color_transfer']:
-        phases = add_head(args, phases, "ColorTransfer")
+        phases = add_head(args, phases, ColorTransfer)
 
     return phases

processing/worker.py

@@ -1,31 +1,56 @@
 """ Wokers definition """
-# TODO Implement this with a queue and mutliprocessing
-import inspect
+# TODO Implement this with a queue and mutliprocessingt
+import threading
 
+from config import Config as Conf
 from transform.gan.mask import CorrectToMask, MaskrefToMaskdet, MaskfinToNude
 from transform.opencv.correct import DressToCorrect, ColorTransfer
 from transform.opencv.mask import MaskToMaskref, MaskdetToMaskfin
 from transform.opencv.resize import ImageToResized, ImageToCrop, ImageToOverlay, ImageToResizedCrop, ImageToRescale
 
 workers = {
-    "DressToCorrect": DressToCorrect,
-    "CorrectToMask": CorrectToMask,
-    "MaskToMaskref": MaskToMaskref,
-    "MaskrefToMaskdet": MaskrefToMaskdet,
-    "MaskdetToMaskfin": MaskdetToMaskfin,
-    "MaskfinToNude": MaskfinToNude,
-    "ImageToResized": ImageToResized,
-    "ImageToCrop": ImageToCrop,
-    "ImageToOverlay": ImageToOverlay,
-    "ImageToResizedCrop": ImageToResizedCrop,
-    "ImageToRescale": ImageToRescale,
-    "ColorTransfer": ColorTransfer
+    "gan": {
+        CorrectToMask: [],
+        MaskrefToMaskdet: [],
+        MaskfinToNude: [],
+        "sem": threading.Semaphore(1)
+    },
+    "opencv": {
+        DressToCorrect: [],
+        MaskToMaskref: [],
+        ImageToResized: [],
+        ImageToCrop: [],
+        ImageToOverlay: [],
+        ImageToResizedCrop: [],
+        ImageToRescale: [],
+        ColorTransfer: [],
+        MaskdetToMaskfin: [],
+        "sem": threading.Semaphore(Conf.args['n_cores'])
+    }
 }
+select_sem = threading.Semaphore(1)
 
 
-def get_worker(name):
-    w = workers.get(name)
-    if inspect.isclass(w):
-        w = w()
-        workers[name] = w
-    return w
+def run_worker(klass, image_step, config=None):
+    r = None
+    for k in ("gan", "opencv"):
+        if workers.get(k).get(klass) is not None:
+            Conf.log.debug("wk {}".format(workers.get(k).get(klass)))
+
+            workers.get(k).get("sem").acquire()
+
+            select_sem.acquire()
+            if len(workers.get(k).get(klass)) == 0:
+                w = klass()
+            else:
+                w = workers.get(k).get(klass).pop(0)
+            select_sem.release()
+
+            r = w.run(*[image_step[i] for i in w.input_index], config=config)
+
+            select_sem.acquire()
+            workers.get(k).get(klass).append(w)
+            select_sem.release()
+
+            workers.get(k).get("sem").release()
+    return r

transform/__init__.py

@@ -5,6 +5,10 @@ from processing import Processing
 class ImageTransform(Processing):
     """Abstract Image Transformation Class."""
 
+    class InvalidNumberOfArgs(ValueError):
+        def __str__(self):
+            return "Invalid nubmmer of arguments given"
+
     def __init__(self, input_index=(-1,)):
         """
         Image Transformation Class Constructor.
@@ -15,3 +19,8 @@ class ImageTransform(Processing):
 
         super().__init__()
         self.input_index = input_index
+
+    def run(self, *args, config=None):
+        if len(args) != len(self.input_index):
+            raise ImageTransform.InvalidNumberOfArgs()
+        return super().run(*args, config=config)

transform/gan/mask.py

@@ -28,7 +28,6 @@ class CorrectToMask(MaskImageTransformGAN):
         :param input_index: <tuple> index where to take the inputs (default is (-1) for previous transformation)
         :param args: <dict> args parameter to run the image transformation (default use Conf.args)
         """
-        Conf.log.debug("HERE")
         super().__init__("correct_to_mask", input_index=input_index)
 
 

transform/opencv/correct.py

@@ -31,7 +31,7 @@ class DressToCorrect(ImageTransformOpenCV):
         """
         if img.shape[2] != 3:
             raise AssertionError()
-        if not 0 < percent < 100:
+        if not 0 < percent <= 100:
             raise AssertionError()
 
         half_percent = percent / 200.0