Browse Source

Enable multiprocessing on multiple computation (folder,gif)

tags/v1.2.10
PommeDroid 1 year ago
parent
commit
4e3cf3a8c6

+ 4
- 0
.gitignore View File

@@ -65,3 +65,7 @@ venv.bak/
# flake
.flake8
requirements_flake8.txt

# Test
requirements_dev.txt
tests

+ 2
- 2
argv/daemon.py View File

@@ -3,9 +3,9 @@ import os
import daemon
from argv.checkpoints import arg_checkpoints, set_arg_checkpoints, check_arg_checkpoints
from argv.common import arg_debug, arg_help, arg_version
from argv.run import arg_json_folder_name, arg_json_args, arg_n_core, arg_gpu, arg_cpu, arg_preferences, \
from argv.run import arg_json_folder_name, arg_json_args, arg_gpu, arg_cpu, arg_preferences, \
arg_color_transfer, arg_ignore_size, arg_auto_resize_crop, arg_auto_resize, \
arg_auto_rescale
arg_auto_rescale, arg_n_core
from argv.run.config import set_arg_preference, set_gpu_ids



+ 2
- 2
argv/run/__init__.py View File

@@ -3,7 +3,7 @@ from argv.checkpoints import arg_checkpoints
from argv.common import arg_debug, arg_help, arg_version
from argv.run.argument import arg_altered, arg_auto_rescale, arg_auto_resize, arg_auto_resize_crop, arg_color_transfer,\
arg_cpu, arg_gpu, arg_ignore_size, arg_input, arg_json_args, arg_json_folder_name, arg_n_run, \
arg_output, arg_overlay, arg_preferences, arg_step, arg_gan_persistent
arg_output, arg_overlay, arg_preferences, arg_step, arg_gan_persistent, arg_n_core


def init_run_parser(subparsers):
@@ -39,7 +39,7 @@ def init_run_parser(subparsers):
arg_cpu(processing_mod)
arg_gpu(processing_mod)
arg_checkpoints(run_parser)
# arg_n_core(run_parser) TODO Broken Fix this
arg_n_core(run_parser)
arg_gan_persistent(run_parser)

arg_json_args(run_parser)

+ 2
- 2
argv/run/argument.py View File

@@ -120,8 +120,8 @@ def arg_n_core(parser):
parser.add_argument(
"--n-cores",
type=int,
default=1,
help="Number of cpu cores to use. Default : 1",
default=4,
help="Number of cpu cores to use. Default : 4",
)



+ 10
- 2
config.py View File

@@ -60,5 +60,13 @@ class Config:

:return: <boolean> True is multiprocessing can be use
"""
# return Config.args['gpu_ids'] is None and Config.args['n_cores'] > 1
return False # TODO Fix multiprocessing
return Config.args['n_cores'] > 1

@staticmethod
def cuda_multiprocessing():
"""
Return multiprocessing status.

:return: <boolean> True is multiprocessing can be use
"""
return Config.args['gpu_ids'] is None and Config.args['n_cores'] > 1

+ 3
- 1
processing/__init__.py View File

@@ -15,13 +15,15 @@ class Processing:
:param args: <dict> settings for the transformation
:return: <RGB> image
"""
self.__start = time.time()
self.running = True
self._start = time.time()
self._args = Conf.args.copy() if config is None else config.copy()
self._info_start_run()
self._setup(*args)
r = self._execute(*args)
self._clean(*args)
self._info_end_run()
self.running = False
return r

def _info_start_run(self):

+ 3
- 3
processing/image.py View File

@@ -5,7 +5,7 @@ import sys
from config import Config as Conf
from processing import Processing
from processing.utils import select_phases
from processing.worker import get_worker
from processing.worker import run_worker
from utils import camel_case_to_str, write_image
from loader import Loader

@@ -56,8 +56,8 @@ class ImageProcessing(Processing):

:return: None
"""
for p in (get_worker(x) for x in self.__phases[self.__starting_step:self.__ending_step]):
r = p.run(*[self.__image_steps[i] for i in p.input_index], config=self._args)
for p in (x for x in self.__phases[self.__starting_step:self.__ending_step]):
r = run_worker(p, self.__image_steps, config=self._args)
self.__image_steps.append(r)

if self.__altered_path:

+ 13
- 9
processing/utils.py View File

@@ -2,6 +2,10 @@ import os

from config import Config as Conf
from utils import check_shape
from transform.gan.mask import CorrectToMask, MaskrefToMaskdet, MaskfinToNude
from transform.opencv.correct import DressToCorrect, ColorTransfer
from transform.opencv.mask import MaskToMaskref, MaskdetToMaskfin
from transform.opencv.resize import ImageToResized, ImageToCrop, ImageToOverlay, ImageToResizedCrop, ImageToRescale


def shift_step(args, shift_start_add=0, shift_end_add=0):
@@ -37,22 +41,22 @@ def add_head(args, p, add):


def overlay(args, p):
p = add_tail(args, p, "ImageToResized")
p = add_tail(args, p, "ImageToCrop")
p = add_head(args, p, "ImageToOverlay")
p = add_tail(args, p, ImageToResized)
p = add_tail(args, p, ImageToCrop)
p = add_head(args, p, ImageToOverlay)
return p


def auto_resize(args, p):
return add_tail(args, p, "ImageToResized")
return add_tail(args, p, ImageToResized)


def auto_resize_crop(args, p):
return add_tail(args, p, "ImageToResizedCrop")
return add_tail(args, p, ImageToResizedCrop)


def auto_rescale(args, p):
return add_tail(args, p, "ImageToRescale")
return add_tail(args, p, ImageToRescale)


def is_file(args):
@@ -78,12 +82,12 @@ def select_phases(args):
:return: <ImageTransform[]> list of image transformation
"""

phases = ["DressToCorrect", "CorrectToMask", "MaskToMaskref",
"MaskrefToMaskdet", "MaskdetToMaskfin", "MaskfinToNude"]
phases = [DressToCorrect, CorrectToMask, MaskToMaskref,
MaskrefToMaskdet, MaskdetToMaskfin, MaskfinToNude]

phases = scale_mod(args, phases)

if args['color_transfer']:
phases = add_head(args, phases, "ColorTransfer")
phases = add_head(args, phases, ColorTransfer)

return phases

+ 45
- 20
processing/worker.py View File

@@ -1,31 +1,56 @@
""" Wokers definition """
# TODO Implement this with a queue and mutliprocessing
import inspect
# TODO Implement this with a queue and mutliprocessingt
import threading

from config import Config as Conf
from transform.gan.mask import CorrectToMask, MaskrefToMaskdet, MaskfinToNude
from transform.opencv.correct import DressToCorrect, ColorTransfer
from transform.opencv.mask import MaskToMaskref, MaskdetToMaskfin
from transform.opencv.resize import ImageToResized, ImageToCrop, ImageToOverlay, ImageToResizedCrop, ImageToRescale

workers = {
"DressToCorrect": DressToCorrect,
"CorrectToMask": CorrectToMask,
"MaskToMaskref": MaskToMaskref,
"MaskrefToMaskdet": MaskrefToMaskdet,
"MaskdetToMaskfin": MaskdetToMaskfin,
"MaskfinToNude": MaskfinToNude,
"ImageToResized": ImageToResized,
"ImageToCrop": ImageToCrop,
"ImageToOverlay": ImageToOverlay,
"ImageToResizedCrop": ImageToResizedCrop,
"ImageToRescale": ImageToRescale,
"ColorTransfer": ColorTransfer
"gan": {
CorrectToMask: [],
MaskrefToMaskdet: [],
MaskfinToNude: [],
"sem": threading.Semaphore(1)
},
"opencv": {
DressToCorrect: [],
MaskToMaskref: [],
ImageToResized: [],
ImageToCrop: [],
ImageToOverlay: [],
ImageToResizedCrop: [],
ImageToRescale: [],
ColorTransfer: [],
MaskdetToMaskfin: [],
"sem": threading.Semaphore(Conf.args['n_cores'])
}
}
select_sem = threading.Semaphore(1)


def get_worker(name):
w = workers.get(name)
if inspect.isclass(w):
w = w()
workers[name] = w
return w
def run_worker(klass, image_step, config=None):
r = None
for k in ("gan", "opencv"):
if workers.get(k).get(klass) is not None:
Conf.log.debug("wk {}".format(workers.get(k).get(klass)))

workers.get(k).get("sem").acquire()

select_sem.acquire()
if len(workers.get(k).get(klass)) == 0:
w = klass()
else:
w = workers.get(k).get(klass).pop(0)
select_sem.release()

r = w.run(*[image_step[i] for i in w.input_index], config=config)

select_sem.acquire()
workers.get(k).get(klass).append(w)
select_sem.release()

workers.get(k).get("sem").release()
return r

+ 9
- 0
transform/__init__.py View File

@@ -5,6 +5,10 @@ from processing import Processing
class ImageTransform(Processing):
"""Abstract Image Transformation Class."""

class InvalidNumberOfArgs(ValueError):
def __str__(self):
return "Invalid nubmmer of arguments given"

def __init__(self, input_index=(-1,)):
"""
Image Transformation Class Constructor.
@@ -15,3 +19,8 @@ class ImageTransform(Processing):

super().__init__()
self.input_index = input_index

def run(self, *args, config=None):
if len(args) != len(self.input_index):
raise ImageTransform.InvalidNumberOfArgs()
return super().run(*args, config=config)

+ 0
- 1
transform/gan/mask.py View File

@@ -28,7 +28,6 @@ class CorrectToMask(MaskImageTransformGAN):
:param input_index: <tuple> index where to take the inputs (default is (-1) for previous transformation)
:param args: <dict> args parameter to run the image transformation (default use Conf.args)
"""
Conf.log.debug("HERE")
super().__init__("correct_to_mask", input_index=input_index)



+ 1
- 1
transform/opencv/correct.py View File

@@ -31,7 +31,7 @@ class DressToCorrect(ImageTransformOpenCV):
"""
if img.shape[2] != 3:
raise AssertionError()
if not 0 < percent < 100:
if not 0 < percent <= 100:
raise AssertionError()

half_percent = percent / 200.0

Loading…
Cancel
Save