Browse Source

Refactor setup.py/build.py for reuse on DreamTime

master
PommeDroid 3 years ago
parent
commit
7d569cedd0
  1. 9
      requeriments-windows.txt
  2. 13
      requirements-mac.txt
  3. 9
      requirements-ubuntu.txt
  4. 1
      requirements.txt
  5. 25
      scripts/_common.py
  6. 144
      scripts/build.py
  7. 186
      scripts/setup.py

9
requeriments-windows.txt

@ -1,9 +0,0 @@ @@ -1,9 +0,0 @@
numpy==1.16.4
pillow==6.1.0
opencv-python==4.1.0.25
rsa==4.0
https://download.pytorch.org/whl/cu100/torch-1.1.0-cp36-cp36m-win_amd64.whl
torchvision==0.2.2.post3
imageio==2.5.0
python-dotenv==0.10.3
rook==0.1.73

13
requirements-mac.txt

@ -1,13 +0,0 @@ @@ -1,13 +0,0 @@
# NOTE:
# Torch on Mac does not support CUDA!
# To include GPU processing you will need to compile Torch.
numpy==1.16.4
pillow==6.1.0
opencv-python==4.1.0.25
rsa==4.0
torch==1.1.0
torchvision==0.2.2.post3
imageio==2.5.0
python-dotenv==0.10.3
rook==0.1.73

9
requirements-ubuntu.txt

@ -1,9 +0,0 @@ @@ -1,9 +0,0 @@
numpy==1.16.4
pillow==6.1.0
opencv-python==4.1.0.25
rsa==4.0
https://download.pytorch.org/whl/cu100/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
torchvision==0.2.2.post3
imageio==2.5.0
python-dotenv==0.10.3
rook==0.1.73

1
requirements-generic.txt → requirements.txt

@ -7,3 +7,4 @@ torch==1.1.0 @@ -7,3 +7,4 @@ torch==1.1.0
imageio==2.5.0
python-dotenv==0.10.3
rook==0.1.73
sentry-sdk==0.10.2

25
scripts/_common.py

@ -34,14 +34,24 @@ def check_pyinstaller(): @@ -34,14 +34,24 @@ def check_pyinstaller():
return shutil.which("pyinstaller")
def setup_log(log_lvl=logging.DEBUG):
def check_node():
return shutil.which("node")
def check_yarn():
return shutil.which("yarn")
def setup_log(log_lvl=logging.INFO):
log = logging.getLogger(__name__)
out_hdlr = logging.StreamHandler(sys.stdout)
out_hdlr.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
log.addHandler(out_hdlr)
log.setLevel(log_lvl)
if not log.hasHandlers():
out_hdlr = logging.StreamHandler(sys.stdout)
out_hdlr.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
log.addHandler(out_hdlr)
log.setLevel(log_lvl)
return log
@contextmanager
def cd(newdir):
prevdir = os.getcwd()
@ -56,4 +66,7 @@ def create_temporary_copy(path, copy_name): @@ -56,4 +66,7 @@ def create_temporary_copy(path, copy_name):
temp_dir = tempfile.gettempdir()
temp_path = os.path.join(temp_dir, copy_name)
shutil.copy2(path, temp_path)
return temp_path
return temp_path
log = setup_log()

144
scripts/build.py

@ -1,70 +1,88 @@ @@ -1,70 +1,88 @@
import argparse
import importlib
import logging
import os
import subprocess
import shutil
import sys
from _common import get_os, setup_log, OS, get_python_version, cd, check_pyinstaller
parser = argparse.ArgumentParser(description='cli builder')
parser.add_argument('-d', '--debug', action='store_true',
help='Set log level to Debug')
args = parser.parse_args()
log = setup_log(logging.DEBUG if args.debug else logging.INFO)
## System & Dependencies Check
detected_os = get_os()
detected_py = get_python_version()
log.debug("OS : {}".format(detected_os))
log.debug("Python version : {}".format(detected_py))
if detected_os == OS.UNKNOWN:
log.fatal("Unknown OS !")
exit(1)
if detected_py < (3, 5):
log.fatal("Unsupported python version !")
exit(1)
if not check_pyinstaller():
log.fatal("Pyinstaller is not install. It's a required dependency !")
exit(1)
## Build Cli
def pyinstaller_args():
pyinstaller_args = [
'--workpath=./build/',
'--specpath=.',
'-y',
'--onedir',
'--name=cli',
'main.py',
]
if detected_os == OS.LINUX:
return pyinstaller_args
if detected_os == OS.MAC:
return pyinstaller_args
if detected_os == OS.WIN:
pyinstaller_args.extend(['--add-binary=./third/msvcp/msvcp140.dll;.'])
return pyinstaller_args
log.info('Building Cli')
with cd(".."):
cmd = [sys.executable, '-m', 'PyInstaller'] + pyinstaller_args()
log.debug(cmd)
r = subprocess.run(cmd)
if r.returncode != 0:
log.fatal("Cli build failed")
spec = importlib.util.spec_from_file_location("_common",
os.path.join(os.path.dirname(os.path.abspath(__file__)), "./_common.py"))
c = importlib.util.module_from_spec(spec)
spec.loader.exec_module(c)
def add_arg_parser(parser):
parser.add_argument('-d', '--debug', action='store_true',
help='Set log level to Debug')
def check_dependencies():
## System & Dependencies Check
c.log.debug("OS : {}".format(c.get_os()))
c.log.debug("Python version : {}".format(c.get_python_version()))
if c.get_os() == c.OS.UNKNOWN:
c.log.fatal("Unknown OS !")
exit(1)
if c.get_python_version() < (3, 5):
c.log.fatal("Unsupported python version !")
exit(1)
if not c.check_pyinstaller():
c.log.fatal("Pyinstaller is not install. It's a required dependency !")
exit(1)
log.info('Cli successfully built')
log.info('Build completed!')
log.info(
'It should have generated a folder called dist/, inside you will find the final project files that you can share with everyone!')
log.info('Enjoy and remember to respect the License!')
def cli_build(args, dist_path="../dist"):
def pyinstaller_args():
pyinstaller_args = [
'--workpath=./build/',
'--specpath=.',
'-y',
'--onedir',
'--name=dreampower',
'--distpath={}'.format(dist_path),
'main.py',
]
if c.get_os() == c.OS.LINUX:
return pyinstaller_args
if c.get_os() == c.OS.MAC:
return pyinstaller_args
if c.get_os() == c.OS.WIN:
pyinstaller_args.extend(['--add-binary=./third/msvcp/msvcp140.dll;.'])
return pyinstaller_args
c.log.info('Building Cli')
with c.cd(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")):
cmd = [sys.executable, '-m', 'PyInstaller'] + pyinstaller_args()
c.log.debug(cmd)
r = subprocess.run(cmd)
if r.returncode != 0:
c.log.fatal("Cli build failed")
exit(1)
c.log.info('Cli successfully built')
def run(args, dist_path="../dist"):
cli_build(args, dist_path)
c.log.info('Build completed!')
c.log.info(
'It should have generated a folder called dist/, inside you will find '
'the final project files that you can share with everyone!')
c.log.info('Enjoy and remember to respect the License!')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='cli builder')
add_arg_parser(parser)
args = parser.parse_args()
check_dependencies()
if args.debug:
c.log.setLevel(logging.DEBUG)
## Build Cli
run(args)

186
scripts/setup.py

@ -1,89 +1,115 @@ @@ -1,89 +1,115 @@
import argparse
import fileinput
import importlib
import logging
import os
import subprocess
import sys
import os
spec = importlib.util.spec_from_file_location("_common",
os.path.join(os.path.dirname(os.path.abspath(__file__)), "./_common.py"))
c = importlib.util.module_from_spec(spec)
spec.loader.exec_module(c)
import sys
from _common import get_os, setup_log, OS, create_temporary_copy, get_python_version, cd
parser = argparse.ArgumentParser(description='cli dependencies setup')
parser.add_argument('-d', '--debug', action='store_true',
help='Set log level to Debug')
parser.add_argument('-np', '--no_pyinstaller', action='store_true',
help='Don\'t install pyinstaller')
parser.add_argument('--cpu', action='store_true',
help='No cuda support')
parser.add_argument('-pnc', '--pip_no_cache_dir', action='store_true',
help='Use --no_cache_dir for pip commands')
parser.add_argument('-pu', '--pip_user', action='store_true',
help='Use --user for pip commands')
args = parser.parse_args()
log = setup_log(logging.DEBUG if args.debug else logging.INFO)
## System & Dependencies Check
detected_os = get_os()
detected_py = get_python_version()
log.debug("OS : {}".format(detected_os))
log.debug("Python version : {}".format(detected_py))
if detected_os == OS.UNKNOWN:
log.fatal("Unknown OS !")
exit(1)
if detected_py < (3, 5):
log.fatal("Unsupported python version !")
exit(1)
## Cli dependencies
pip_commands_extend = (['--user'] if args.pip_user else []) + (['--no-cache-dir'] if args.pip_no_cache_dir else [])
## Pyinstaller
if not args.no_pyinstaller:
log.info('Installing pyinstaller')
def add_arg_parser(parser):
parser.add_argument('-d', '--debug', action='store_true',
help='Set log level to Debug')
parser.add_argument('-np', '--no_pyinstaller', action='store_true',
help='Don\'t install pyinstaller')
parser.add_argument('--cpu', action='store_true',
help='No cuda support')
parser.add_argument('-pnc', '--pip_no_cache_dir', action='store_true',
help='Use --no_cache_dir for pip commands')
parser.add_argument('-pu', '--pip_user', action='store_true',
help='Use --user for pip commands')
def check_dependencies():
c.log.debug("OS : {}".format(c.get_os()))
c.log.debug("Python version : {}".format(c.get_python_version()))
if c.get_os() == c.OS.UNKNOWN:
c.log.fatal("Unknown OS !")
exit(1)
if c.get_python_version() < (3, 5):
c.log.fatal("Unsupported python version !")
exit(1)
def pyinstaller(args, pip_commands_extend=None):
if pip_commands_extend is None:
pip_commands_extend = []
c.log.info('Installing pyinstaller')
r = subprocess.run([sys.executable, '-m', 'pip', 'install', 'pyinstaller'] + pip_commands_extend)
if r.returncode != 0:
log.fatal("Pyinstaller installation failed")
c.log.fatal("Pyinstaller installation failed")
exit(1)
log.info('Pyinstaller successfully installed')
def torch_version():
if args.cpu:
if detected_os == OS.LINUX:
return "https://download.pytorch.org/whl/cpu/torch-1.1.0-cp{0}{1}-cp{0}{1}m-linux_x86_64.whl".format(
*get_python_version())
if detected_os == OS.MAC:
return "torch"
if detected_os == OS.WIN:
return "https://download.pytorch.org/whl/cpu/torch-1.1.0-cp{0}{1}-cp{0}{1}m-win_amd64.whl".format(
*get_python_version())
else:
if detected_os == OS.LINUX:
return "https://download.pytorch.org/whl/cu100/torch-1.1.0-cp{0}{1}-cp{0}{1}m-linux_x86_64.whl".format(
*get_python_version())
if detected_os == OS.MAC:
log.warning(
"# MacOS Binaries dont support CUDA, install from source if CUDA is needed. This script will install "
"the cpu version.")
return "torch"
if detected_os == OS.WIN:
return "https://download.pytorch.org/whl/cu100/torch-1.1.0-cp{0}{1}-cp{0}{1}m-win_amd64.whl".format(
*get_python_version())
log.info('Installing Cli dependencies')
path = create_temporary_copy("../requirements-generic.txt", "cli-requirements.txt")
with fileinput.FileInput(path, inplace=True) as f:
for l in f:
print(l.replace("torch==1.1.0", torch_version()), end='')
r = subprocess.run([sys.executable, '-m', 'pip', 'install', '-r', path] + pip_commands_extend)
os.remove(path)
if r.returncode != 0:
log.fatal("Cli dependencies installation failed")
exit(1)
log.info('Cli dependencies successfully installed')
c.log.info('Pyinstaller successfully installed')
def cli_setup(args, pip_commands_extend=None):
if pip_commands_extend is None:
pip_commands_extend = []
def torch_version():
if args.cpu:
if c.get_os() == c.OS.LINUX:
return "https://download.pytorch.org/whl/cpu/torch-1.1.0-cp{0}{1}-cp{0}{1}m-linux_x86_64.whl".format(
*c.get_python_version())
if c.get_os() == c.OS.MAC:
return "torch"
if c.get_os() == c.OS.WIN:
return "https://download.pytorch.org/whl/cpu/torch-1.1.0-cp{0}{1}-cp{0}{1}m-win_amd64.whl".format(
*c.get_python_version())
else:
if c.get_os() == c.OS.LINUX:
return "https://download.pytorch.org/whl/cu100/torch-1.1.0-cp{0}{1}-cp{0}{1}m-linux_x86_64.whl".format(
*c.get_python_version())
if c.get_os() == c.OS.MAC:
c.log.warning(
"# MacOS Binaries dont support CUDA, install from source if CUDA is needed. "
"This script will install the cpu version.")
return "torch"
if c.get_os() == c.OS.WIN:
return "https://download.pytorch.org/whl/cu100/torch-1.1.0-cp{0}{1}-cp{0}{1}m-win_amd64.whl".format(
*c.get_python_version())
c.log.info('Installing Cli dependencies')
path = c.create_temporary_copy(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "../requirements.txt"), "cli-requirements.txt")
with fileinput.FileInput(path, inplace=True) as f:
for l in f:
print(l.replace("torch==1.1.0", torch_version()), end='')
r = subprocess.run([sys.executable, '-m', 'pip', 'install', '-r', path] + pip_commands_extend)
os.remove(path)
if r.returncode != 0:
c.log.fatal("Cli dependencies installation failed")
exit(1)
c.log.info('Cli dependencies successfully installed')
def run(args):
## System & Dependencies Check
check_dependencies()
if args.debug:
c.log.setLevel(logging.DEBUG)
## Cli dependencies
pip_commands_extend = (['--user'] if args.pip_user else []) + (['--no-cache-dir'] if args.pip_no_cache_dir else [])
## Pyinstaller
if not args.no_pyinstaller:
pyinstaller(pip_commands_extend)
cli_setup(args, pip_commands_extend)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='cli dependencies setup')
add_arg_parser(parser)
args = parser.parse_args()
run(args)

Loading…
Cancel
Save