diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 66cb3564d1..0000000000 --- a/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!requirements* -!_requirements* diff --git a/.gitignore b/.gitignore index ba4b84e9eb..3c540834e3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ !/requirements/*.py # Root files -!Dockerfile* !pyproject.toml !.gitignore !.travis.yml diff --git a/Dockerfile.cpu b/Dockerfile.cpu deleted file mode 100755 index 0c27ec9b69..0000000000 --- a/Dockerfile.cpu +++ /dev/null @@ -1,19 +0,0 @@ -FROM ubuntu:22.04 - -# To disable tzdata and others from asking for input -ENV DEBIAN_FRONTEND noninteractive -ENV FACESWAP_BACKEND cpu - -RUN apt-get update -qq -y -RUN apt-get upgrade -y -RUN apt-get install -y libgl1 libglib2.0-0 python3 python3-pip python3-tk git - -RUN ln -s $(which python3) /usr/local/bin/python - -RUN git clone --depth 1 --no-single-branch https://github.com/deepfakes/faceswap.git -WORKDIR "/faceswap" - -RUN python -m pip install --upgrade pip -RUN python -m pip --no-cache-dir install -r ./requirements/requirements_cpu.txt - -CMD ["/bin/bash"] diff --git a/Dockerfile.gpu b/Dockerfile.gpu deleted file mode 100755 index 5b9c0abd0a..0000000000 --- a/Dockerfile.gpu +++ /dev/null @@ -1,18 +0,0 @@ -FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04 - -ENV DEBIAN_FRONTEND=noninteractive -ENV FACESWAP_BACKEND nvidia - -RUN apt-get update -qq -y -RUN apt-get upgrade -y -RUN apt-get install -y libgl1 libglib2.0-0 python3 python3-pip python3-tk git - -RUN ln -s $(which python3) /usr/local/bin/python - -RUN git clone --depth 1 --no-single-branch https://github.com/deepfakes/faceswap.git -WORKDIR "/faceswap" - -RUN python -m pip install --upgrade pip -RUN python -m pip --no-cache-dir install -r ./requirements/requirements_nvidia.txt - -CMD ["/bin/bash"] diff --git a/INSTALL.md b/INSTALL.md index 63ba045f91..83a60f104b 100755 --- a/INSTALL.md +++ b/INSTALL.md @@ -38,9 +38,6 @@ - [Getting the faceswap code](#getting-the-faceswap-code) - [Setup](#setup-2) - [About some of the options](#about-some-of-the-options) -- [Docker Install Guide](#docker-install-guide) - - [Docker CPU](#docker-cpu) - - [Docker Nvidia](#docker-nvidia) - [Run the project](#run-the-project) - [Notes](#notes) @@ -74,8 +71,6 @@ The type of computations that the process does are well suited for graphics card Intel based macOS systems should work, but you will need to follow the [Manual Install](#manual-install) instructions. - All operating systems must be 64-bit. -Alternatively, there is a docker image that is based on Debian. - # Important before you proceed **In its current iteration, the project relies heavily on the use of the command line, although a gui is available. if you are unfamiliar with command line tools, you may have difficulty setting up the environment and should perhaps not attempt any of the steps described in this guide.** This guide assumes you have intermediate knowledge of the command line. @@ -131,7 +126,7 @@ To enter the virtual environment: - If you have issues/errors follow the Manual install steps below. #### Manual install -Do not follow these steps if the Easy Install above completed succesfully. +Do not follow these steps if the Easy Install above completed successfully. If you are using an Nvidia card make sure you have the correct versions of Cuda/cuDNN installed for the required version of Torch - Install tkinter (required for the GUI) by typing: `conda install tk` - Install requirements: @@ -251,7 +246,7 @@ Alternatively you can install Python (3.14 64-bit) for your distribution (links If using Conda3 then setting up virtual environments is relatively straight forward. More information can be found at [Conda Docs](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) - If using a default Python distribution then [virtualenv](https://github.com/pypa/virtualenv) and [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io) may help when you are not using docker. + If using a default Python distribution then [virtualenv](https://github.com/pypa/virtualenv) and [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io) may help. ## Getting the faceswap code @@ -270,87 +265,8 @@ If setup fails for any reason you can still manually install the packages listed ### About some of the options - CUDA: For acceleration. Requires a good nVidia Graphics Card (which supports CUDA inside) - - Docker: Provide a ready-made image. Hide trivial details. Get you straight to the project. - - nVidia-Docker: Access to the nVidia GPU on host machine from inside container. - -# Docker Install Guide - -This Faceswap repo contains Docker build scripts for CPU and Nvidia backends. The scripts will set up a Docker container for you and install the latest version of the Faceswap software. - -You must first ensure that Docker is installed and running on your system. Follow the guide for downloading and installing Docker from their website: - - - https://www.docker.com/get-started + - ROCm: For AMD GPUs under Linux/WSL2 only. Make sure you install the correct version of faceswap for your installed ROCm version -Once Docker is installed and running, follow the relevant steps for your chosen backend -## Docker CPU -To run the CPU version of Faceswap follow these steps: - -1. Build the Docker image For faceswap: -``` -docker build \ --t faceswap-cpu \ -https://raw.githubusercontent.com/deepfakes/faceswap/master/Dockerfile.cpu -``` -2. Launch and enter the Faceswap container: - - a. For the **headless/command line** version of Faceswap run: - ``` - docker run --rm -it faceswap-cpu - ``` - You can then execute faceswap the standard way: - ``` - python faceswap.py --help - ``` - b. For the **GUI** version of Faceswap run: - ``` - xhost +local: && \ - docker run --rm -it \ - -v /tmp/.X11-unix:/tmp/.X11-unix \ - -e DISPLAY=${DISPLAY} \ - faceswap-cpu - ``` - You can then launch the GUI with - ``` - python faceswap.py gui - ``` - ## Docker Nvidia -To build the NVIDIA GPU version of Faceswap, follow these steps: - -1. Nvidia Docker builds need extra resources to provide the Docker container with access to your GPU. - - a. Follow the instructions to install and apply the `Nvidia Container Toolkit` for your distribution from: - - https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html - - b. If Docker is already running, restart it to pick up the changes made by the Nvidia Container Toolkit. - -2. Build the Docker image For faceswap -``` -docker build \ --t faceswap-gpu \ -https://raw.githubusercontent.com/deepfakes/faceswap/master/Dockerfile.gpu -``` -1. Launch and enter the Faceswap container: - - a. For the **headless/command line** version of Faceswap run: - ``` - docker run --runtime=nvidia --rm -it faceswap-gpu - ``` - You can then execute faceswap the standard way: - ``` - python faceswap.py --help - ``` - b. For the **GUI** version of Faceswap run: - ``` - xhost +local: && \ - docker run --runtime=nvidia --rm -it \ - -v /tmp/.X11-unix:/tmp/.X11-unix \ - -e DISPLAY=${DISPLAY} \ - faceswap-gpu - ``` - You can then launch the GUI with - ``` - python faceswap.py gui - ``` # Run the project Once all these requirements are installed, you can attempt to run the faceswap tools. Use the `-h` or `--help` options for a list of options. diff --git a/lib/cli/launcher.py b/lib/cli/launcher.py index 4f319cdf3b..cf0919ea73 100644 --- a/lib/cli/launcher.py +++ b/lib/cli/launcher.py @@ -11,6 +11,7 @@ from lib.gpu_stats import GPUStats from lib.logger import crash_log, log_setup +from lib.system.system import VALID_TORCH from lib.utils import (FaceswapError, get_backend, get_torch_version, get_module_objects, safe_shutdown, set_backend) @@ -77,8 +78,7 @@ def _test_for_torch_version(self) -> None: FaceswapError If PyTorch is not found, or is not between versions 2.3 and 2.11 """ - min_ver = (2, 3) - max_ver = (2, 11) + min_ver, max_ver = VALID_TORCH try: import torch # noqa:F401 pylint:disable=unused-import,import-outside-toplevel except ImportError as err: diff --git a/lib/config/ini.py b/lib/config/ini.py index e48c91862c..e7537a97b8 100644 --- a/lib/config/ini.py +++ b/lib/config/ini.py @@ -1,5 +1,5 @@ #! /usr/env/bin/python3 -""" Handles interfacing between Faceswap Configs and ConfigParser .ini files """ +"""Handles interfacing between Faceswap Configs and ConfigParser .ini files""" from __future__ import annotations import logging @@ -19,13 +19,13 @@ class ConfigFile(): - """ Handles the interfacing between saved faceswap .ini configs and internal Config objects + """Handles the interfacing between saved faceswap .ini configs and internal Config objects Parameters ---------- - plugin_group : str + plugin_group The plugin group that is requesting a config file - ini_path : str | None, optional + ini_path Optional path to a .ini config file. ``None`` for default location. Default: ``None`` """ def __init__(self, plugin_group: str, ini_path: str | None = None) -> None: @@ -38,21 +38,20 @@ def __init__(self, plugin_group: str, ini_path: str | None = None) -> None: @property def _exists(self) -> bool: - """ bool : ``True`` if the config.ini file exists """ + """``True`` if the config.ini file exists""" return os.path.isfile(self._file_path) def _get_config_path(self, ini_path: str | None) -> str: - """ Return the path to the config file from the calling folder or the provided file + """Return the path to the config file from the calling folder or the provided file Parameters ---------- - ini_path : str | None + ini_path Path to a config ini file. ``None`` for default location. Returns ------- - str - The full path to the configuration file + The full path to the configuration file """ if ini_path is not None: if not os.path.isfile(ini_path): @@ -66,12 +65,11 @@ def _get_config_path(self, ini_path: str | None) -> str: return retval def _get_new_configparser(self) -> ConfigParser: - """ Obtain a fresh ConfigParser object and set it to case-sensitive + """Obtain a fresh ConfigParser object and set it to case-sensitive Returns ------- - :class:`configparser.ConfigParser` - A new ConfigParser object set to case-sensitive + A new ConfigParser object set to case-sensitive """ retval = ConfigParser(allow_no_value=True) retval.optionxform = str # type:ignore[assignment,method-assign] @@ -79,34 +77,33 @@ def _get_new_configparser(self) -> ConfigParser: # I/O def load(self) -> None: - """ Load values from the saved config ini file into our Config object """ + """Load values from the saved config ini file into our Config object""" logger.verbose("[%s] Loading config: '%s'", # type:ignore[attr-defined] self._plugin_group, self._file_path) self._parser.read(self._file_path, encoding="utf-8") def save(self) -> None: - """ Save a config file """ + """Save a config file""" logger.debug("[%s] %s config: '%s'", self._plugin_group, "Updating" if self._exists else "Saving", self._file_path) # TODO in python >= 3.14 this will error when there are delimiters in the comments - with open(self._file_path, "w", encoding="utf-8", errors="replace") as f_cfgfile: - self._parser.write(f_cfgfile) + with open(self._file_path, "w", encoding="utf-8", errors="replace") as f_cfg_file: + self._parser.write(f_cfg_file) logger.info("[%s] Saved config: '%s'", self._plugin_group, self._file_path) # .ini vs Faceswap Config checking def _sections_synced(self, app_config: dict[str, ConfigSection]) -> bool: - """ Validate that all of the sections within the application config match with all of the + """Validate that all of the sections within the application config match with all of the sections in the ini file Parameters ---------- - app_config : dict[str, :class:`ConfigSection`] + app_config The latest configuration settings from the application. Section name is key Returns ------- - bool - ``True`` if application sections and saved ini sections match + ``True`` if application sections and saved ini sections match """ given_sections = set(app_config) loaded_sections = set(self._parser.sections()) @@ -117,7 +114,7 @@ def _sections_synced(self, app_config: dict[str, ConfigSection]) -> bool: return retval def _options_synced(self, app_config: dict[str, ConfigSection]) -> bool: - """ Validate that all of the option names within the application config match with all of + """Validate that all of the option names within the application config match with all of the option names in the ini file Note @@ -126,13 +123,12 @@ def _options_synced(self, app_config: dict[str, ConfigSection]) -> bool: Parameters ---------- - app_config : dict[str, :class:`ConfigSection`] + app_config The latest configuration settings from the application. Section name is key Returns ------- - bool - ``True`` if application option names match with saved ini option names + ``True`` if application option names match with saved ini option names """ for name, section in app_config.items(): given_opts = set(opt for opt in section.options) @@ -144,20 +140,19 @@ def _options_synced(self, app_config: dict[str, ConfigSection]) -> bool: return True def _values_synced(self, app_section: ConfigSection, section: str) -> bool: - """ Validate that all of the option values within the application config match with all of + """Validate that all of the option values within the application config match with all of the option values in the ini file Parameters ---------- - app_section : :class:`ConfigSection` + app_section The latest configuration settings from the application for the given section - section : str + section The section name to check the option values for Returns ------- - bool - ``True`` if application option values match with saved ini option values + ``True`` if application option values match with saved ini option values """ # Need to also pull in keys as False is omitted from the set with just values which can # cause edge-case false negatives @@ -170,18 +165,17 @@ def _values_synced(self, app_section: ConfigSection, section: str) -> bool: return retval def _is_synced_structure(self, app_config: dict[str, ConfigSection]) -> bool: - """ Validate that all the given sections and option names within the application config + """Validate that all the given sections and option names within the application config match with their corresponding items in the save .ini file Parameters ---------- - app_config: dict[str, :class:`ConfigSection`] + app_config The latest configuration settings from the application. Section name is key Returns ------- - bool - ``True`` if the app config and saved ini config structure match + ``True`` if the app config and saved ini config structure match """ if not self._sections_synced(app_config): return False @@ -193,20 +187,19 @@ def _is_synced_structure(self, app_config: dict[str, ConfigSection]) -> bool: # .ini file insertion def format_help(self, helptext: str, is_section: bool = False) -> str: - """ Format comments for insertion into a config ini file + """Format comments for insertion into a config ini file Parameters ---------- - helptext : str + helptext The help text to be formatted - is_section : bool, optional + is_section ``True`` if the help text pertains to a section. ``False`` if it pertains to an option. Default: ``True`` Returns ------- - str - The formatted help text + The formatted help text """ logger.debug("[%s] Formatting help: (helptext: '%s', is_section: '%s')", self._plugin_group, helptext, is_section) @@ -223,15 +216,15 @@ def format_help(self, helptext: str, is_section: bool = False) -> str: return helptext def _insert_section(self, section: str, helptext: str, config: ConfigParser) -> None: - """ Insert a section into the config + """Insert a section into the config Parameters ---------- - section : str + section The section title to insert - helptext : str + helptext The help text for the config section - config : :class:`configparser.ConfigParser` + config The config parser object to insert the section into. """ logger.debug("[%s:%s] Inserting section: (helptext: '%s', config: '%s')", @@ -246,19 +239,19 @@ def _insert_option(self, helptext: str, value: str, config: ConfigParser) -> None: - """ Insert an option into a config section + """Insert an option into a config section Parameters ---------- - section : str + section The section to insert the option into - name : str + name The name of the option to insert - helptext : str + helptext The help text for the option - value : str + value The value for the option - config : :class:`configparser.ConfigParser` + config The config parser object to insert the option into """ logger.debug( @@ -269,7 +262,7 @@ def _insert_option(self, config.set(section, name, value) def _sync_from_app(self, app_config: dict[str, ConfigSection]) -> None: - """ Update the saved config.ini file from the values stored in the application config + """Update the saved config.ini file from the values stored in the application config Existing options keep their saved values as per the .ini files. New options are added with their application defined default value. Options in the .ini file not in application @@ -281,7 +274,7 @@ def _sync_from_app(self, app_config: dict[str, ConfigSection]) -> None: Parameters ---------- - app_config: dict[str, :class:`ConfigSection`] + app_config The latest configuration settings from the application. Section name is key """ logger.debug("[%s] Syncing from app", self._plugin_group) @@ -306,21 +299,20 @@ def _sync_from_app(self, app_config: dict[str, ConfigSection]) -> None: # .ini extraction def _get_converted_value(self, section: str, option: str, datatype: type) -> ConfigValueType: - """ Return a config item from the .ini file in it's correct type. + """Return a config item from the .ini file in it's correct type. Parameters ---------- - section : str + section The configuration section to obtain the config option for - option : str + option The configuration option to obtain the converted value for - datatype : type + datatype The type to return the value as Returns ------- - bool | int | float | list[str] | str - The selected configuration option in the correct data format + The selected configuration option in the correct data format """ logger.debug("[%s:%s] Getting config item: (option: '%s', datatype: %s)", self._plugin_group, section, option, datatype) @@ -343,11 +335,11 @@ def _get_converted_value(self, section: str, option: str, datatype: type) -> Con return retval def _sync_to_app(self, app_config: dict[str, ConfigSection]) -> None: - """ Update the values in the application config to those loaded from the saved config.ini. + """Update the values in the application config to those loaded from the saved config.ini. Parameters ---------- - app_config: dict[str, :class:`ConfigSection`] + app_config The latest configuration settings from the application. Section name is key """ logger.debug("[%s] Syncing to app", self._plugin_group) @@ -369,13 +361,13 @@ def _sync_to_app(self, app_config: dict[str, ConfigSection]) -> None: # .ini insertion and extraction def on_load(self, app_config: dict[str, ConfigSection]) -> None: - """ Check whether there has been any change between the current application config and + """Check whether there has been any change between the current application config and the loaded ini config. If so, update the relevant object(s) appropriately. This check will also create new config.ini files if they do not pre-exist Parameters ---------- - app_config : dict[str, :class:`ConfigSection`] + app_config The latest configuration settings from the application. Section name is key """ if not self._exists: @@ -388,12 +380,12 @@ def on_load(self, app_config: dict[str, ConfigSection]) -> None: self._sync_to_app(app_config) def update_from_app(self, app_config: dict[str, ConfigSection]) -> None: - """ Update the config.ini file to those values that are currently in Faceswap's app + """Update the config.ini file to those values that are currently in Faceswap's app config Parameters ---------- - app_config : dict[str, :class:`ConfigSection`] + app_config The latest configuration settings from the application. Section name is key """ logger.debug("[%s] Updating saved config", self._plugin_group) diff --git a/lib/config/objects.py b/lib/config/objects.py index c2800e52b0..52c14abf9f 100644 --- a/lib/config/objects.py +++ b/lib/config/objects.py @@ -440,7 +440,7 @@ class ConfigSection: options: dict[str, ConfigItem] -class ConfigReprMeta(type): +class _ConfigReprMeta(type): # Must be private or breaks automodsumm """A custom repr for printing currently selected config values""" def __repr__(cls) -> str: params = ", ".join(f"{k}={repr(v.value)}" @@ -450,7 +450,7 @@ def __repr__(cls) -> str: @dataclass -class GlobalSection(metaclass=ConfigReprMeta): +class GlobalSection(metaclass=_ConfigReprMeta): """A dataclass for holding and identifying global sub-sections for plugin groups. Any global subsections must inherit from this. diff --git a/lib/model/initializers.py b/lib/model/initializers.py index 908dd6e892..6c45540c05 100644 --- a/lib/model/initializers.py +++ b/lib/model/initializers.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -""" Custom Initializers for faceswap.py """ +"""Custom Initializers for faceswap.py""" from __future__ import annotations import logging @@ -7,8 +7,10 @@ import inspect import typing as T -from keras import backend as K, initializers, ops -from keras import saving, Variable +import torch + +from keras import backend as K, initializers +from keras import saving from keras.src.initializers.random_initializers import compute_fans import numpy as np @@ -16,27 +18,23 @@ from lib.logger import parse_class_init from lib.utils import get_module_objects -if T.TYPE_CHECKING: - from keras import KerasTensor - logger = logging.getLogger(__name__) class ICNR(initializers.Initializer): - """ ICNR initializer for checkerboard artifact free sub pixel convolution + """ICNR initializer for checkerboard artifact free sub pixel convolution Parameters ---------- - initializer: :class:`keras.initializers.Initializer` + initializer The initializer used for sub kernels (orthogonal, glorot uniform, etc.) - scale: int, optional + scale scaling factor of sub pixel convolution (up sampling from 8x8 to 16x16 is scale 2). Default: `2` Returns ------- - :class:`keras.KerasTensor` - The modified kernel weights + The modified kernel weights Example ------- @@ -46,95 +44,44 @@ class ICNR(initializers.Initializer): ---------- Andrew Aitken et al. Checkerboard artifact free sub-pixel convolution https://arxiv.org/pdf/1707.02937.pdf, https://distill.pub/2016/deconv-checkerboard/ + https://gist.github.com/A03ki/2305398458cb8e2155e8e81333f0a965 """ def __init__(self, initializer: dict[str, T.Any] | initializers.Initializer, scale: int = 2) -> None: logger.debug(parse_class_init(locals())) - self._scale = scale self._initializer = initializer - logger.debug("Initialized %s", self.__class__.__name__) - def __call__(self, shape: list[int] | tuple[int, ...], - dtype: str | None = "float32") -> KerasTensor: - """ Call function for the ICNR initializer. - - Parameters - ---------- - shape: list[int] | tuple[int, ...] - The required resized shape for the output tensor - dtype: str - The data type for the tensor - kwargs: dict[str, Any] - Standard keras initializer keyword arguments - - Returns - ------- - :class:`keras.KerasTensor` - The modified kernel weights - """ + dtype: str | None = "float32") -> torch.Tensor: shape = list(shape) - - if self._scale == 1: + if self._scale == 1: # TODO validate when moved to full torch if isinstance(self._initializer, dict): return next(i for i in self._initializer.values()) return self._initializer(shape) new_shape = shape[:3] + [shape[3] // (self._scale ** 2)] - size = [s * self._scale for s in new_shape[:2]] - if isinstance(self._initializer, dict): + if isinstance(self._initializer, dict): # TODO remove when full torch self._initializer = initializers.deserialize(self._initializer) - var_x = self._initializer(new_shape, dtype) - var_x = ops.transpose(var_x, [2, 0, 1, 3]) - var_x = ops.image.resize(var_x, - size, - interpolation="nearest", - data_format="channels_last") - var_x = self._space_to_depth(T.cast("KerasTensor", var_x)) - var_x = ops.transpose(var_x, [1, 2, 0, 3]) + x: torch.Tensor = self._initializer(new_shape, dtype) - logger.debug("ICNR Output shape: %s", var_x.shape) - return T.cast("KerasTensor", var_x) - - def _space_to_depth(self, input_tensor: KerasTensor) -> KerasTensor: - """ Space to depth Keras implementation. - - Parameters - ---------- - input_tensor: :class:`keras.KerasTensor` - The tensor to be manipulated - - Returns - ------- - :class:`keras.KerasTensor` - The manipulated input tensor - """ - batch, height, width, depth = input_tensor.shape - assert height is not None and width is not None - new_height, new_width = height // 2, width // 2 - inter_shape = (batch, new_height, self._scale, new_width, self._scale, depth) - - var_x = ops.reshape(input_tensor, inter_shape) - var_x = ops.transpose(var_x, (0, 1, 3, 2, 4, 5)) - retval = ops.reshape(var_x, (batch, new_height, new_width, -1)) - - logger.debug("Space to depth - Input shape: %s, Output shape: %s", - input_tensor.shape, retval.shape) - return T.cast("KerasTensor", retval) + # TODO repeat needs to be replaced with repeat_interleave when pixel-shuffler is ported: + # x = x.repeat_interleave(self._scale ** 2, dim = -1) + x = x.repeat(*([1] * (x.dim() - 1)), self._scale ** 2) + logger.debug("ICNR Output shape: %s", x.shape) + return x def get_config(self) -> dict[str, T.Any]: - """ Return the ICNR Initializer configuration. + """Return the ICNR Initializer configuration. Returns ------- - dict[str, Any] - The configuration for ICNR Initialization + The configuration for ICNR Initialization """ config = {"scale": self._scale, "initializer": self._initializer} base_config = super().get_config() @@ -142,8 +89,7 @@ def get_config(self) -> dict[str, T.Any]: class ConvolutionAware(initializers.Initializer): - """ - Initializer that generates orthogonal convolution filters in the Fourier space. If this + """Initializer that generates orthogonal convolution filters in the Fourier space. If this initializer is passed a shape that is not 3D or 4D, orthogonal initialization will be used. Adapted, fixed and optimized from: @@ -151,26 +97,26 @@ class ConvolutionAware(initializers.Initializer): Parameters ---------- - eps_std: float, optional + eps_std The Standard deviation for the random normal noise used to break symmetry in the inverse Fourier transform. Default: 0.05 - seed: int | None, optional + seed Used to seed the random generator. Default: ``None`` - initialized: bool, optional + initialized This should always be set to ``False``. To avoid Keras re-calculating the values every time the model is loaded, this parameter is internally set on first time initialization. Default:``False`` Returns ------- - :class:`keras.Variable` - The modified kernel weights + The modified kernel weights References ---------- Armen Aghajanyan, https://arxiv.org/abs/1702.06295 """ - + # TODO this needs to be done after porting models to torch as it depends on underlying model + # structure def __init__(self, eps_std: float = 0.05, seed: int | None = None, @@ -187,17 +133,16 @@ def __init__(self, @classmethod def _symmetrize(cls, inputs: np.ndarray) -> np.ndarray: - """ Make the given tensor symmetrical. + """Make the given tensor symmetrical. Parameters ---------- - inputs: :class:`numpy.ndarray` + inputs The input tensor to make symmetrical Returns ------- - :class:`numpy.ndarray` - The symmetrical output + The symmetrical output """ var_a = np.transpose(inputs, axes=(0, 1, 3, 2)) diag = var_a.diagonal(axis1=2, axis2=3) @@ -207,21 +152,20 @@ def _symmetrize(cls, inputs: np.ndarray) -> np.ndarray: return retval def _create_basis(self, filters_size: int, filters: int, size: int, dtype: str) -> np.ndarray: - """ Create the basis for convolutional aware initialization + """Create the basis for convolutional aware initialization Parameters ---------- - filters_size: int + filters_size The size of the filter - filters: int + filters The number of filters - dtype: str + dtype The data type Returns ------- - :class:`numpy.ndarray` - The output array + The output array """ if size == 1: return np.random.normal(0.0, self._eps_std, (filters_size, filters, size)) @@ -236,19 +180,18 @@ def _create_basis(self, filters_size: int, filters: int, size: int, dtype: str) @classmethod def _scale_filters(cls, filters: np.ndarray, variance: float) -> np.ndarray: - """ Scale the given filters. + """Scale the given filters. Parameters ---------- - filters: :class:`numpy.ndarray` + filters The filters to scale - variance: float + variance The amount of variance Returns ------- - :class:`numpy.ndarray` - The scaled filters + The scaled filters """ c_var = np.var(filters) var_p = np.sqrt(variance / c_var) @@ -259,23 +202,22 @@ def _scale_filters(cls, filters: np.ndarray, variance: float) -> np.ndarray: def __call__(self, # pylint: disable=too-many-locals shape: list[int] | tuple[int, ...], - dtype: str | None = None) -> Variable: - """ Call function for the ICNR initializer. + dtype: str | None = None) -> torch.Tensor: + """Call function for the ICNR initializer. Parameters ---------- - shape: list[int] | tuple[int, ...] + shape The required shape for the output tensor - dtype: str + dtype The data type for the tensor Returns ------- - :class:`keras.Variable` - The modified kernel weights + The modified kernel weights """ if self._initialized: # Avoid re-calculating initializer when loading a saved model - return T.cast("Variable", self._he_uniform(shape, dtype=dtype)) + return T.cast(torch.Tensor, self._he_uniform(shape, dtype=dtype)) dtype = K.floatx() if dtype is None else dtype logger.info("Calculating Convolution Aware Initializer for shape: %s", shape) rank = len(shape) @@ -317,7 +259,7 @@ def __call__(self, # pylint: disable=too-many-locals else: self._initialized = True - return Variable(self._orthogonal(shape), dtype=dtype) + return T.cast(torch.Tensor, self._orthogonal(shape)) kernel_fourier_shape = correct_fft(np.zeros(kernel_shape)).shape @@ -328,19 +270,18 @@ def __call__(self, # pylint: disable=too-many-locals basis = basis.reshape((filters_size, stack_size,) + kernel_fourier_shape) randoms = np.random.normal(0, self._eps_std, basis.shape[:-2] + kernel_shape) init = correct_ifft(basis, kernel_shape) + randoms - init = self._scale_filters(init, variance) + init = self._scale_filters(init, variance).astype(dtype) self._initialized = True - retval = Variable(init.transpose(transpose_dimensions), dtype=dtype, name="conv_aware") + retval = torch.from_numpy(init.transpose(transpose_dimensions)) logger.debug("ConvAware output: %s", retval) return retval def get_config(self) -> dict[str, T.Any]: - """ Return the Convolutional Aware Initializer configuration. + """Return the Convolutional Aware Initializer configuration. Returns ------- - dict[str, Any] - The configuration for Convolutional Aware Initialization + The configuration for Convolutional Aware Initialization """ config = {"eps_std": self._eps_std, "seed": self._seed, diff --git a/lib/model/layers.py b/lib/model/layers.py index 51bc644eb7..8a1d563087 100644 --- a/lib/model/layers.py +++ b/lib/model/layers.py @@ -295,6 +295,7 @@ class PixelShuffler(Layer): # pylint:disable=too-many-ancestors,abstract-method ---------- https://gist.github.com/t-ae/6e1016cc188104d123676ccef3264981 """ + # TODO. When this is ported to nn.PixelShuffle: ICNR init must be updated as commented in code def __init__(self, size: int | tuple[int, int] = (2, 2), data_format: str | None = None, @@ -464,7 +465,7 @@ def compute_output_shape(self, input_shape: tuple[int, ...] # pylint:disable=ar def call(self, inputs: KerasTensor, *args, **kwargs # pylint:disable=arguments-differ ) -> KerasTensor: - """ Call the QuickGELU layerr + """ Call the QuickGELU layer Parameters ---------- diff --git a/lib/system/ml_libs.py b/lib/system/ml_libs.py index d0c956db30..c469c4cae4 100644 --- a/lib/system/ml_libs.py +++ b/lib/system/ml_libs.py @@ -1,6 +1,5 @@ #! /usr/env/bin/python -""" -Queries information about system installed Machine Learning Libraries. +"""Queries information about system installed Machine Learning Libraries. NOTE: Only packages from Python's Standard Library should be imported in this module """ from __future__ import annotations @@ -31,20 +30,20 @@ _TORCH_ROCM_REQUIREMENTS = {">=2.2.1,<2.4.0": ((6, 0), (6, 0))} -"""dict[str, tuple[tuple[int, int], tuple[int, int]]]: Minumum and maximum ROCm versions """ +"""Minimum and maximum ROCm versions""" def _check_dynamic_linker(lib: str) -> list[str]: - """ Locate the folders that contain a given library in ldconfig and $LD_LIBRARY_PATH + """Locate the folders that contain a given library in ldconfig and $LD_LIBRARY_PATH Parameters ---------- - lib: str The library to locate + lib + The library to locate Returns ------- - list[str] - All real existing folders from ldconfig or $LD_LIBRARY_PATH that contain the given lib + All real existing folders from ldconfig or $LD_LIBRARY_PATH that contain the given lib """ paths: set[str] = set() ldconfig = which("ldconfig") @@ -65,19 +64,18 @@ def _check_dynamic_linker(lib: str) -> list[str]: def _files_from_folder(folder: str, prefix: str) -> list[str]: - """ Obtain all filenames from the given folder that start with the given prefix + """Obtain all filenames from the given folder that start with the given prefix Parameters ---------- - folder : str + folder The folder to search for files in - prefix : str + prefix The filename prefix to search for Returns ------- - list[str] - All filenames that exist in the given folder with the given prefic + All filenames that exist in the given folder with the given prefix """ if not os.path.exists(folder): return [] @@ -85,11 +83,11 @@ def _files_from_folder(folder: str, prefix: str) -> list[str]: class _Alternatives: - """ Holds output from the update-alternatives command for the given package + """Holds output from the update-alternatives command for the given package Parameters ---------- - package : str + package The package to query update-alternatives for information """ def __init__(self, package: str) -> None: @@ -101,7 +99,7 @@ def __init__(self, package: str) -> None: @property def alternatives(self) -> list[str]: - """ list[str] : Full path to alternatives listed for the given package """ + """Full path to alternatives listed for the given package""" if self._output is None: self._query() if not self._output: @@ -113,7 +111,7 @@ def alternatives(self) -> list[str]: @property def default(self) -> str: - """ str : Full path to the default package """ + """Full path to the default package""" if self._output is None: self._query() if not self._output: @@ -125,8 +123,8 @@ def default(self) -> str: return retval def _query(self) -> None: - """ Query update-alternatives for the given package and place stripped output into - :attr:`_output` """ + """Query update-alternatives for the given package and place stripped output into + :attr:`_output`""" if not self._bin: self._output = [] return @@ -138,17 +136,17 @@ def _query(self) -> None: class _Cuda(ABC): - """ Find the location of system installed Cuda and cuDNN on Windows and Linux. """ + """Find the location of system installed Cuda and cuDNN on Windows and Linux.""" def __init__(self) -> None: self.versions: list[tuple[int, int]] = [] - """ list[tuple[int, int]] : All detected globally installed Cuda versions """ + """All detected globally installed Cuda versions""" self.version: tuple[int, int] = (0, 0) - """ tuple[int, int] : Default installed Cuda version. (0, 0) if not detected """ + """Default installed Cuda version. (0, 0) if not detected""" self.cudnn_versions: dict[tuple[int, int], tuple[int, int, int]] = {} - """ dict[tuple[int, int], tuple[int, int, int]] : Detected cuDNN version for each installed - Cuda. key (0, 0) denotes globally installed cudnn """ + """Detected cuDNN version for each installed Cuda. key (0, 0) denotes globally installed + cudnn""" self._paths: list[str] = [] - """ list[str] : list of path to Cuda install folders relating to :attr:`versions` """ + """list of path to Cuda install folders relating to :attr:`versions`""" self._version_file = "version.json" self._lib = "libcudart.so" @@ -162,24 +160,23 @@ def __init__(self) -> None: self._get_cudnn_versions() def __repr__(self) -> str: - """ Pretty representation of this class """ + """Pretty representation of this class""" attrs = ", ".join(f"{k}={repr(v)}" for k, v in self.__dict__.items() if not k.startswith("_")) return f"{self.__class__.__name__}({attrs})" @classmethod def _tuple_from_string(cls, version: str) -> tuple[int, int] | None: - """ Convert a Cuda version string to a version tuple + """Convert a Cuda version string to a version tuple Parameters ---------- - version : str + version The Cuda version string to convert Returns ------- - tuple[int, int] | None - The converted Cuda version string. ``None`` if not a valid version string + The converted Cuda version string. ``None`` if not a valid version string """ if version.startswith("."): version = version[1:] @@ -193,47 +190,42 @@ def _tuple_from_string(cls, version: str) -> tuple[int, int] | None: @abstractmethod def get_versions(self) -> dict[tuple[int, int], str]: - """ Overide to Attempt to detect all installed Cuda versions on Linux or Windows systems + """Override to Attempt to detect all installed Cuda versions on Linux or Windows systems Returns ------- - dict[tuple[int, int], str] - The Cuda versions to the folder path on the system + The Cuda versions to the folder path on the system """ @abstractmethod def get_version(self) -> tuple[int, int] | None: - """ Override to attempt to locate the default Cuda version on Linux or Windows + """Override to attempt to locate the default Cuda version on Linux or Windows Returns ------- - tuple[int, int] | None - The Default global Cuda version or ``None`` if not found + The Default global Cuda version or ``None`` if not found """ @abstractmethod def get_cudnn_versions(self) -> dict[tuple[int, int], tuple[int, int, int]]: - """ Override to attempt to locate any installed cuDNN versions + """Override to attempt to locate any installed cuDNN versions Returns ------- - dict[tuple[int, int], tuple[int, int, int]] - Detected cuDNN version for each installed Cuda. key (0, 0) denotes globally installed - cudnn + Detected cuDNN version for each installed Cuda. key (0, 0) denotes globally installed cudnn """ def version_from_version_file(self, folder: str) -> tuple[int, int] | None: - """ Attempt to get an installed Cuda version from its version.json file + """Attempt to get an installed Cuda version from its version.json file Parameters ---------- - folder : str + folder Full path to the folder to check for a version file Returns ------- - tuple[int, int] | None - The detected Cuda version or ``None`` if not detected + The detected Cuda version or ``None`` if not detected """ vers_file = os.path.join(folder, self._version_file) if not os.path.exists(vers_file): @@ -245,12 +237,11 @@ def version_from_version_file(self, folder: str) -> tuple[int, int] | None: return retval def _version_from_nvcc(self) -> tuple[int, int] | None: - """ Obtain the version from NVCC output if it is on PATH + """Obtain the version from NVCC output if it is on PATH Returns ------- - tuple[int, int] | None - The detected default Cuda version. ``None`` if not version detected + The detected default Cuda version. ``None`` if not version detected """ retval = None nvcc = which("nvcc") @@ -266,7 +257,7 @@ def _version_from_nvcc(self) -> tuple[int, int] | None: return retval def _get_versions(self) -> None: - """ Attempt to detect all installed Cuda versions and populate to :attr:`versions` """ + """Attempt to detect all installed Cuda versions and populate to :attr:`versions`""" versions = self.get_versions() if versions: logger.debug("Cuda Versions: %s", versions) @@ -276,7 +267,7 @@ def _get_versions(self) -> None: logger.debug("Could not locate any Cuda versions") def _get_version(self) -> None: - """ Attempt to detect the default Cuda version and populate to :attr:`version` """ + """Attempt to detect the default Cuda version and populate to :attr:`version`""" version: tuple[int, int] | None = None if len(self.versions) == 1: version = self.versions[0] @@ -290,7 +281,7 @@ def _get_version(self) -> None: logger.debug("Cuda version: %s", self.version if version else "not detected") def _get_cudnn_versions(self) -> None: - """ Attempt to locate any installed cuDNN versions and add to :attr`cudnn_versions` """ + """Attempt to locate any installed cuDNN versions and add to :attr`cudnn_versions`""" versions = self.get_cudnn_versions() if versions: logger.debug("cudnn versions: %s", versions) @@ -299,17 +290,16 @@ def _get_cudnn_versions(self) -> None: logger.debug("No cudnn versions found") def cudnn_version_from_header(self, folder: str) -> tuple[int, int, int] | None: - """ Attempt to detect the cuDNN version from the version header file + """Attempt to detect the cuDNN version from the version header file Parameters ---------- - folder : str + folder The folder to check for the cuDNN header file Returns ------- - tuple[int, int, int] | None - The cuDNN version found from the given folder or ``None`` if not detected + The cuDNN version found from the given folder or ``None`` if not detected """ path = os.path.join(folder, self._cudnn_header) if not os.path.exists(path): @@ -331,25 +321,24 @@ def cudnn_version_from_header(self, folder: str) -> tuple[int, int, int] | None: class CudaLinux(_Cuda): - """ Find the location of system installed Cuda and cuDNN on Linux. """ + """Find the location of system installed Cuda and cuDNN on Linux.""" def __init__(self) -> None: self._folder_prefix = "cuda-" super().__init__() def _version_from_lib(self, folder: str) -> tuple[int, int] | None: - """ Attempt to locate the version from the existence of libcudart.so within a Cuda + """Attempt to locate the version from the existence of libcudart.so within a Cuda targets/x86_64-linux/lib folder Parameters ---------- - folder : str + folder Full file path to the Cuda folder Returns ------- - tuple[int, int] | None - The Cuda version identified by the existence of the libcudart.so file. ``None`` if - not detected + The Cuda version identified by the existence of the libcudart.so file. ``None`` if not + detected """ lib_folder = os.path.join(folder, "targets", "x86_64-linux", "lib") lib_versions = [f.replace(self._lib, "") @@ -366,15 +355,14 @@ def _version_from_lib(self, folder: str) -> tuple[int, int] | None: return retval def _versions_from_usr(self) -> dict[tuple[int, int], str]: - """ Attempt to detect all installed Cuda versions from the /usr/local folder + """Attempt to detect all installed Cuda versions from the /usr/local folder Scan /usr/local for cuda-x.x folders containing either a version.json file or include/lib/libcudart.so.x. Returns ------- - dict[tuple[int, int], str] - A dictionary of detected Cuda versions to their install paths + A dictionary of detected Cuda versions to their install paths """ retval: dict[tuple[int, int], str] = {} usr = os.path.join(os.sep, "usr", "local") @@ -389,13 +377,11 @@ def _versions_from_usr(self) -> dict[tuple[int, int], str]: return retval def _versions_from_alternatives(self) -> dict[tuple[int, int], str]: - """ Attempt to detect all installed Cuda versions from update-alternatives + """Attempt to detect all installed Cuda versions from update-alternatives Returns ------- - list[tuple[int, int, int]] - A dictionary of detected Cuda versions to their install paths found in - update-alternatives + A dictionary of detected Cuda versions to their install paths found in update-alternatives """ retval: dict[tuple[int, int], str] = {} alts = self._alternatives.alternatives @@ -407,28 +393,26 @@ def _versions_from_alternatives(self) -> dict[tuple[int, int], str]: return retval def _parent_from_targets(self, folder: str) -> str: - """ Obtain the Cuda parent folder from a path obtained from child targets folder + """Obtain the Cuda parent folder from a path obtained from child targets folder Parameters ---------- - folder : str + folder Full path to a folder that has a 'targets' folder in its path Returns ------- - str - The potential parent Cuda folder, or an empty string if not detected + The potential parent Cuda folder, or an empty string if not detected """ split = folder.split(os.sep) return os.sep.join(split[:split.index("targets")]) if "targets" in split else "" def _versions_from_dynamic_linker(self) -> dict[tuple[int, int], str]: - """ Attempt to detect all installed Cuda versions from ldconfig + """Attempt to detect all installed Cuda versions from ldconfig Returns ------- - dict[tuple[int, int], str] - The Cuda version to the folder path found from ldconfig + The Cuda version to the folder path found from ldconfig """ retval: dict[tuple[int, int], str] = {} folders = _check_dynamic_linker(self._lib) @@ -444,12 +428,11 @@ def _versions_from_dynamic_linker(self) -> dict[tuple[int, int], str]: return retval def get_versions(self) -> dict[tuple[int, int], str]: - """ Attempt to detect all installed Cuda versions on Linux systems + """Attempt to detect all installed Cuda versions on Linux systems Returns ------- - dict[tuple[int, int], str] - The Cuda version to the folder path on Linux + The Cuda version to the folder path on Linux """ versions = (self._versions_from_usr() | self._versions_from_alternatives() | @@ -457,12 +440,11 @@ def get_versions(self) -> dict[tuple[int, int], str]: return {k: versions[k] for k in sorted(versions)} def _version_from_alternatives(self) -> tuple[int, int] | None: - """ Attempt to get the default Cuda version from update-alternatives + """Attempt to get the default Cuda version from update-alternatives Returns ------- - tuple[int, int] | None - The detected default Cuda version. ``None`` if not version detected + The detected default Cuda version. ``None`` if not version detected """ default = self._alternatives.default if not default: @@ -472,12 +454,11 @@ def _version_from_alternatives(self) -> tuple[int, int] | None: return retval def _version_from_link(self) -> tuple[int, int] | None: - """ Attempt to get the default Cuda version from the /usr/local/cuda file + """Attempt to get the default Cuda version from the /usr/local/cuda file Returns ------- - tuple[int, int] | None - The detected default Cuda version. ``None`` if not version detected + The detected default Cuda version. ``None`` if not version detected """ path = os.path.join(os.sep, "usr", "local", "cuda") if not os.path.exists(path): @@ -488,12 +469,11 @@ def _version_from_link(self) -> tuple[int, int] | None: return retval def _version_from_dynamic_linker(self) -> tuple[int, int] | None: - """ Attempt to get the default version from ldconfig or $LD_LIBRARY_PATH + """Attempt to get the default version from ldconfig or $LD_LIBRARY_PATH Returns ------- - tuple[int, int, int] | None - The detected default ROCm version. ``None`` if not version detected + The detected default ROCm version. ``None`` if not version detected """ paths = _check_dynamic_linker(self._lib) if len(paths) != 1: # Multiple or None @@ -504,27 +484,24 @@ def _version_from_dynamic_linker(self) -> tuple[int, int] | None: return retval def get_version(self) -> tuple[int, int] | None: - """ Attempt to locate the default Cuda version on Linux + """Attempt to locate the default Cuda version on Linux Checks, in order: update-alternatives, /usr/local/cuda, ldconfig, nvcc Returns ------- - tuple[int, int] | None - The Default global Cuda version or ``None`` if not found + The Default global Cuda version or ``None`` if not found """ return (self._version_from_alternatives() or self._version_from_link() or self._version_from_dynamic_linker()) def get_cudnn_versions(self) -> dict[tuple[int, int], tuple[int, int, int]]: - """ Attempt to locate any installed cuDNN versions on Linux + """Attempt to locate any installed cuDNN versions on Linux Returns ------- - dict[tuple[int, int], tuple[int, int, int]] - Detected cuDNN version for each installed Cuda. key (0, 0) denotes globally installed - cudnn + Detected cuDNN version for each installed Cuda. key (0, 0) denotes globally installed cudnn """ retval: dict[tuple[int, int], tuple[int, int, int]] = {} gbl = ["/usr/include", "/usr/local/include"] @@ -543,21 +520,20 @@ def get_cudnn_versions(self) -> dict[tuple[int, int], tuple[int, int, int]]: class CudaWindows(_Cuda): - """ Find the location of system installed Cuda and cuDNN on Windows. """ + """Find the location of system installed Cuda and cuDNN on Windows.""" @classmethod - def _enum_subkeys(cls, key: HKEYType) -> T.Generator[str, None, None]: - """ Iterate through a Registry key's sub-keys + def _enum_sub_keys(cls, key: HKEYType) -> T.Generator[str, None, None]: + """Iterate through a Registry key's sub-keys Parameters ---------- - key : :class:`winreg.HKEYType` + key The Registry key to iterate Yields ------ - str - A sub-key name from the given registry key + A sub-key name from the given registry key """ assert winreg is not None i = 0 @@ -569,12 +545,11 @@ def _enum_subkeys(cls, key: HKEYType) -> T.Generator[str, None, None]: i += 1 def get_versions(self) -> dict[tuple[int, int], str]: - """ Attempt to detect all installed Cuda versions on Windows systems from the registry + """Attempt to detect all installed Cuda versions on Windows systems from the registry Returns ------- - dict[tuple[int, int], str] - The Cuda version to the folder path on Windows + The Cuda version to the folder path on Windows """ retval: dict[tuple[int, int], str] = {} assert winreg is not None @@ -585,7 +560,7 @@ def get_versions(self) -> dict[tuple[int, int], str]: try: with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, # type:ignore[attr-defined] reg_key) as key: - for version in self._enum_subkeys(key): + for version in self._enum_sub_keys(key): vers_tuple = self._tuple_from_string(version[1:]) if vers_tuple is not None: retval[vers_tuple] = paths.get(version, "") @@ -594,12 +569,11 @@ def get_versions(self) -> dict[tuple[int, int], str]: return {k: retval[k] for k in sorted(retval)} def get_version(self) -> tuple[int, int] | None: - """ Attempt to get the default Cuda version from the Environment Variable + """Attempt to get the default Cuda version from the Environment Variable Returns ------- - tuple[int, int] | None - The Default global Cuda version or ``None`` if not found + The Default global Cuda version or ``None`` if not found """ path = os.environ.get("CUDA_PATH") if not path or path not in self._paths: @@ -610,12 +584,11 @@ def get_version(self) -> tuple[int, int] | None: return retval def _get_cudnn_paths(self) -> list[str]: # noqa[C901] - """ Attempt to locate the locations of cuDNN installs for Windows + """Attempt to locate the locations of cuDNN installs for Windows Returns ------- - list[str] - Full path to existing cuDNN installs under Windows + Full path to existing cuDNN installs under Windows """ assert winreg is not None paths: set[str] = set() @@ -628,24 +601,24 @@ def _get_cudnn_paths(self) -> list[str]: # noqa[C901] key = winreg.OpenKey(lookup, reg_key) # type:ignore[attr-defined] except FileNotFoundError: continue - for name in self._enum_subkeys(key): + for name in self._enum_sub_keys(key): if cudnn_key not in name.lower(): - logger.debug("Skipping subkey '%s'", name) + logger.debug("Skipping sub_keys '%s'", name) continue try: - subkey = winreg.OpenKey(key, name) # type:ignore[attr-defined] - logger.debug("Skipping subkey not found '%s'", name) + sub_keys = winreg.OpenKey(key, name) # type:ignore[attr-defined] + logger.debug("Skipping sub_keys not found '%s'", name) except FileNotFoundError: continue logger.debug("Parsing cudnn key '%s'", cudnn_key) try: - path, _ = winreg.QueryValueEx(subkey, # type:ignore[attr-defined] + path, _ = winreg.QueryValueEx(sub_keys, # type:ignore[attr-defined] "InstallLocation") except (FileNotFoundError, OSError): - logger.debug("Skipping missing InstallLocation for sub-key '%s'", subkey) + logger.debug("Skipping missing InstallLocation for sub-key '%s'", sub_keys) continue if not os.path.isdir(path): - logger.debug("Skipping non-existant path '%s'", path) + logger.debug("Skipping non-existent path '%s'", path) continue paths.add(path) retval = list(paths) @@ -653,13 +626,11 @@ def _get_cudnn_paths(self) -> list[str]: # noqa[C901] return retval def get_cudnn_versions(self) -> dict[tuple[int, int], tuple[int, int, int]]: - """ Attempt to locate any installed cuDNN versions on Windows + """Attempt to locate any installed cuDNN versions on Windows Returns ------- - dict[tuple[int, int], tuple[int, int, int]] - Detected cuDNN version for each installed Cuda. key (0, 0) denotes globally installed - cudnn + Detected cuDNN version for each installed Cuda. key (0, 0) denotes globally installed cudnn """ retval: dict[tuple[int, int], tuple[int, int, int]] = {} gbl = self._get_cudnn_paths() @@ -682,8 +653,7 @@ def get_cuda_finder() -> type[_Cuda]: Returns ------- - type[_Cuda] - The OS specific finder for system-wide Cuda + The OS specific finder for system-wide Cuda """ if platform.system().lower() == "windows": return CudaWindows @@ -694,14 +664,14 @@ def get_cuda_finder() -> type[_Cuda]: class ROCm(): - """ Find the location of system installed ROCm on Linux """ + """Find the location of system installed ROCm on Linux""" def __init__(self) -> None: self.version_min = min(v[0] for v in _TORCH_ROCM_REQUIREMENTS.values()) self.version_max = max(v[1] for v in _TORCH_ROCM_REQUIREMENTS.values()) self.versions: list[tuple[int, int, int]] = [] - """ list[tuple[int, int, int]] : All detected ROCm installed versions """ + """All detected ROCm installed versions""" self.version: tuple[int, int, int] = (0, 0, 0) - """ tuple[int, int, int] : Default ROCm installed version. (0, 0, 0) if not detected """ + """Default ROCm installed version. (0, 0, 0) if not detected""" self._folder_prefix = "rocm-" self._version_files = ["version-rocm", "version"] @@ -713,39 +683,38 @@ def __init__(self) -> None: self._rocm_check() def __repr__(self) -> str: - """ Pretty representation of this class """ + """Pretty representation of this class""" attrs = ", ".join(f"{k}={repr(v)}" for k, v in self.__dict__.items() if not k.startswith("_")) return f"{self.__class__.__name__}({attrs})" @property def valid_versions(self) -> list[tuple[int, int, int]]: - """ list[tuple[int, int, int]] """ + """Valid ROCm versions""" return [v for v in self.versions if self.version_min <= v[:2] <= self.version_max] @property def valid_installed(self) -> bool: - """ bool : ``True`` if a valid version of ROCm is installed """ + """``True`` if a valid version of ROCm is installed""" return any(self.valid_versions) @property def is_valid(self): - """ bool : ``True`` if the default ROCm version is valid """ + """``True`` if the default ROCm version is valid""" return self.version_min <= self.version[:2] <= self.version_max @classmethod def _tuple_from_string(cls, version: str) -> tuple[int, int, int] | None: - """ Convert a ROCm version string to a version tuple + """Convert a ROCm version string to a version tuple Parameters ---------- - version : str + version The ROCm version string to convert Returns ------- - tuple[int, int, int] | None - The converted ROCm version string. ``None`` if not a valid version string + The converted ROCm version string. ``None`` if not a valid version string """ split = version.split(".") if len(split) != 3: @@ -755,17 +724,16 @@ def _tuple_from_string(cls, version: str) -> tuple[int, int, int] | None: return (int(split[0]), int(split[1]), int(split[2])) def _version_from_string(self, string: str) -> tuple[int, int, int] | None: - """ Obtain the ROCm version from the end of a string + """Obtain the ROCm version from the end of a string Parameters ---------- - string : str + string The string to test for a valid ROCm version Returns ------- - tuple[int, int, int] | None - The ROCm version from the end of the string or ``None`` if not detected + The ROCm version from the end of the string or ``None`` if not detected """ re_vers = self._re_version.search(string) if re_vers is None: @@ -773,18 +741,17 @@ def _version_from_string(self, string: str) -> tuple[int, int, int] | None: return self._tuple_from_string(re_vers.group(1)) def _version_from_info(self, folder: str) -> tuple[int, int, int] | None: - """ Attempt to locate the version from a version file within a ROCm .info folder + """Attempt to locate the version from a version file within a ROCm .info folder Parameters ---------- - file_path : str + file_path Full path to the ROCm .info folder Returns ------- - tuple[int, int, int] | None - The ROCm version extracted from a version file within the .info folder. ``None`` if - not detected + The ROCm version extracted from a version file within the .info folder. ``None`` if not + detected """ info_loc = [os.path.join(folder, ".info", v) for v in self._version_files] for info_file in info_loc: @@ -802,19 +769,18 @@ def _version_from_info(self, folder: str) -> tuple[int, int, int] | None: return None def _version_from_lib(self, folder: str) -> tuple[int, int, int] | None: - """ Attempt to locate the version from the existence of librocm-core.so within a ROCm + """Attempt to locate the version from the existence of librocm-core.so within a ROCm lib folder Parameters ---------- - folder : str + folder Full file path to the ROCm folder Returns ------- - tuple[int, int, int] | None - The ROCm version identified by the existence of the librocm-core.so file. ``None`` if - not detected + The ROCm version identified by the existence of the librocm-core.so file. ``None`` if not + detected """ lib_folder = os.path.join(folder, "lib") lib_files = _files_from_folder(lib_folder, self._lib) @@ -830,14 +796,13 @@ def _version_from_lib(self, folder: str) -> tuple[int, int, int] | None: return retval def _versions_from_opt(self) -> list[tuple[int, int, int]]: - """ Attempt to detect all installed ROCm versions from the /opt folder + """Attempt to detect all installed ROCm versions from the /opt folder Scan /opt for rocm.x.x.x folders containing either .info or lib/librocm-core.so.x Returns ------- - list[tuple[int, int, int]] - Any ROCm versions found in the /opt folder + Any ROCm versions found in the /opt folder """ retval: list[tuple[int, int, int]] = [] opt = os.path.join(os.sep, "opt") @@ -851,12 +816,11 @@ def _versions_from_opt(self) -> list[tuple[int, int, int]]: return retval def _versions_from_alternatives(self) -> list[tuple[int, int, int]]: - """ Attempt to detect all installed ROCm versions from update-alternatives + """Attempt to detect all installed ROCm versions from update-alternatives Returns ------- - list[tuple[int, int, int]] - Any ROCm versions found in update-alternatives + Any ROCm versions found in update-alternatives """ alts = self._alternatives.alternatives if not alts: @@ -867,12 +831,11 @@ def _versions_from_alternatives(self) -> list[tuple[int, int, int]]: return retval def _versions_from_dynamic_linker(self) -> list[tuple[int, int, int]]: - """ Attempt to detect all installed ROCm versions from ldconfig + """Attempt to detect all installed ROCm versions from ldconfig Returns ------- - dict[tuple[int, int], str] - The ROCm versions found from ldconfig + The ROCm versions found from ldconfig """ retval: list[tuple[int, int, int]] = [] folders = _check_dynamic_linker(self._lib) @@ -886,7 +849,7 @@ def _versions_from_dynamic_linker(self) -> list[tuple[int, int, int]]: return retval def _get_versions(self) -> None: - """ Attempt to detect all installed ROCm versions and populate to :attr:`rocm_versions` """ + """Attempt to detect all installed ROCm versions and populate to :attr:`rocm_versions`""" versions = list(sorted(set(self._versions_from_opt()) | set(self._versions_from_alternatives()) | set(self._versions_from_dynamic_linker()))) @@ -897,12 +860,11 @@ def _get_versions(self) -> None: logger.debug("Could not locate any ROCm versions") def _version_from_hipconfig(self) -> tuple[int, int, int] | None: - """ Attempt to get the default version from hipconfig + """Attempt to get the default version from hipconfig Returns ------- - tuple[int, int, int] | None - The detected default ROCm version. ``None`` if not version detected + The detected default ROCm version. ``None`` if not version detected """ retval: tuple[int, int, int] | None = None exe = which("hipconfig") @@ -925,12 +887,11 @@ def _version_from_hipconfig(self) -> tuple[int, int, int] | None: return retval def _version_from_alternatives(self) -> tuple[int, int, int] | None: - """ Attempt to get the default version from update-alternatives + """Attempt to get the default version from update-alternatives Returns ------- - tuple[int, int, int] | None - The detected default ROCm version. ``None`` if not version detected + The detected default ROCm version. ``None`` if not version detected """ default = self._alternatives.default if not default: @@ -940,12 +901,11 @@ def _version_from_alternatives(self) -> tuple[int, int, int] | None: return retval def _version_from_link(self) -> tuple[int, int, int] | None: - """ Attempt to get the default version from the /opt/rocm file + """Attempt to get the default version from the /opt/rocm file Returns ------- - tuple[int, int, int] | None - The detected default ROCm version. ``None`` if not version detected + The detected default ROCm version. ``None`` if not version detected """ path = os.path.join(os.sep, "opt", "rocm") if not os.path.exists(path): @@ -956,12 +916,11 @@ def _version_from_link(self) -> tuple[int, int, int] | None: return retval def _version_from_dynamic_linker(self) -> tuple[int, int, int] | None: - """ Attempt to get the default version from ldconfig or $LD_LIBRARY_PATH + """Attempt to get the default version from ldconfig or $LD_LIBRARY_PATH Returns ------- - tuple[int, int, int] | None - The detected default ROCm version. ``None`` if not version detected + The detected default ROCm version. ``None`` if not version detected """ paths = _check_dynamic_linker("librocm-core.so.") if len(paths) != 1: # Multiple or None @@ -972,7 +931,7 @@ def _version_from_dynamic_linker(self) -> tuple[int, int, int] | None: return retval def _get_version(self) -> None: - """ Attempt to detect the default ROCm version """ + """Attempt to detect the default ROCm version""" version = (self._version_from_hipconfig() or self._version_from_alternatives() or self._version_from_link() or @@ -984,7 +943,7 @@ def _get_version(self) -> None: logger.debug("Could not locate default ROCm version") def _rocm_check(self) -> None: - """ Attempt to locate the installed ROCm versions and the default ROCm version """ + """Attempt to locate the installed ROCm versions and the default ROCm version""" self._get_versions() self._get_version() logger.debug("ROCm Versions: %s, Version: %s", self.versions, self.version) diff --git a/lib/system/system.py b/lib/system/system.py index a69ee09a29..9aa8b463c1 100644 --- a/lib/system/system.py +++ b/lib/system/system.py @@ -1,6 +1,5 @@ #! /usr/env/bin/python3 -""" -Holds information about the running system. Used in setup.py and lib.sysinfo +"""Holds information about the running system. Used in setup.py and lib.sysinfo NOTE: Only packages from Python's Standard Library should be imported in this module """ from __future__ import annotations @@ -23,28 +22,24 @@ VALID_PYTHON = ((3, 11), (3, 13)) -""" tuple[tuple[int, int], tuple[int, int]] : The minimum and maximum versions of Python that can -run Faceswap """ -VALID_TORCH = ((2, 3), (2, 11)) -""" tuple[tuple[int, int], tuple[int, int]] : The minimum and maximum versions of Torch that can -run Faceswap """ -VALID_KERAS = ((3, 13), (3, 14)) -""" tuple[tuple[int, int], tuple[int, int]] : The minimum and maximum versions of Keras that can -run Faceswap """ +"""The minimum and maximum versions of Python that can run Faceswap""" +VALID_TORCH = ((2, 3), (2, 12)) +"""The minimum and maximum versions of Torch that can run Faceswap""" +VALID_KERAS = ((3, 14), (3, 14)) +"""The minimum and maximum versions of Keras that can run Faceswap""" def _lines_from_command(command: list[str]) -> list[str]: - """ Output stdout lines from an executed command. + """Output stdout lines from an executed command. Parameters ---------- - command : list[str] + command The command to run Returns ------- - list[str] - The output lines from the given command + The output lines from the given command """ logger.debug("Running command %s", command) try: @@ -60,65 +55,64 @@ def _lines_from_command(command: list[str]) -> list[str]: class System: # pylint:disable=too-many-instance-attributes - """ Holds information about the currently running system and environment """ + """Holds information about the currently running system and environment""" def __init__(self) -> None: self.platform = platform.platform() - """ str : Human readable platform identifier """ + """Human readable platform identifier""" self.system: T.Literal["darwin", "linux", "windows"] = T.cast( T.Literal["darwin", "linux", "windows"], platform.system().lower()) - """ str : The system (OS type) that this code is running on. Always lowercase """ + """The system (OS type) that this code is running on. Always lowercase""" self.machine = platform.machine() - """ str : The machine type (eg: "x86_64") """ + """The machine type (eg: "x86_64")""" self.release = platform.release() - """ str : The OS Release that this code is running on """ + """The OS Release that this code is running on""" self.processor = platform.processor() - """ str : The processor in use, if detected """ + """The processor in use, if detected""" self.cpu_count = os.cpu_count() - """ int : The number of CPU cores on the system """ + """The number of CPU cores on the system""" self.python_implementation = platform.python_implementation() - """ str : The python implementation in use""" + """The python implementation in use""" self.python_version = platform.python_version() - """ str : The .. version of Python that is running """ + """The .. version of Python that is running""" self.python_architecture = platform.architecture()[0] - """ str : The Python architecture that is running (eg: 64bit/32bit)""" + """The Python architecture that is running (eg: 64bit/32bit)""" self.encoding = locale.getpreferredencoding() - """ str : The system encoding """ + """The system encoding""" self.is_conda = ("conda" in sys.version.lower() or os.path.exists(os.path.join(sys.prefix, 'conda-meta'))) - """ bool : ``True`` if running under Conda otherwise ``False`` """ + """``True`` if running under Conda otherwise ``False``""" self.is_admin = self._get_permissions() - """ bool : ``True`` if we are running with Admin privileges """ + """``True`` if we are running with Admin privileges""" self.is_virtual_env = self._check_virtual_env() - """ bool : ``True`` if Python is being run inside a virtual environment """ + """``True`` if Python is being run inside a virtual environment""" @property def is_linux(self) -> bool: - """ bool : `True` if running on a Linux system otherwise ``False``. """ + """``True`` if running on a Linux system otherwise ``False``.""" return self.system == "linux" @property def is_macos(self) -> bool: - """ bool : `True` if running on a macOS system otherwise ``False``. """ + """``True`` if running on a macOS system otherwise ``False``.""" return self.system == "darwin" @property def is_windows(self) -> bool: - """ bool : `True` if running on a Windows system otherwise ``False``. """ + """``True`` if running on a Windows system otherwise ``False``.""" return self.system == "windows" def __repr__(self) -> str: - """ Pretty print the system information for logging """ + """Pretty print the system information for logging""" attrs = ", ".join(f"{k}={repr(v)}" for k, v in self.__dict__.items() if not k.startswith("_")) return f"{self.__class__.__name__}({attrs})" def _get_permissions(self) -> bool: - """ Check whether user is admin + """Check whether user is admin Returns ------- - bool - ``True`` if we are running with Admin privileges + ``True`` if we are running with Admin privileges """ if self.is_windows: retval = ctypes.windll.shell32.IsUserAnAdmin() != 0 # type:ignore[attr-defined] @@ -127,12 +121,11 @@ def _get_permissions(self) -> bool: return retval def _check_virtual_env(self) -> bool: - """ Check whether we are in a virtual environment + """Check whether we are in a virtual environment Returns ------- - bool - ``True`` if Python is being run inside a virtual environment + ``True`` if Python is being run inside a virtual environment """ if not self.is_conda: retval = (hasattr(sys, "real_prefix") or @@ -143,18 +136,17 @@ def _check_virtual_env(self) -> bool: return retval def validate_python(self, max_version: tuple[int, int] | None = None) -> bool: - """ Check that the running Python version is valid + """Check that the running Python version is valid Parameters ---------- - max_version: tuple[int, int] | None, Optional + max_version The max version to validate Python against. ``None`` for the project Maximum. Default: ``None`` (project maximum) Returns ------- - bool - ``True`` if the running Python version is valid, otherwise logs an error and exits + ``True`` if the running Python version is valid, otherwise logs an error and exits """ max_python = VALID_PYTHON[1] if max_version is None else max_version retval = (VALID_PYTHON[0] <= sys.version_info[:2] <= max_python @@ -186,8 +178,8 @@ def validate_python(self, max_version: tuple[int, int] | None = None) -> bool: return retval def validate(self) -> None: - """ Perform validation that the running system can be used for faceswap. Log an error and - exit if it cannot """ + """Perform validation that the running system can be used for faceswap. Log an error and + exit if it cannot""" if not any((self.is_linux, self.is_macos, self.is_windows)): logger.error("Your system %s is not supported!", self.system.title()) sys.exit(1) @@ -199,7 +191,7 @@ def validate(self) -> None: class Packages(): - """ Holds information about installed python and conda packages. + """Holds information about installed python and conda packages. Note: Packaging library is lazy loaded as it may not be available during setup.py """ @@ -211,20 +203,19 @@ def __init__(self) -> None: @property def installed_python(self) -> dict[str, str]: - """ dict[str, str] : Installed Python package names to Python package versions """ + """Installed Python package names to Python package versions""" return self._installed_python @property def installed_python_pretty(self) -> str: - """ str: A pretty printed representation of installed Python packages """ + """A pretty printed representation of installed Python packages""" pkgs = self._installed_python align = max(len(x) for x in pkgs) + 1 return "\n".join(f"{k.ljust(align)} {v}" for k, v in pkgs.items()) @property def installed_conda(self) -> dict[str, tuple[str, str, str]]: - """ dict[str, tuple[str, str]] : Installed Conda package names to the version and - channel """ + """Installed Conda package names to the version and channel""" if not self._installed_conda: return {} @@ -239,13 +230,13 @@ def installed_conda(self) -> dict[str, tuple[str, str, str]]: @property def installed_conda_pretty(self) -> str: - """ str: A pretty printed representation of installed conda packages """ + """A pretty printed representation of installed conda packages""" if not self._installed_conda: return "Could not get Conda package list" return "\n".join(self._installed_conda) def __repr__(self) -> str: - """ Pretty print the installed packages for logging """ + """Pretty print the installed packages for logging""" props = ", ".join( f"{k}={repr(getattr(self, k))}" for k, v in self.__class__.__dict__.items() @@ -253,12 +244,11 @@ def __repr__(self) -> str: return f"{self.__class__.__name__}({props})" def _get_installed_python(self) -> dict[str, str]: - """ Parse the installed python modules + """Parse the installed python modules Returns ------- - dict[str, str] - Installed Python package names to Python package versions + Installed Python package names to Python package versions """ installed = _lines_from_command([sys.executable, "-m", "pip", "freeze", "--local"]) retval = {} @@ -271,13 +261,12 @@ def _get_installed_python(self) -> dict[str, str]: return retval def _get_installed_conda(self) -> None: - """ Collect the output from 'conda list' for the installed Conda packages and + """Collect the output from 'conda list' for the installed Conda packages and populate :attr:`_installed_conda` Returns ------- - list[str] - Each line of output from the 'conda list' command + Each line of output from the 'conda list' command """ if not self._conda_exe: logger.debug("Conda not found. Not collecting packages") diff --git a/requirements/_requirements_base.txt b/requirements/_requirements_base.txt index e8d372487d..940c0425e2 100644 --- a/requirements/_requirements_base.txt +++ b/requirements/_requirements_base.txt @@ -12,6 +12,6 @@ av>=17.0 ffmpeg-binaries>=1.1 ffmpy>=1.0.0 pywin32>=305 ; sys_platform == "win32" -torchvision>=0.18.0,<0.27.0 +torchvision>=0.18.0,<0.28.0 tensorboard>=2.20.0 -keras>=3.13.0,<3.14.0 +keras>=3.14.1,<3.15.0 diff --git a/requirements/requirements_apple-silicon.txt b/requirements/requirements_apple-silicon.txt index 48599420c0..127b9a498a 100644 --- a/requirements/requirements_apple-silicon.txt +++ b/requirements/requirements_apple-silicon.txt @@ -2,4 +2,4 @@ # These next 2 should have been installed, but some users complain of errors decorator cloudpickle -torch>=2.3.0,<2.10.0 +torch>=2.3.0,<2.13.0 diff --git a/requirements/requirements_cpu.txt b/requirements/requirements_cpu.txt index e3567a428b..be63be48b3 100644 --- a/requirements/requirements_cpu.txt +++ b/requirements/requirements_cpu.txt @@ -1,3 +1,3 @@ -r _requirements_base.txt --extra-index-url https://download.pytorch.org/whl/cpu -torch>=2.3.0,<2.10.0 +torch>=2.3.0,<2.13.0 diff --git a/requirements/requirements_nvidia_12.txt b/requirements/requirements_nvidia_12.txt index 153335e50a..dcc9a120e6 100644 --- a/requirements/requirements_nvidia_12.txt +++ b/requirements/requirements_nvidia_12.txt @@ -4,4 +4,4 @@ # Exclude badly numbered Python2 version of nvidia-ml-py nvidia-ml-py>=12.535,<300 --extra-index-url https://download.pytorch.org/whl/cu126 -torch>=2.7.0,<2.12.0 +torch>=2.7.0,<2.13.0 diff --git a/requirements/requirements_nvidia_13.txt b/requirements/requirements_nvidia_13.txt index 5f1891cb40..a192c07a6c 100644 --- a/requirements/requirements_nvidia_13.txt +++ b/requirements/requirements_nvidia_13.txt @@ -4,4 +4,4 @@ # Exclude badly numbered Python2 version of nvidia-ml-py nvidia-ml-py>=12.535,<300 --extra-index-url https://download.pytorch.org/whl/cu130 -torch>=2.9.0,<2.12.0 +torch>=2.9.0,<2.13.0 diff --git a/requirements/requirements_rocm.txt b/requirements/requirements_rocm.txt index 76f61581ea..65316d6b98 100644 --- a/requirements/requirements_rocm.txt +++ b/requirements/requirements_rocm.txt @@ -1,2 +1,2 @@ # Meta requirements file for latest ROCm version --r _requirements_rocm_64.txt +-r _requirements_rocm_72.txt diff --git a/requirements/requirements_rocm_71.txt b/requirements/requirements_rocm_71.txt new file mode 100644 index 0000000000..79937d2846 --- /dev/null +++ b/requirements/requirements_rocm_71.txt @@ -0,0 +1,3 @@ +-r _requirements_base.txt +--extra-index-url https://download.pytorch.org/whl/rocm7.1 +torch>=2.10.0,<2.11.0 diff --git a/requirements/requirements_rocm_72.txt b/requirements/requirements_rocm_72.txt new file mode 100644 index 0000000000..505e5b549e --- /dev/null +++ b/requirements/requirements_rocm_72.txt @@ -0,0 +1,3 @@ +-r _requirements_base.txt +--extra-index-url https://download.pytorch.org/whl/rocm7.2 +torch>=2.11.0,<2.13.0 diff --git a/setup.py b/setup.py index e1d332abba..513082431c 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -""" Install packages for faceswap.py """ +"""Install packages for faceswap.py""" # pylint:disable=too-many-lines from __future__ import annotations @@ -43,17 +43,17 @@ class _InstallState: # pylint:disable=too-few-public-methods - """ Marker to track if a step has failed installing """ + """Marker to track if a step has failed installing""" failed = False messages: list[str] = [] class Environment(): - """ The current install environment + """The current install environment Parameters ---------- - updater : bool, Optional + updater ``True`` if the script is being called by Faceswap's internal updater. ``False`` if full setup is running. Default: ``False`` """ @@ -68,7 +68,6 @@ def __init__(self, updater: bool = False) -> None: self.is_installer: bool = False # Flag setup is being run by installer to skip steps self.include_dev_tools: bool = False self.backend: T.Literal["nvidia", "apple_silicon", "cpu", "rocm"] | None = None - self.enable_docker: bool = False self.cuda_cudnn = ["", ""] self.requirement_version = "" self.rocm_version: tuple[int, ...] = (0, 0, 0) @@ -78,32 +77,32 @@ def __init__(self, updater: bool = False) -> None: @property def cuda_version(self) -> str: - """ str : The detected globally installed Cuda Version """ + """The detected globally installed Cuda Version""" return self.cuda_cudnn[0] @property def cudnn_version(self) -> str: - """ str : The detected globally installed cuDNN Version """ + """The detected globally installed cuDNN Version""" return self.cuda_cudnn[1] def set_backend(self, backend: T.Literal["nvidia", "apple_silicon", "cpu", "rocm"]) -> None: - """ Set the backend to install for + """Set the backend to install for Parameters ---------- - backend : Literal["nvidia", "apple_silicon", "cpu", "rocm"] + backend The backend to setup faceswap for """ logger.debug("Setting backend to '%s'", backend) self.backend = backend def set_requirements(self, requirements: str) -> None: - """ Validate that the requirements are compatible with the running Python version and + """Validate that the requirements are compatible with the running Python version and set the requirements file version to install use Parameters ---------- - backend : str + backend The requirements file version to use for install """ if requirements in PYTHON_VERSIONS: @@ -112,11 +111,11 @@ def set_requirements(self, requirements: str) -> None: self.requirement_version = requirements def _parse_backend_from_cli(self, arg: str) -> None: - """ Parse a command line argument and populate :attr:`backend` if valid + """Parse a command line argument and populate :attr:`backend` if valid Parameters ---------- - arg : str + arg The command line argument to parse """ arg = arg.lower() @@ -145,7 +144,7 @@ def _parse_backend_from_cli(self, arg: str) -> None: self.set_requirements(req_files[lookup.index(arg)]) def _process_arguments(self) -> None: - """ Process any cli arguments and dummy in cli arguments if calling from updater. """ + """Process any cli arguments and dummy in cli arguments if calling from updater.""" args = sys.argv[:] if self.updater: get_backend = T.cast("lib_utils", # type:ignore[attr-defined,valid-type] @@ -166,7 +165,7 @@ def _process_arguments(self) -> None: self._parse_backend_from_cli(arg[2:]) def _output_runtime_info(self) -> None: - """ Output run time info """ + """Output run time info""" logger.info("Setup in %s %s", self.system.system.title(), self.system.release) logger.info("Running as %s", "Root/Admin" if self.system.is_admin else "User") if self.system.is_conda: @@ -176,7 +175,7 @@ def _output_runtime_info(self) -> None: logger.info("Encoding: %s", self.system.encoding) def _check_pip(self) -> None: - """ Check installed pip version """ + """Check installed pip version""" for i in range(2): try: _pip = T.cast("pip", import_module("pip")) # type:ignore[valid-type] @@ -191,7 +190,7 @@ def _check_pip(self) -> None: logger.info("Pip version: %s", _pip.__version__) # type:ignore[attr-defined] def _configure_keras(self) -> None: - """ Set up the keras.json file to use Torch as the backend """ + """Set up the keras.json file to use Torch as the backend""" if "KERAS_HOME" in os.environ: keras_dir = os.environ["KERAS_HOME"] else: @@ -215,7 +214,7 @@ def _configure_keras(self) -> None: logger.info("Keras config written to: %s", conf_file) def set_config(self) -> None: - """ Set the backend in the faceswap config file """ + """Set the backend in the faceswap config file""" config = {"backend": self.backend} py_path = os.path.dirname(os.path.realpath(__file__)) config_file = os.path.join(py_path, "config", ".faceswap") @@ -226,12 +225,12 @@ def set_config(self) -> None: class RequiredPackages(): - """ Holds information about installed and required packages. + """Holds information about installed and required packages. Handles updating dependencies based on running platform/backend Parameters ---------- - environment : :class:`Environment` + environment Environment class holding information about the running system """ def __init__(self, environment: Environment) -> None: @@ -246,16 +245,16 @@ def __init__(self, environment: Environment) -> None: x.strip() for p in self._requirements.global_options[self._env.requirement_version] for x in p.split()] - """ list[str] : Any additional pip arguments that are required for installing from pip for - the given backend """ + """Any additional pip arguments that are required for installing from pip for the given + backend""" @property def packages_need_install(self) -> bool: - """bool : ``True`` if there are packages available that need to be installed """ + """``True`` if there are packages available that need to be installed""" return bool(self.conda or self.python) def _check_packaging(self) -> None: - """ Install packaging if it is not available """ + """Install packaging if it is not available""" if self._requirements.packaging_available: return cmd = [sys.executable, "-u", "-m", "pip", "install", "--no-cache-dir"] @@ -270,17 +269,16 @@ def _check_packaging(self) -> None: def _get_missing_python(self, requirements: list[Requirement] ) -> list[dict[T.Literal["name", "package"], str]]: - """ Check for missing Python dependencies + """Check for missing Python dependencies Parameters ---------- - requirements : list[:class:`packaging.requirements.Requirement]` + requirements The packages that are required to be installed Returns ------- - list[dict[Literal["name", "package"], str]] - List of missing Python packages to install + List of missing Python packages to install """ retval: list[dict[T.Literal["name", "package"], str]] = [] for req in requirements: @@ -302,12 +300,11 @@ def _get_missing_python(self, requirements: list[Requirement] return retval def _get_required_conda(self) -> list[dict[T.Literal["package", "channel"], str]]: - """ Add backend specific packages to Conda required packages + """Add backend specific packages to Conda required packages Returns ------- - list[tuple[Literal["package", "channel"], str]] - List of required Conda package names and the channel to install from + List of required Conda package names and the channel to install from """ retval: list[dict[T.Literal["package", "channel"], str]] = [] assert self._env.backend is not None @@ -328,12 +325,11 @@ def _get_required_conda(self) -> list[dict[T.Literal["package", "channel"], str] return retval def _get_missing_conda(self) -> dict[str, list[dict[T.Literal["name", "package"], str]]]: - """ Check for conda missing dependencies + """Check for conda missing dependencies Returns ------- - dict[str, list[dict[Literal["name", "package"], str]]] - The Conda packages to install grouped by channel + The Conda packages to install grouped by channel """ retval: dict[str, list[dict[T.Literal["name", "package"], str]]] = {} if not self._env.system.is_conda: @@ -375,11 +371,11 @@ def _get_missing_conda(self) -> dict[str, list[dict[T.Literal["name", "package"] class Checks(): # pylint:disable=too-few-public-methods - """ Pre-installation checks + """Pre-installation checks Parameters ---------- - environment : :class:`Environment` + environment Environment class holding information about the running system """ def __init__(self, environment: Environment) -> None: @@ -398,7 +394,7 @@ def __init__(self, environment: Environment) -> None: self._tips.pip() def _rocm_ask_enable(self) -> None: - """ Set backend to 'rocm' if OS is Linux and ROCm support required """ + """Set backend to 'rocm' if OS is Linux and ROCm support required""" if not self._env.system.is_linux: return logger.info("ROCm support:\r\nIf you are using an AMD GPU, then select 'yes'." @@ -423,22 +419,8 @@ def _rocm_ask_enable(self) -> None: logger.info("ROCm Version %s Selected", i) self._env.set_requirements(f"rocm_{i.replace('.', '')}") - def _docker_ask_enable(self) -> None: - """ Enable or disable Docker """ - i = input("Enable Docker? [y/N] ").strip() - if i not in ("", "Y", "y", "n", "N"): - logger.warning("Invalid selection '%s'", i) - self._docker_ask_enable() - return - if i in ("Y", "y"): - logger.info("Docker Enabled") - self._env.enable_docker = True - else: - logger.info("Docker Disabled") - self._env.enable_docker = False - def _cuda_ask_enable(self) -> None: - """ Enable or disable CUDA """ + """Enable or disable CUDA""" i = input("Enable CUDA? [Y/n] ").strip() if i not in ("", "Y", "y", "n", "N"): logger.warning("Invalid selection '%s'", i) @@ -459,39 +441,15 @@ def _cuda_ask_enable(self) -> None: logger.info("CUDA Version %s Selected", i) self._env.set_requirements(f"nvidia_{i}") - def _docker_confirm(self) -> None: - """ Warn if nvidia-docker on non-Linux system """ - logger.warning("Nvidia-Docker is only supported on Linux.\r\n" - "Only CPU is supported in Docker for your system") - self._docker_ask_enable() - if self._env.enable_docker: - logger.warning("CUDA Disabled") - self._env.set_backend("cpu") - - def _docker_tips(self) -> None: - """ Provide tips for Docker use """ - if self._env.backend != "nvidia": - self._tips.docker_no_cuda() - else: - self._tips.docker_cuda() - def _user_input(self) -> None: - """ Get user input for AMD/ROCm/Cuda/Docker """ + """Get user input for AMD/ROCm/Cuda""" if self._env.backend is None: self._rocm_ask_enable() if self._env.backend is None: - self._docker_ask_enable() self._cuda_ask_enable() - if not self._env.system.is_linux and (self._env.enable_docker - and self._env.backend == "nvidia"): - self._docker_confirm() - if self._env.enable_docker: - self._docker_tips() - self._env.set_config() - sys.exit(0) def _check_cuda(self) -> None: - """ Check for Cuda and cuDNN Locations. """ + """Check for Cuda and cuDNN Locations.""" if self._env.backend != "nvidia": logger.debug("Skipping Cuda checks as not enabled") return @@ -518,7 +476,7 @@ def _check_cuda(self) -> None: logger.debug("cuDNN version: %s", self._env.cudnn_version) def _check_rocm(self) -> None: - """ Check for ROCm version """ + """Check for ROCm version""" if self._env.backend != "rocm" or not self._env.system.is_linux: logger.debug("Skipping ROCm checks as not enabled") return @@ -551,11 +509,11 @@ def _check_rocm(self) -> None: class Status(): - """ Simple Status output for intercepting Conda/Pip installs and keeping the terminal clean + """Simple Status output for intercepting Conda/Pip installs and keeping the terminal clean Parameters ---------- - is_conda : bool + is_conda ``True`` if installing packages from Conda. ``False`` if installing from pip """ def __init__(self, is_conda: bool): @@ -571,15 +529,15 @@ def __init__(self, is_conda: bool): r"(?P^\S+)\s+\|\s+(?P\d+\.?\d*\s\w+).*\|\s+(?P\d+)%") def _clear_line(self) -> None: - """ Clear the last printed line from the console """ + """Clear the last printed line from the console""" print(" " * self._max_width, end="\r") def _print(self, line: str) -> None: - """ Clear the last line and print the new line to the console + """Clear the last line and print the new line to the console Parameters ---------- - line : str + line The line to print """ full_line = f"{self._prefix}{line}" @@ -592,17 +550,16 @@ def _print(self, line: str) -> None: print(output, end="\r") def _parse_size(self, size: str) -> float: - """ Parse the string representation of a package size and return as megabytes + """Parse the string representation of a package size and return as megabytes Parameters ---------- - size : str + size The string representation of a package size Returns ------- - float - The size in megabytes + The size in megabytes """ size, unit = size.strip().split(" ", maxsplit=1) if unit.lower() == "b": @@ -616,11 +573,11 @@ def _parse_size(self, size: str) -> float: return float(size) # Should never happen, but to prevent error def _print_conda(self, line: str) -> None: - """ Output progress for Conda installs + """Output progress for Conda installs Parameters ---------- - line : str + line The conda install line to parse """ progress = self._re_conda.match(line) @@ -639,11 +596,11 @@ def _print_conda(self, line: str) -> None: self._print(f"Downloading {count} packages ({total_size:.1f} MB) {prog:.1f}%") def _print_pip(self, line: str) -> None: - """ Output progress for Pip installs + """Output progress for Pip installs Parameters ---------- - line : str + line The pip install line to parse """ if (line.lower().startswith("installing collected packages:") and @@ -664,11 +621,11 @@ def _print_pip(self, line: str) -> None: self._print(f"{last_line} {done:.1f}%") def __call__(self, line: str) -> None: - """ Update the output status with the given line + """Update the output status with the given line Parameters ---------- - line : str + line A cleansed line from either Conda or Pip installers """ if self._is_conda: @@ -677,24 +634,24 @@ def __call__(self, line: str) -> None: self._print_pip(line.strip()) def close(self) -> None: - """ Reset all progress bars and re-enable the cursor """ + """Reset all progress bars and re-enable the cursor """ self._clear_line() class Installer(): - """ Uses the python Subprocess module to install packages. + """Uses the python Subprocess module to install packages. Parameters ---------- - environment : :class:`Environment` + environment Environment class holding information about the running system - packages : list[str] + packages The list of package names that are to be installed - command : list + command The command to run - is_conda : bool + is_conda ``True`` if conda install command is running. ``False`` if pip install command is running - is_gui : bool + is_gui ``True`` if the process is being called from the Faceswap GUI """ def __init__(self, # pylint:disable=too-many-positional-arguments @@ -717,12 +674,12 @@ def __init__(self, # pylint:disable=too-many-positional-arguments @classmethod def _output_information(cls, packages: list[str]): - """ INFO log the packages to be installed, splitting along multiple lines for long package + """INFO log the packages to be installed, splitting along multiple lines for long package lists (68 chars = 79 chars - (log-level spacing + indent)) Parameters ---------- - packages : list[str] + packages The list of package names that are to be installed """ output = "" @@ -742,33 +699,30 @@ def _clean_line(self, text: str) -> str: Parameters ---------- - text : str + text The text to clean Returns ------- - str - The cleansed text + The cleansed text """ clean = self._re_ansi_escape.sub("", text.rstrip()) return ''.join(c for c in clean if c in set(printable)) def _seen_line_log(self, text: str, is_error: bool = False) -> str: - """ Output gets spammed to the log file when conda is waiting/processing. Only log each + """Output gets spammed to the log file when conda is waiting/processing. Only log each unique line once. Parameters ---------- - text : str + text The text to log - is_error : bool, optional + is_error ``True`` if the line comes from an error. Default: ``False`` Returns ------- - str - The cleansed log line - + The cleansed log line """ clean = self._clean_line(text) if clean in self._seen_lines: @@ -779,12 +733,11 @@ def _seen_line_log(self, text: str, is_error: bool = False) -> str: return clean def __call__(self) -> int: - """ Install a package using the Subprocess module + """Install a package using the Subprocess module Returns ------- - int - The return code of the package install process + The return code of the package install process """ with Popen(self._command, bufsize=0, stdout=PIPE, stderr=PIPE) as proc: @@ -813,16 +766,16 @@ def __call__(self) -> int: class Install(): # pylint:disable=too-few-public-methods - """ Handles installation of Faceswap requirements + """Handles installation of Faceswap requirements Parameters ---------- - environment : :class:`Environment` + environment Environment class holding information about the running system - is_gui : bool, Optional + is_gui ``True`` if the caller is the Faceswap GUI. Used to prevent output of progress bars which get scrambled in the GUI - """ + """ def __init__(self, environment: Environment, is_gui: bool = False) -> None: self._env = environment self._is_gui = is_gui @@ -836,7 +789,7 @@ def __init__(self, environment: Environment, is_gui: bool = False) -> None: self._finalize() def _ask_continue(self) -> None: - """ Ask Continue with Install """ + """Ask Continue with Install""" if _InstallState.messages: for msg in _InstallState.messages: logger.warning(msg) @@ -853,13 +806,13 @@ def _ask_continue(self) -> None: def _from_pip(self, packages: list[dict[T.Literal["name", "package"], str]], extra_args: list[str] | None = None) -> None: - """ Install packages from pip + """Install packages from pip Parameters ---------- - packages : list[dict[T.Literal["name", "package"], str] + packages The formatted list of packages to be installed - extra_args : list[str] | None, optional + extra_args Any extra arguments to provide to pip. Default: ``None`` (no extra arguments) """ pip_exe = [sys.executable, @@ -882,19 +835,18 @@ def _from_pip(self, def _from_conda(self, packages: list[dict[T.Literal["name", "package"], str]], channel: str) -> None: - """ Install packages from conda + """Install packages from conda Parameters ---------- - packages : list[dict[T.Literal["name", "package"], str]] + packages The full formatted packages to be installed - channel : str + channel The Conda channel to install from. Returns ------- - bool - ``True`` if the package was successfully installed otherwise ``False`` + ``True`` if the package was successfully installed otherwise ``False`` """ conda = which("conda") assert conda is not None @@ -909,7 +861,7 @@ def _from_conda(self, _InstallState.failed = True def _install_packages(self) -> None: - """ Install the required packages """ + """Install the required packages""" if self._packages.conda: logger.info("Installing Conda packages...") for channel, packages in self._packages.conda.items(): @@ -920,7 +872,7 @@ def _install_packages(self) -> None: self._from_pip(packages, extra_args=self._packages.pip_arguments) def _finalize(self) -> None: - """ Output final information on completion """ + """Output final information on completion""" if self._env.updater: return if not _InstallState.failed: @@ -945,53 +897,10 @@ def _finalize(self) -> None: class Tips(): - """ Display installation Tips """ - @classmethod - def docker_no_cuda(cls) -> None: - """ Output Tips for Docker without Cuda """ - logger.info( - "1. Install Docker from: https://www.docker.com/get-started\n\n" - "2. Enter the Faceswap folder and build the Docker Image For Faceswap:\n" - " docker build -t faceswap-cpu -f Dockerfile.cpu .\n\n" - "3. Launch and enter the Faceswap container:\n" - " a. Headless:\n" - " docker run --rm -it -v ./:/srv faceswap-cpu\n\n" - " b. GUI:\n" - " xhost +local: && \\ \n" - " docker run --rm -it \\ \n" - " -v ./:/srv \\ \n" - " -v /tmp/.X11-unix:/tmp/.X11-unix \\ \n" - " -e DISPLAY=${DISPLAY} \\ \n" - " faceswap-cpu \n") - logger.info("That's all you need to do with docker. Have fun.") - - @classmethod - def docker_cuda(cls) -> None: - """ Output Tips for Docker with Cuda""" - logger.info( - "1. Install Docker from: https://www.docker.com/get-started\n\n" - "2. Install latest CUDA 11 and cuDNN 8 from: https://developer.nvidia.com/cuda-" - "downloads\n\n" - "3. Install the the Nvidia Container Toolkit from https://docs.nvidia.com/datacenter/" - "cloud-native/container-toolkit/latest/install-guide\n\n" - "4. Restart Docker Service\n\n" - "5. Enter the Faceswap folder and build the Docker Image For Faceswap:\n" - " docker build -t faceswap-gpu -f Dockerfile.gpu .\n\n" - "6. Launch and enter the Faceswap container:\n" - " a. Headless:\n" - " docker run --runtime=nvidia --rm -it -v ./:/srv faceswap-gpu\n\n" - " b. GUI:\n" - " xhost +local: && \\ \n" - " docker run --runtime=nvidia --rm -it \\ \n" - " -v ./:/srv \\ \n" - " -v /tmp/.X11-unix:/tmp/.X11-unix \\ \n" - " -e DISPLAY=${DISPLAY} \\ \n" - " faceswap-gpu \n") - logger.info("That's all you need to do with docker. Have fun.") - + """Display installation Tips""" @classmethod def macos(cls) -> None: - """ Output Tips for macOS""" + """Output Tips for macOS""" logger.info( "setup.py does not directly support macOS. The following tips should help:\n\n" "1. Install system dependencies:\n" @@ -1000,7 +909,7 @@ def macos(cls) -> None: @classmethod def pip(cls) -> None: - """ Pip Tips """ + """Pip Tips""" logger.info("1. Install PIP requirements\n" "You may want to execute `chcp 65001` in cmd line\n" "to fix Unicode issues on Windows when installing dependencies")