diff --git a/node_cli/cli/mirage_boot.py b/node_cli/cli/mirage_boot.py index b3b555e5..8f5bf44e 100644 --- a/node_cli/cli/mirage_boot.py +++ b/node_cli/cli/mirage_boot.py @@ -19,10 +19,11 @@ import click -from node_cli.core.node import get_node_signature, register_node as register, get_node_info -from node_cli.mirage.mirage_boot import init, migrate, update from node_cli.configs import DEFAULT_NODE_BASE_PORT -from node_cli.utils.helper import streamed_cmd, IP_TYPE, error_exit, abort_if_false +from node_cli.core.node import get_node_info, get_node_signature +from node_cli.core.node import register_node as register +from node_cli.mirage.mirage_boot import init, update +from node_cli.utils.helper import IP_TYPE, abort_if_false, error_exit, streamed_cmd @click.group() @@ -76,21 +77,6 @@ def signature_boot(validator_id): print(f'Signature: {res}') -@boot.command('migrate', help='Migrate mirage node from Mirage Boot Phase to Mirage Main Phase.') -@click.option( - '--yes', - is_flag=True, - callback=abort_if_false, - expose_value=False, - prompt='Are you sure you want to mirage node from Mirage Boot Phase to Mirage Main Phase?', -) -@click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) -@click.argument('env_file') -@streamed_cmd -def migrate_boot(env_file, pull_config_for_schain): - migrate(env_file, pull_config_for_schain) - - @boot.command('update', help='Update Mirage node from .env file') @click.option( '--yes', diff --git a/node_cli/cli/mirage_node.py b/node_cli/cli/mirage_node.py index af4d8455..4d9c846c 100644 --- a/node_cli/cli/mirage_node.py +++ b/node_cli/cli/mirage_node.py @@ -19,9 +19,10 @@ import click -from node_cli.core.node import get_node_signature, backup, get_node_info -from node_cli.mirage.mirage_node import restore_mirage, request_repair -from node_cli.utils.helper import error_exit, streamed_cmd, abort_if_false, URL_TYPE +from node_cli.core.node import backup, get_node_info, get_node_signature +from node_cli.mirage.mirage_node import cleanup as mirage_cleanup +from node_cli.mirage.mirage_node import migrate_from_boot, request_repair, restore_mirage +from node_cli.utils.helper import URL_TYPE, abort_if_false, error_exit, streamed_cmd from node_cli.utils.texts import safe_load_texts TEXTS = safe_load_texts() @@ -99,14 +100,21 @@ def restore_node(backup_path, env_file, config_only): restore_mirage(backup_path, env_file, config_only) -@node.command('repair', help='Toggle mirage chain repair mode') +@node.command('migrate', help='Switch from boot to regular Mirage node.') +@click.argument('env_filepath') @click.option( '--yes', is_flag=True, callback=abort_if_false, expose_value=False, - prompt=TEXTS['mirage']['node']['repair']['warning'], + prompt='Are you sure you want to migrate to regular Mirage node? The action cannot be undone', ) +@streamed_cmd +def migrate_node(env_filepath: str) -> None: + migrate_from_boot(env_filepath=env_filepath) + + +@node.command('repair', help='Toggle mirage chain repair mode') @click.option( '--snapshot-from', type=URL_TYPE, @@ -114,5 +122,25 @@ def restore_node(backup_path, env_file, config_only): hidden=True, help=TEXTS['mirage']['node']['repair']['snapshot_from'], ) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt=TEXTS['mirage']['node']['repair']['warning'], +) def repair(snapshot_from: str = '') -> None: request_repair(snapshot_from=snapshot_from) + + +@node.command('cleanup', help='Cleanup Mirage node.') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to cleanup Mirage node?', +) +@streamed_cmd +def cleanup_node(): + mirage_cleanup() diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index a7f3d17c..61d71af7 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -37,11 +37,11 @@ run_checks, ) from node_cli.configs import DEFAULT_NODE_BASE_PORT -from node_cli.configs.env import ALLOWED_ENV_TYPES +from node_cli.configs.user import ALLOWED_ENV_TYPES from node_cli.utils.decorators import check_inited from node_cli.utils.helper import abort_if_false, streamed_cmd, IP_TYPE from node_cli.utils.texts import safe_load_texts -from node_cli.utils.meta import get_meta_info +from node_cli.utils.meta import CliMetaManager from node_cli.utils.print_formatters import print_meta_info @@ -248,7 +248,7 @@ def configure_firewall(monitoring): @check_inited @click.option('--json', 'raw', is_flag=True, help=TEXTS['common']['json']) def version(raw: bool) -> None: - meta_info = get_meta_info(raw=raw) + meta_info = CliMetaManager().get_meta_info(raw=raw) if raw: print(meta_info) else: diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index be9f8bdd..adfd0e46 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -19,8 +19,8 @@ import os import sys -from node_cli.utils.global_config import read_g_config +from node_cli.utils.global_config import read_g_config GLOBAL_SKALE_DIR = os.getenv('GLOBAL_SKALE_DIR') or '/etc/skale' GLOBAL_SKALE_CONF_FILENAME = 'conf.json' @@ -146,7 +146,7 @@ def _get_env(): SKALE_NODE_REPO_URL = 'https://github.com/skalenetwork/skale-node.git' DOCKER_LVMPY_REPO_URL = 'https://github.com/skalenetwork/docker-lvmpy.git' -DOCKER_DEAMON_CONFIG_PATH = '/etc/docker/daemon.json' +DOCKER_DAEMON_CONFIG_PATH = '/etc/docker/daemon.json' DOCKER_DAEMON_HOSTS = ('fd://', 'unix:///var/run/skale/docker.sock') DOCKER_SERVICE_CONFIG_DIR = '/etc/systemd/system/docker.service.d' DOCKER_SERVICE_CONFIG_PATH = '/etc/systemd/system/docker.service.d/no-host.conf' diff --git a/node_cli/configs/alias_address_validation.py b/node_cli/configs/alias_address_validation.py index e0dcaa5e..ffdbb2ee 100644 --- a/node_cli/configs/alias_address_validation.py +++ b/node_cli/configs/alias_address_validation.py @@ -38,7 +38,7 @@ class ContractType(Enum): MANAGER = 'skale-manager' -def validate_env_alias_or_address( +def validate_alias_or_address( alias_or_address: str, contract_type: ContractType, endpoint: str ) -> None: if is_contract_address(alias_or_address): diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py deleted file mode 100644 index 9979beef..00000000 --- a/node_cli/configs/env.py +++ /dev/null @@ -1,146 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019-Present SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import os -from typing import Dict, List - -from dotenv import load_dotenv - -from node_cli.configs import SKALE_DIR, CONTAINER_CONFIG_PATH -from node_cli.configs.alias_address_validation import validate_env_alias_or_address, ContractType -from node_cli.utils.node_type import NodeType -from node_cli.utils.helper import error_exit - -SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') -CONFIGS_ENV_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, '.env') - -ALLOWED_ENV_TYPES = ['mainnet', 'testnet', 'qanet', 'devnet'] - -CORE_REQUIRED_PARAMS: Dict[str, str] = { - 'CONTAINER_CONFIGS_STREAM': '', - 'ENDPOINT': '', - 'MANAGER_CONTRACTS': '', - 'DISK_MOUNTPOINT': '', - 'SGX_SERVER_URL': '', - 'ENV_TYPE': '', -} - -REQUIRED_PARAMS_SKALE: Dict[str, str] = { - **CORE_REQUIRED_PARAMS, - 'IMA_CONTRACTS': '', - 'DOCKER_LVMPY_STREAM': '', - 'FILEBEAT_HOST': '', -} - -REQUIRED_PARAMS_MIRAGE_BOOT: Dict[str, str] = { - **CORE_REQUIRED_PARAMS, - 'IMA_CONTRACTS': '', - 'FILEBEAT_HOST': '', -} -REQUIRED_PARAMS_MIRAGE: Dict[str, str] = { - **CORE_REQUIRED_PARAMS, - 'FILEBEAT_HOST': '', -} - -REQUIRED_PARAMS_SYNC: Dict[str, str] = { - **CORE_REQUIRED_PARAMS, - 'SCHAIN_NAME': '', - 'IMA_CONTRACTS': '', - 'DOCKER_LVMPY_STREAM': '', -} - -OPTIONAL_PARAMS: Dict[str, str] = { - 'MONITORING_CONTAINERS': '', - 'TELEGRAF': '', - 'INFLUX_TOKEN': '', - 'INFLUX_URL': '', - 'TG_API_KEY': '', - 'TG_CHAT_ID': '', - 'CONTAINER_CONFIGS_DIR': '', - 'DISABLE_DRY_RUN': '', - 'DEFAULT_GAS_LIMIT': '', - 'DEFAULT_GAS_PRICE_WEI': '', - 'SKIP_DOCKER_CONFIG': '', - 'ENFORCE_BTRFS': '', - 'SKIP_DOCKER_CLEANUP': '', -} - - -def absent_required_params(params: Dict[str, str]) -> List[str]: - return [key for key in params if key not in OPTIONAL_PARAMS and not params[key]] - - -def get_validated_env_config( - node_type: NodeType, - env_filepath: str = SKALE_DIR_ENV_FILEPATH, - is_mirage_boot: bool = False, -) -> Dict[str, str]: - load_env_file(env_filepath) - params = build_env_params(node_type=node_type, is_mirage_boot=is_mirage_boot) - populate_env_params(params) - validate_env_params(params=params) - return params - - -def load_env_file(env_filepath: str) -> None: - if not load_dotenv(dotenv_path=env_filepath): - error_exit(f'Failed to load environment from {env_filepath}') - - -def build_env_params( - node_type: NodeType, - is_mirage_boot: bool = False, -) -> Dict[str, str]: - if node_type == NodeType.MIRAGE and is_mirage_boot: - params = REQUIRED_PARAMS_MIRAGE_BOOT.copy() - elif node_type == NodeType.MIRAGE: - params = REQUIRED_PARAMS_MIRAGE.copy() - elif node_type == NodeType.SYNC: - params = REQUIRED_PARAMS_SYNC.copy() - else: - params = REQUIRED_PARAMS_SKALE.copy() - - params.update(OPTIONAL_PARAMS) - return params - - -def populate_env_params(params: Dict[str, str]) -> None: - for key in params: - env_value = os.getenv(key) - if env_value is not None: - params[key] = str(env_value) - - -def validate_env_params( - params: Dict[str, str], -) -> None: - missing = absent_required_params(params) - if missing: - error_exit(f'Missing required parameters: {missing}') - validate_env_type(env_type=params['ENV_TYPE']) - endpoint = params['ENDPOINT'] - validate_env_alias_or_address(params['MANAGER_CONTRACTS'], ContractType.MANAGER, endpoint) - - if 'IMA_CONTRACTS' in params.keys(): - validate_env_alias_or_address(params['IMA_CONTRACTS'], ContractType.IMA, endpoint) - - -def validate_env_type(env_type: str) -> None: - if env_type not in ALLOWED_ENV_TYPES: - error_exit(f'Allowed ENV_TYPE values are {ALLOWED_ENV_TYPES}. Actual: "{env_type}"') diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py new file mode 100644 index 00000000..f1d07cc6 --- /dev/null +++ b/node_cli/configs/user.py @@ -0,0 +1,188 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2019-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import inspect +import os +from abc import ABC +from dataclasses import dataclass +from typing import Dict, NamedTuple + +from dotenv.main import DotEnv + +from node_cli.configs import CONTAINER_CONFIG_PATH, SKALE_DIR +from node_cli.configs.alias_address_validation import ContractType, validate_alias_or_address +from node_cli.utils.helper import error_exit +from node_cli.utils.node_type import NodeType + +SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') +CONFIGS_ENV_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, '.env') + +ALLOWED_ENV_TYPES = ['mainnet', 'testnet', 'qanet', 'devnet'] + + +class ValidationResult(NamedTuple): + result: bool + missing: set + extra: set + + +@dataclass(kw_only=True) +class BaseUserConfig(ABC): + container_configs_stream: str + env_type: str + filebeat_host: str + disk_mountpoint: str + + container_configs_dir: str = '' + skip_docker_config: str = '' + skip_docker_cleanup: str = '' + + def to_env(self) -> Dict[str, str]: + result = {} + for field_name, field_value in self.__dict__.items(): + upper_key = field_name.upper() + result[upper_key] = str(field_value) if field_value is not None else '' + return result + + @classmethod + def validate_params(cls, params: Dict) -> ValidationResult: + parameters = inspect.signature(cls.__init__).parameters + missing = [] + keys = params.keys() + expected_keys = { + name.upper() + for name, value in parameters.items() + if name != 'self' and value.default == inspect._empty + } + optional_keys = { + name.upper() + for name, value in parameters.items() + if name != 'self' and value.default != inspect._empty + } + missing = expected_keys - keys + extra = keys - expected_keys - optional_keys + return ValidationResult(missing == set() and extra == set(), missing, extra) + + +@dataclass +class MirageUserConfig(BaseUserConfig): + mirage_contracts: str + boot_endpoint: str + sgx_server_url: str + enforce_btrfs: str = '' + + +@dataclass +class MirageBootUserConfig(BaseUserConfig): + endpoint: str + manager_contracts: str + ima_contracts: str + sgx_server_url: str + enforce_btrfs: str = '' + + +@dataclass +class SkaleUserConfig(BaseUserConfig): + endpoint: str + manager_contracts: str + ima_contracts: str + docker_lvmpy_stream: str + sgx_server_url: str + monitoring_containers: str = '' + telegraf: str = '' + influx_token: str = '' + influx_url: str = '' + tg_api_key: str = '' + tg_chat_id: str = '' + disable_dry_run: str = '' + default_gas_limit: str = '' + default_gas_price_wei: str = '' + + +@dataclass +class SyncUserConfig(BaseUserConfig): + endpoint: str + manager_contracts: str + schain_name: str = '' + ima_contracts: str = '' + enforce_btrfs: str = '' + + +def get_validated_user_config( + node_type: NodeType, + env_filepath: str = SKALE_DIR_ENV_FILEPATH, + is_mirage_boot: bool = False, +) -> BaseUserConfig: + params = parse_env_file(env_filepath) + user_config_class = get_user_config_class(node_type, is_mirage_boot) + _, missing_params, extra_params = user_config_class.validate_params(params) + + if len(missing_params) > 0: + error_exit(f'Missing required parameters: {missing_params}') + + if len(extra_params) > 0: + error_exit(f'Extra parameters: {extra_params}') + + params = to_lower_keys(params) + user_config = user_config_class(**params) + validate_user_config(user_config) + + return user_config + + +def validate_user_config(user_config: BaseUserConfig) -> None: + validate_env_type(env_type=user_config.env_type) + + if not isinstance(user_config, MirageUserConfig): + validate_alias_or_address( + user_config.manager_contracts, ContractType.MANAGER, user_config.endpoint + ) + + if isinstance(user_config, (SkaleUserConfig, MirageBootUserConfig)): + validate_alias_or_address(user_config.ima_contracts, ContractType.IMA, user_config.endpoint) + + +def to_lower_keys(params: Dict[str, str]) -> Dict[str, str]: + return {key.lower(): value for key, value in params.items()} + + +def parse_env_file(env_filepath: str) -> Dict: + if not os.path.isfile(env_filepath): + error_exit(f'Failed to load environment from {env_filepath}') + return DotEnv(env_filepath).dict() + + +def get_user_config_class( + node_type: NodeType, + is_mirage_boot: bool = False, +) -> type[BaseUserConfig]: + if node_type == NodeType.MIRAGE and is_mirage_boot: + user_config_class = MirageBootUserConfig + elif node_type == NodeType.MIRAGE: + user_config_class = MirageUserConfig + elif node_type == NodeType.SYNC: + user_config_class = SyncUserConfig + else: + user_config_class = SkaleUserConfig + return user_config_class + + +def validate_env_type(env_type: str) -> None: + if env_type not in ALLOWED_ENV_TYPES: + error_exit(f'Allowed ENV_TYPE values are {ALLOWED_ENV_TYPES}. Actual: "{env_type}"') diff --git a/node_cli/core/docker_config.py b/node_cli/core/docker_config.py index d0f07051..c6102eec 100644 --- a/node_cli/core/docker_config.py +++ b/node_cli/core/docker_config.py @@ -1,5 +1,5 @@ -import grp import enum +import grp import json import logging import os @@ -8,9 +8,8 @@ import typing from typing import Optional, Tuple - from node_cli.configs import ( - DOCKER_DEAMON_CONFIG_PATH, + DOCKER_DAEMON_CONFIG_PATH, DOCKER_DAEMON_HOSTS, DOCKER_SERVICE_CONFIG_DIR, DOCKER_SERVICE_CONFIG_PATH, @@ -18,9 +17,8 @@ NODE_DOCKER_CONFIG_PATH, SKALE_RUN_DIR, ) -from node_cli.utils.helper import run_cmd from node_cli.utils.docker_utils import docker_client, get_containers - +from node_cli.utils.helper import run_cmd logger = logging.getLogger(__name__) @@ -109,7 +107,7 @@ def ensure_service_overriden_config( def ensure_docker_daemon_config( - daemon_config_path: Path = DOCKER_DEAMON_CONFIG_PATH, daemon_hosts: Path = DOCKER_DAEMON_HOSTS + daemon_config_path: Path = DOCKER_DAEMON_CONFIG_PATH, daemon_hosts: Path = DOCKER_DAEMON_HOSTS ) -> None: logger.info('Ensuring docker daemon config') config = {} @@ -184,3 +182,51 @@ def configure_docker() -> None: save_docker_group_id(group_id) logger.info('Docker configuration finished') + + +def remove_docker_service_override_config() -> None: + if os.path.isfile(DOCKER_SERVICE_CONFIG_PATH): + logger.info('Removing docker service override config') + os.remove(DOCKER_SERVICE_CONFIG_PATH) + + +def reset_docker_daemon_config() -> None: + if os.path.isfile(DOCKER_DAEMON_CONFIG_PATH): + logger.info('Resetting docker daemon config') + with open(DOCKER_DAEMON_CONFIG_PATH, 'r') as daemon_config: + config = json.load(daemon_config) + + config.pop('live-restore', None) + config.pop('hosts', None) + + if config: + with open(DOCKER_DAEMON_CONFIG_PATH, 'w') as daemon_config: + json.dump(config, daemon_config) + else: + os.remove(DOCKER_DAEMON_CONFIG_PATH) + + +def remove_node_docker_config() -> None: + if os.path.isfile(NODE_DOCKER_CONFIG_PATH): + logger.info('Removing node docker config') + os.remove(NODE_DOCKER_CONFIG_PATH) + + +def remove_skale_run_dir() -> None: + if os.path.isdir(SKALE_RUN_DIR): + os.rmdir(SKALE_RUN_DIR) + logger.info('Removed SKALE run directory') + + +def cleanup_docker_configuration() -> None: + """Cleanup all skale specific docker configuration files and directories""" + logger.info('Cleaning up docker configuration') + + remove_docker_service_override_config() + reset_docker_daemon_config() + remove_node_docker_config() + remove_skale_run_dir() + restart_docker_service() + wait_for_socket_initialization() + + logger.info('Docker configuration cleanup finished') diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 7c9b2df6..8c684455 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -50,7 +50,7 @@ NGINX_CONFIG_FILEPATH, ) from node_cli.configs.cli_logger import LOG_DATA_PATH -from node_cli.configs.env import SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH +from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH from node_cli.core.nftables import NFTablesManager from node_cli.utils.helper import safe_mkdir diff --git a/node_cli/core/node.py b/node_cli/core/node.py index f6b534cd..ca60ba5e 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -42,7 +42,7 @@ TM_INIT_TIMEOUT, ) from node_cli.cli import __version__ -from node_cli.configs.env import get_validated_env_config, SKALE_DIR_ENV_FILEPATH +from node_cli.configs.user import get_validated_user_config, SKALE_DIR_ENV_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH as CLI_LOG_DATA_PATH from node_cli.core.host import is_node_inited, save_env_params, get_flask_secret_key @@ -69,7 +69,7 @@ get_request, post_request, ) -from node_cli.utils.meta import get_meta_info +from node_cli.utils.meta import CliMetaManager from node_cli.utils.texts import safe_load_texts from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.decorators import check_not_inited, check_inited, check_user @@ -198,7 +198,7 @@ def init_sync( @check_user def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: logger.info('Node update started') - prev_version = get_meta_info().version + prev_version = CliMetaManager().get_meta_info().version if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': migrate_2_6() env = compose_node_env(env_filepath, node_type=NodeType.SYNC) @@ -233,7 +233,7 @@ def compose_node_env( is_mirage_boot: bool = False, ) -> dict[str, str]: if env_filepath is not None: - env_params = get_validated_env_config( + user_config = get_validated_user_config( node_type=node_type, env_filepath=env_filepath, is_mirage_boot=is_mirage_boot, @@ -241,7 +241,7 @@ def compose_node_env( if save: save_env_params(env_filepath) else: - env_params = get_validated_env_config( + user_config = get_validated_user_config( node_type=node_type, env_filepath=INIT_ENV_FILEPATH, is_mirage_boot=is_mirage_boot, @@ -257,7 +257,7 @@ def compose_node_env( 'SCHAINS_MNT_DIR': mnt_dir, 'FILESTORAGE_MAPPING': FILESTORAGE_MAPPING, 'SKALE_LIB_PATH': SKALE_STATE_DIR, - **env_params, + **user_config.to_env(), } if inited_node and not node_type == NodeType.SYNC: @@ -284,7 +284,7 @@ def update( error_msg = 'Cannot update safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) - prev_version = get_meta_info().version + prev_version = CliMetaManager().get_meta_info().version if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': migrate_2_6() logger.info('Node update started') @@ -502,8 +502,8 @@ def run_checks( return if disk is None: - env = get_validated_env_config(node_type=node_type) - disk = env['DISK_MOUNTPOINT'] + env_config = get_validated_user_config(node_type=node_type) + disk = env_config.disk_mountpoint failed_checks = run_host_checks(disk, node_type, network, container_config_path) if not failed_checks: print('Requirements checking successfully finished!') diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 669f8d0f..216aa657 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -24,7 +24,7 @@ import psutil -from node_cli.configs.env import get_validated_env_config +from node_cli.configs.user import get_validated_user_config from node_cli.utils.docker_utils import ensure_volume from node_cli.utils.schain_types import SchainTypes from node_cli.utils.helper import write_json, read_json, run_cmd, safe_load_yml @@ -102,12 +102,10 @@ def generate_resource_allocation_config( logger.debug(msg) print(msg) return - env_params = get_validated_env_config(node_type=node_type, env_filepath=env_file) - if env_params is None: - return + user_config = get_validated_user_config(node_type=node_type, env_filepath=env_file) logger.info('Generating resource allocation file ...') try: - update_resource_allocation(env_params['ENV_TYPE']) + update_resource_allocation(user_config.env_type) except Exception as e: logger.exception(e) print("Can't generate resource allocation file, check out CLI logs") diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index 783722fa..eb420fdb 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -34,7 +34,7 @@ SCHAIN_NODE_DATA_PATH, SCHAINS_MNT_DIR_SINGLE_CHAIN, ) -from node_cli.configs.env import get_validated_env_config +from node_cli.configs.user import get_validated_user_config from node_cli.utils.docker_utils import ensure_volume, is_volume_exists from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import ( @@ -58,6 +58,10 @@ BLUEPRINT_NAME = 'schains' +class NoDataDirForChainError(Exception): + """Raised when no data directory is found""" + + def get_schain_firewall_rules(schain: str) -> None: status, payload = get_request( blueprint=BLUEPRINT_NAME, method='firewall-rules', params={'schain_name': schain} @@ -212,8 +216,8 @@ def restore_schain_from_snapshot( schain_type: str = 'medium', ) -> None: if env_type is None: - env_config = get_validated_env_config(node_type=node_type) - env_type = env_config['ENV_TYPE'] + user_config = get_validated_user_config(node_type=node_type) + env_type = user_config.env_type ensure_schain_volume(schain, schain_type, env_type) block_number = get_block_number_from_path(snapshot_path) if block_number == -1: @@ -249,8 +253,18 @@ def ensure_schain_volume(schain: str, schain_type: str, env_type: str) -> None: logger.warning('Volume %s already exists', schain) -def cleanup_sync_datadir(schain_name: str, base_path: str = SCHAINS_MNT_DIR_SINGLE_CHAIN) -> None: - base_path = os.path.join(base_path, schain_name) +def cleanup_datadir_for_single_chain_node( + chain_name: str = '', base_path: str = SCHAINS_MNT_DIR_SINGLE_CHAIN +) -> None: + if not chain_name: + folders = [f for f in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, f))] + if not folders: + raise NoDataDirForChainError( + f'No data directory found in {base_path}. ' + 'Please check the path or specify a chain name.' + ) + chain_name = folders[0] + base_path = os.path.join(base_path, chain_name) regular_folders_pattern = f'{base_path}/[!snapshots]*' logger.info('Removing regular folders') for filepath in glob.glob(regular_folders_pattern): diff --git a/node_cli/core/static_config.py b/node_cli/core/static_config.py index 721251ce..4bc84f42 100644 --- a/node_cli/core/static_config.py +++ b/node_cli/core/static_config.py @@ -44,3 +44,9 @@ def get_static_params( with open(static_params_filepath) as requirements_file: ydata = yaml.load(requirements_file, Loader=yaml.Loader) return ydata['envs'][env_type] + + +def get_mirage_chain_name(env: dict) -> str: + node_type = NodeType.MIRAGE + params = get_static_params(node_type, env['ENV_TYPE']) + return params['info']['chain_name'] diff --git a/node_cli/migrations/mirage/from_boot.py b/node_cli/migrations/mirage/from_boot.py new file mode 100644 index 00000000..04578e68 --- /dev/null +++ b/node_cli/migrations/mirage/from_boot.py @@ -0,0 +1,59 @@ +import glob +import logging +import os +from pathlib import Path + +from node_cli.core.docker_config import restart_docker_service +from node_cli.utils.helper import run_cmd + +logger = logging.getLogger(__name__) + +NFT_CHAIN_BASE_PATH = '/etc/nft.conf.d/skale/chains' +NFT_COMMITTEE_SCOPE_CHAIN_NAME = 'mirage-committee' + + +class NoLegacyNFTChainConfigError(Exception): + pass + + +def rename_chain_file(old_filepath: str, new_filepath: str) -> None: + old_path = Path(old_filepath) + new_path = Path(new_filepath) + if not old_path.exists(): + raise NoLegacyNFTChainConfigError(f'File {old_filepath} does not exists') + old_path.rename(Path(new_path)) + + +def rename_chain_in_config(config_path: str, old_chain_name: str, new_chain_name: str) -> None: + content = '' + with open(config_path, 'r') as f: + content = f.read() + + updated_content = content.replace(old_chain_name, new_chain_name) + + with open(config_path, 'w') as f: + f.write(updated_content) + + +def migrate_nft_chain() -> None: + after_boot_chain_path = glob.glob(os.path.join(NFT_CHAIN_BASE_PATH, '*'))[0] + old_chain_name = Path(after_boot_chain_path).name.removesuffix('.conf') + new_chain_name = NFT_COMMITTEE_SCOPE_CHAIN_NAME + rename_chain_in_config(after_boot_chain_path, old_chain_name, new_chain_name) + after_migration_chain_path = os.path.join( + NFT_CHAIN_BASE_PATH, f'{NFT_COMMITTEE_SCOPE_CHAIN_NAME}.conf' + ) + rename_chain_file(after_boot_chain_path, after_migration_chain_path) + + +def reload_nft(): + run_cmd(['nft', '-f', '/etc/nftables.conf']) + + +def migrate_nftables_from_boot(): + logger.info('Starting nftables migration from boot') + migrate_nft_chain() + logger.info('Reloading nftables rules') + reload_nft() + logger.info('Restart docker service') + restart_docker_service() diff --git a/node_cli/mirage/mirage_boot.py b/node_cli/mirage/mirage_boot.py index 5d16affa..5d6586b9 100644 --- a/node_cli/mirage/mirage_boot.py +++ b/node_cli/mirage/mirage_boot.py @@ -23,7 +23,7 @@ from node_cli.configs import TM_INIT_TIMEOUT from node_cli.core.node import compose_node_env, is_base_containers_alive -from node_cli.operations import init_mirage_boot_op, migrate_mirage_boot_op, update_mirage_boot_op +from node_cli.operations import init_mirage_boot_op, update_mirage_boot_op from node_cli.utils.decorators import check_not_inited, check_inited, check_user from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import error_exit @@ -50,29 +50,6 @@ def init(env_filepath: str) -> None: logger.info('Init mirage procedure finished') -@check_inited -@check_user -def migrate(env_filepath: str, pull_config_for_schain: str) -> None: - logger.info('Mirage node migration started') - env = compose_node_env( - env_filepath, - inited_node=True, - sync_schains=False, - pull_config_for_schain=pull_config_for_schain, - node_type=NodeType.MIRAGE, - ) - migrate_ok = migrate_mirage_boot_op(env_filepath, env) - if migrate_ok: - logger.info('Waiting for containers initialization') - time.sleep(TM_INIT_TIMEOUT) - alive = is_base_containers_alive(node_type=NodeType.MIRAGE) - if not migrate_ok or not alive: - print_node_cmd_error() - return - else: - logger.info('Node migration from Mirage Boot to Mirage Main finished successfully!') - - @check_inited @check_user def update(env_filepath: str, pull_config_for_schain: str) -> None: diff --git a/node_cli/mirage/mirage_node.py b/node_cli/mirage/mirage_node.py index e63c0401..34e5aff8 100644 --- a/node_cli/mirage/mirage_node.py +++ b/node_cli/mirage/mirage_node.py @@ -21,20 +21,25 @@ import logging import time -from node_cli.configs import SKALE_DIR, RESTORE_SLEEP_TIMEOUT -from node_cli.configs.env import SKALE_DIR_ENV_FILEPATH -from node_cli.core.node import compose_node_env +from node_cli.configs import RESTORE_SLEEP_TIMEOUT, SKALE_DIR +from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH +from node_cli.core.docker_config import cleanup_docker_configuration from node_cli.core.host import save_env_params -from node_cli.core.static_config import get_static_params -from node_cli.mirage.record.chain_record import ChainRecord -from node_cli.operations import restore_mirage_op -from node_cli.utils.decorators import check_inited, check_not_inited +from node_cli.core.node import compose_node_env, is_base_containers_alive +from node_cli.mirage.record.chain_record import get_mirage_chain_record +from node_cli.operations import ( + MirageUpdateType, + cleanup_mirage_op, + restore_mirage_op, + update_mirage_op, +) +from node_cli.utils.decorators import check_inited, check_not_inited, check_user from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import error_exit from node_cli.utils.node_type import NodeType +from node_cli.utils.print_formatters import print_node_cmd_error from node_cli.utils.texts import safe_load_texts - logger = logging.getLogger(__name__) TEXTS = safe_load_texts() @@ -55,12 +60,38 @@ def restore_mirage(backup_path, env_filepath, config_only=False): @check_inited +@check_user +def migrate_from_boot( + env_filepath: str, +) -> None: + logger.info('Migrating from boot to mirage node...') + env = compose_node_env( + env_filepath, + inited_node=True, + sync_schains=False, + node_type=NodeType.MIRAGE, + ) + migrate_ok = update_mirage_op(env_filepath, env, update_type=MirageUpdateType.FROM_BOOT) + alive = is_base_containers_alive(node_type=NodeType.MIRAGE) + if not migrate_ok or not alive: + print_node_cmd_error() + return + else: + logger.info('Migration from boot to mirage completed successfully') + + def request_repair(snapshot_from: str = '') -> None: - node_type = NodeType.MIRAGE - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=node_type) - params = get_static_params(node_type, env['ENV_TYPE']) - record = ChainRecord(params['info']['chain_name']) + env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.MIRAGE) + record = get_mirage_chain_record(env) record.set_repair_ts(int(time.time())) record.set_snapshot_from(snapshot_from) - print(TEXTS['mirage']['node']['repair']['repair_requested']) + + +@check_inited +@check_user +def cleanup() -> None: + env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.MIRAGE) + cleanup_mirage_op(env) + logger.info('Mirage node was cleaned up, all containers and data removed') + cleanup_docker_configuration() diff --git a/node_cli/mirage/record/chain_record.py b/node_cli/mirage/record/chain_record.py index 34968be4..36e0dbeb 100644 --- a/node_cli/mirage/record/chain_record.py +++ b/node_cli/mirage/record/chain_record.py @@ -18,11 +18,11 @@ # along with this program. If not, see . - import logging from typing import cast from datetime import datetime +from node_cli.core.static_config import get_mirage_chain_name from node_cli.mirage.record.redis_record import FlatRedisRecord, FieldInfo logger = logging.getLogger(__name__) @@ -30,6 +30,7 @@ CHAIN_RECORD_FIELDS: dict[str, FieldInfo] = { 'name': FieldInfo('name', str, ''), + 'config_version': FieldInfo('config_version', str, '0.0.0'), 'repair_date': FieldInfo('repair_date', datetime, datetime.fromtimestamp(0)), 'repair_ts': FieldInfo('repair_ts', int, None), 'snapshot_from': FieldInfo('snapshot_from', str, None), @@ -40,6 +41,10 @@ class ChainRecord(FlatRedisRecord): def _record_fields(self) -> dict[str, FieldInfo]: return CHAIN_RECORD_FIELDS + @property + def config_version(self) -> str: + return cast(str, self._get_field('config_version')) + @property def repair_date(self) -> datetime: return cast(datetime, self._get_field('repair_date')) @@ -52,6 +57,9 @@ def snapshot_from(self) -> str | None: def repair_ts(self) -> int | None: return cast(int | None, self._get_field('repair_ts')) + def set_config_version(self, version: str) -> None: + self._set_field('config_version', version) + def set_repair_date(self, date: datetime) -> None: self._set_field('repair_date', date) @@ -60,3 +68,14 @@ def set_snapshot_from(self, value: str | None) -> None: def set_repair_ts(self, value: int | None) -> None: self._set_field('repair_ts', value) + + +def get_mirage_chain_record(env: dict) -> ChainRecord: + return ChainRecord(get_mirage_chain_name(env)) + + +def migrate_chain_record(env: dict) -> None: + version = env['CONTAINER_CONFIGS_STREAM'] + logger.info('Migrating mirage chain record, setting config version to %s', version) + record = get_mirage_chain_record(env) + record.set_config_version(version) diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index 159c0e16..44786972 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -22,13 +22,17 @@ init as init_op, init_sync as init_sync_op, init_mirage_boot as init_mirage_boot_op, - migrate_mirage_boot as migrate_mirage_boot_op, update_mirage_boot as update_mirage_boot_op, update_sync as update_sync_op, turn_off as turn_off_op, turn_on as turn_on_op, restore as restore_op, - restore_mirage as restore_mirage_op, cleanup_sync as cleanup_sync_op, configure_nftables, ) +from node_cli.operations.mirage import ( # noqa + update_mirage as update_mirage_op, + MirageUpdateType, + restore_mirage as restore_mirage_op, + cleanup as cleanup_mirage_op, +) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index ccc29a1b..7cf51910 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -17,20 +17,22 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import time - -import distro import functools import logging +import time from typing import Dict, Optional +import distro + +from node_cli.cli.info import TYPE, VERSION from node_cli.configs import ( CONTAINER_CONFIG_PATH, CONTAINER_CONFIG_TMP_PATH, - SKALE_DIR, GLOBAL_SKALE_DIR, + SKALE_DIR, ) -from node_cli.core.checks import CheckType, run_checks as run_host_checks +from node_cli.core.checks import CheckType +from node_cli.core.checks import run_checks as run_host_checks from node_cli.core.docker_config import configure_docker from node_cli.core.host import ( ensure_btrfs_kernel_module_autoloaded, @@ -40,19 +42,18 @@ from node_cli.core.nftables import configure_nftables from node_cli.core.nginx import generate_nginx_config from node_cli.core.node_options import NodeOptions -from node_cli.core.resources import update_resource_allocation, init_shared_space_volume +from node_cli.core.resources import init_shared_space_volume, update_resource_allocation from node_cli.core.schains import ( + cleanup_datadir_for_single_chain_node, update_node_cli_schain_status, - cleanup_sync_datadir, ) -from node_cli.cli.info import VERSION, TYPE from node_cli.operations.common import configure_filebeat, configure_flask, unpack_backup_archive -from node_cli.operations.docker_lvmpy import lvmpy_install -from node_cli.operations.skale_node import ( +from node_cli.operations.config_repo import ( download_skale_node, sync_skale_node, update_images, ) +from node_cli.operations.docker_lvmpy import lvmpy_install from node_cli.operations.volume import ( cleanup_volume_artifacts, ensure_filestorage_mapping, @@ -64,12 +65,11 @@ docker_cleanup, remove_dynamic_containers, ) -from node_cli.utils.helper import str_to_bool, rm_dir -from node_cli.utils.meta import get_meta_info, update_meta +from node_cli.utils.helper import rm_dir, str_to_bool +from node_cli.utils.meta import CliMetaManager, MirageCliMetaManager from node_cli.utils.node_type import NodeType from node_cli.utils.print_formatters import print_failed_requirements_checks - logger = logging.getLogger(__name__) @@ -127,7 +127,8 @@ def update(env_filepath: str, env: Dict, node_type: NodeType) -> bool: prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) init_shared_space_volume(env['ENV_TYPE']) - current_stream = get_meta_info().config_stream + meta_manager = CliMetaManager() + current_stream = meta_manager.get_meta_info().config_stream skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: logger.info( @@ -137,7 +138,7 @@ def update(env_filepath: str, env: Dict, node_type: NodeType) -> bool: ) docker_cleanup() - update_meta( + meta_manager.update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], env['DOCKER_LVMPY_STREAM'], @@ -149,46 +150,6 @@ def update(env_filepath: str, env: Dict, node_type: NodeType) -> bool: return True -@checked_host -def migrate_mirage_boot(env_filepath: str, env: Dict) -> bool: - compose_rm(node_type=NodeType.MIRAGE, env=env) - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) - - sync_skale_node() - ensure_btrfs_kernel_module_autoloaded() - - if env.get('SKIP_DOCKER_CONFIG') != 'True': - configure_docker() - - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) - - generate_nginx_config() - - prepare_host(env_filepath, env['ENV_TYPE']) - - current_stream = get_meta_info().config_stream - skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: - logger.info( - 'Stream version was changed from %s to %s', - current_stream, - env['CONTAINER_CONFIGS_STREAM'], - ) - docker_cleanup() - - update_meta( - VERSION, - env['CONTAINER_CONFIGS_STREAM'], - None, - distro.id(), - distro.version(), - ) - update_images(env=env, node_type=NodeType.MIRAGE) - compose_up(env=env, node_type=NodeType.MIRAGE) - return True - - @checked_host def update_mirage_boot(env_filepath: str, env: Dict) -> bool: compose_rm(node_type=NodeType.MIRAGE, env=env) @@ -209,7 +170,8 @@ def update_mirage_boot(env_filepath: str, env: Dict) -> bool: prepare_host(env_filepath, env['ENV_TYPE']) - current_stream = get_meta_info().config_stream + meta_manager = MirageCliMetaManager() + current_stream = meta_manager.get_meta_info().config_stream skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: logger.info( @@ -219,10 +181,9 @@ def update_mirage_boot(env_filepath: str, env: Dict) -> bool: ) docker_cleanup() - update_meta( + meta_manager.update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - None, distro.id(), distro.version(), ) @@ -252,7 +213,8 @@ def init(env_filepath: str, env: dict, node_type: NodeType) -> None: lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) - update_meta( + meta_manager = CliMetaManager() + meta_manager.update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], env['DOCKER_LVMPY_STREAM'], @@ -285,10 +247,10 @@ def init_mirage_boot(env_filepath: str, env: dict) -> None: generate_nginx_config() prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') - update_meta( + meta_manager = MirageCliMetaManager() + meta_manager.update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - None, distro.id(), distro.version(), ) @@ -331,10 +293,11 @@ def init_sync( generate_nginx_config() prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') - update_meta( + meta_manager = CliMetaManager() + meta_manager.update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'], + None, distro.id(), distro.version(), ) @@ -370,7 +333,8 @@ def update_sync(env_filepath: str, env: Dict) -> bool: prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) - update_meta( + meta_manager = CliMetaManager() + meta_manager.update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], env['DOCKER_LVMPY_STREAM'], @@ -392,7 +356,8 @@ def turn_off(env: dict, node_type: NodeType) -> None: def turn_on(env: dict, node_type: NodeType) -> None: logger.info('Turning on the node...') - update_meta( + meta_manager = CliMetaManager() + meta_manager.update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], env['DOCKER_LVMPY_STREAM'], @@ -434,15 +399,14 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) - update_meta( + meta_manager = CliMetaManager() + meta_manager.update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], env['DOCKER_LVMPY_STREAM'], distro.id(), distro.version(), ) - update_resource_allocation(env_type=env['ENV_TYPE']) - if not config_only: compose_up(env=env, node_type=node_type) @@ -459,55 +423,8 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): return True -def restore_mirage(env, backup_path, config_only=False): - unpack_backup_archive(backup_path) - failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], - TYPE, - env['ENV_TYPE'], - CONTAINER_CONFIG_PATH, - check_type=CheckType.PREINSTALL, - ) - if failed_checks: - print_failed_requirements_checks(failed_checks) - return False - - ensure_btrfs_kernel_module_autoloaded() - - if env.get('SKIP_DOCKER_CONFIG') != 'True': - configure_docker() - - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) - - link_env_file() - - update_meta( - VERSION, - env['CONTAINER_CONFIGS_STREAM'], - None, - distro.id(), - distro.version(), - ) - - if not config_only: - compose_up(env=env, node_type=NodeType.MIRAGE) - - failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], - TYPE, - env['ENV_TYPE'], - CONTAINER_CONFIG_PATH, - check_type=CheckType.POSTINSTALL, - ) - if failed_checks: - print_failed_requirements_checks(failed_checks) - return False - return True - - def cleanup_sync(env, schain_name: str) -> None: turn_off(env, node_type=NodeType.SYNC) - cleanup_sync_datadir(schain_name=schain_name) + cleanup_datadir_for_single_chain_node(schain_name=schain_name) rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 7e484cdf..7c876fa8 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -17,19 +17,18 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import logging import os +import secrets +import shutil import stat import tarfile -import logging -import shutil -import secrets - from shutil import copyfile from node_cli.configs import ( - G_CONF_HOME, FILEBEAT_CONFIG_PATH, FLASK_SECRET_KEY_FILE, + G_CONF_HOME, SRC_FILEBEAT_CONFIG_PATH, ) diff --git a/node_cli/operations/skale_node.py b/node_cli/operations/config_repo.py similarity index 100% rename from node_cli/operations/skale_node.py rename to node_cli/operations/config_repo.py diff --git a/node_cli/operations/mirage.py b/node_cli/operations/mirage.py new file mode 100644 index 00000000..f45b5a6d --- /dev/null +++ b/node_cli/operations/mirage.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2021-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import time +from enum import Enum + +import distro + +from node_cli.cli.info import TYPE, VERSION +from node_cli.configs import ( + CONTAINER_CONFIG_PATH, + GLOBAL_SKALE_DIR, + SKALE_DIR, +) +from node_cli.core.checks import CheckType +from node_cli.core.checks import run_checks as run_host_checks +from node_cli.core.docker_config import cleanup_docker_configuration, configure_docker +from node_cli.core.host import ensure_btrfs_kernel_module_autoloaded, link_env_file, prepare_host +from node_cli.core.nftables import configure_nftables +from node_cli.core.nginx import generate_nginx_config +from node_cli.core.schains import cleanup_datadir_for_single_chain_node +from node_cli.migrations.mirage.from_boot import migrate_nftables_from_boot +from node_cli.mirage.record.chain_record import migrate_chain_record +from node_cli.operations.base import checked_host, turn_off +from node_cli.operations.common import unpack_backup_archive +from node_cli.operations.config_repo import ( + sync_skale_node, + update_images, +) +from node_cli.utils.docker_utils import ( + REDIS_SERVICE_DICT, + REDIS_START_TIMEOUT, + NodeType, + compose_rm, + compose_up, + docker_cleanup, + remove_dynamic_containers, + wait_for_container, +) +from node_cli.utils.helper import rm_dir, str_to_bool +from node_cli.utils.meta import MirageCliMetaManager +from node_cli.utils.print_formatters import print_failed_requirements_checks + +logger = logging.getLogger(__name__) + + +class MirageUpdateType(Enum): + REGULAR = 'regular' + INFRA_ONLY = 'infra_only' + FROM_BOOT = 'from_boot' + + +@checked_host +def update_mirage(env_filepath: str, env: dict, update_type: MirageUpdateType) -> bool: + compose_rm(node_type=NodeType.MIRAGE, env=env) + if update_type not in (MirageUpdateType.INFRA_ONLY, MirageUpdateType.FROM_BOOT): + remove_dynamic_containers() + + sync_skale_node() + ensure_btrfs_kernel_module_autoloaded() + + if env.get('SKIP_DOCKER_CONFIG') != 'True': + configure_docker() + + configure_nftables() + generate_nginx_config() + + prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) + meta_manager = MirageCliMetaManager() + current_stream = meta_manager.get_meta_info().config_stream + skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' + if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: + logger.info( + 'Stream version was changed from %s to %s', + current_stream, + env['CONTAINER_CONFIGS_STREAM'], + ) + docker_cleanup() + + meta_manager.update_meta( + VERSION, + env['CONTAINER_CONFIGS_STREAM'], + distro.id(), + distro.version(), + ) + + if update_type == MirageUpdateType.FROM_BOOT: + migrate_nftables_from_boot() + + update_images(env=env, node_type=NodeType.MIRAGE) + + compose_up(env=env, node_type=NodeType.MIRAGE, services=list(REDIS_SERVICE_DICT)) + wait_for_container(REDIS_SERVICE_DICT['redis']) + time.sleep(REDIS_START_TIMEOUT) + if update_type == MirageUpdateType.FROM_BOOT: + migrate_chain_record(env) + + compose_up(env=env, node_type=NodeType.MIRAGE) + return True + + +def restore_mirage(env, backup_path, config_only=False): + unpack_backup_archive(backup_path) + failed_checks = run_host_checks( + env['DISK_MOUNTPOINT'], + TYPE, + env['ENV_TYPE'], + CONTAINER_CONFIG_PATH, + check_type=CheckType.PREINSTALL, + ) + if failed_checks: + print_failed_requirements_checks(failed_checks) + return False + + ensure_btrfs_kernel_module_autoloaded() + + if env.get('SKIP_DOCKER_CONFIG') != 'True': + configure_docker() + + enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) + configure_nftables(enable_monitoring=enable_monitoring) + + link_env_file() + + meta_manager = MirageCliMetaManager() + meta_manager.update_meta( + VERSION, + env['CONTAINER_CONFIGS_STREAM'], + distro.id(), + distro.version(), + ) + + if not config_only: + compose_up(env=env, node_type=NodeType.MIRAGE) + + failed_checks = run_host_checks( + env['DISK_MOUNTPOINT'], + TYPE, + env['ENV_TYPE'], + CONTAINER_CONFIG_PATH, + check_type=CheckType.POSTINSTALL, + ) + if failed_checks: + print_failed_requirements_checks(failed_checks) + return False + return True + + +def cleanup(env) -> None: + turn_off(env, node_type=NodeType.MIRAGE) + cleanup_datadir_for_single_chain_node() + rm_dir(GLOBAL_SKALE_DIR) + rm_dir(SKALE_DIR) + cleanup_docker_configuration() diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index d6438253..11fe3992 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -19,33 +19,36 @@ import io import itertools -import os import logging +import os +import time from typing import Optional import docker from docker.client import DockerClient +from docker.errors import NotFound from docker.models.containers import Container -from node_cli.utils.helper import run_cmd, str_to_bool from node_cli.configs import ( COMPOSE_PATH, - SYNC_COMPOSE_PATH, MIRAGE_COMPOSE_PATH, + NGINX_CONTAINER_NAME, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTIFICATES_DIR_NAME, - NGINX_CONTAINER_NAME, + SYNC_COMPOSE_PATH, ) +from node_cli.utils.helper import run_cmd, str_to_bool from node_cli.utils.node_type import NodeType - logger = logging.getLogger(__name__) SCHAIN_REMOVE_TIMEOUT = 300 IMA_REMOVE_TIMEOUT = 20 TELEGRAF_REMOVE_TIMEOUT = 20 +REDIS_START_TIMEOUT = 10 + +REDIS_SERVICE_DICT = {'redis': 'skale_redis'} -# Services have format : CORE_COMMON_COMPOSE_SERVICES = { 'transaction-manager': 'skale_transaction-manager', 'redis': 'skale_redis', @@ -72,6 +75,7 @@ BASE_MIRAGE_BOOT_COMPOSE_SERVICES = { **CORE_MIRAGE_COMPOSE_SERVICES, 'mirage-boot': 'mirage_boot_admin', + 'mirage-boot-api': 'mirage_boot_api', } BASE_SYNC_COMPOSE_SERVICES = { @@ -245,7 +249,7 @@ def is_volume_exists(name: str, dutils=None): dutils = dutils or docker_client() try: dutils.volumes.get(name) - except docker.errors.NotFound: + except NotFound: return False return True @@ -300,7 +304,7 @@ def get_compose_services(node_type: NodeType) -> list[str]: return result -def get_up_compose_cmd(node_type: NodeType, services: Optional[list[str]] = None) -> tuple: +def get_up_compose_cmd(node_type: NodeType, services: list[str] | None = None) -> tuple: compose_path = get_compose_path(node_type) if services is None: @@ -309,7 +313,9 @@ def get_up_compose_cmd(node_type: NodeType, services: Optional[list[str]] = None return ('docker', 'compose', '-f', compose_path, 'up', '-d', *services) -def compose_up(env, node_type: NodeType, is_mirage_boot: bool = False): +def compose_up( + env, node_type: NodeType, is_mirage_boot: bool = False, services: list[str] | None = None +): if node_type == NodeType.SYNC: logger.info('Running containers for sync node') run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.SYNC), env=env) @@ -320,10 +326,7 @@ def compose_up(env, node_type: NodeType, is_mirage_boot: bool = False): if node_type == NodeType.MIRAGE: logger.info('Running mirage base set of containers') - if not is_mirage_boot: - logger.debug('Launching mirage containers with env %s', env) - run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.MIRAGE), env=env) - else: + if is_mirage_boot: logger.debug('Launching mirage boot containers with env %s', env) run_cmd( cmd=get_up_compose_cmd( @@ -331,6 +334,9 @@ def compose_up(env, node_type: NodeType, is_mirage_boot: bool = False): ), env=env, ) + else: + logger.debug('Launching mirage containers with env %s', env) + run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.MIRAGE, services=services), env=env) else: logger.info('Running skale node base set of containers') logger.debug('Launching skale node containers with env %s', env) @@ -385,7 +391,7 @@ def is_container_running(name: str, dclient: Optional[DockerClient] = None) -> b try: container = dc.containers.get(name) return container.status == 'running' - except docker.errors.NotFound: + except NotFound: return False @@ -421,3 +427,19 @@ def docker_cleanup(dclient=None, ignore=None): system_prune() except Exception as e: logger.warning('Image cleanup errored with %s', e) + + +def wait_for_container(container_name: str, attempts: int = 10, interval: int = 3) -> bool: + logger.info('Waiting for container %s to be up', container_name) + dc = docker_client() + + for i in range(attempts): + try: + container = dc.containers.get(container_name) + if container.status == 'running': + logger.info('Container %s is up', container_name) + return True + except NotFound: + logger.warning('Container %s not found, retrying...', container_name) + time.sleep(interval) + return False diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index a8237026..556f5002 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -1,7 +1,8 @@ +import abc import json import os -from collections import namedtuple -from typing import Optional +from dataclasses import dataclass + from node_cli.configs import META_FILEPATH DEFAULT_VERSION = '1.0.0' @@ -11,62 +12,141 @@ DEFAULT_OS_VERSION = '18.04' -class CliMeta( - namedtuple('Node', ['version', 'config_stream', 'docker_lvmpy_stream', 'os_id', 'os_version']) -): - __slots__ = () - - def __new__( - cls, - version=DEFAULT_VERSION, - config_stream=DEFAULT_CONFIG_STREAM, - docker_lvmpy_stream: Optional[str] = DEFAULT_DOCKER_LVMPY_STREAM, - os_id=DEFAULT_OS_ID, - os_version=DEFAULT_OS_VERSION, - ): - return super(CliMeta, cls).__new__( - cls, version, config_stream, docker_lvmpy_stream, os_id, os_version - ) +@dataclass +class CliMetaBase(abc.ABC): + version: str = DEFAULT_VERSION + config_stream: str = DEFAULT_CONFIG_STREAM + os_id: str = DEFAULT_OS_ID + os_version: str = DEFAULT_OS_VERSION + + @abc.abstractmethod + def asdict(self) -> dict: + pass + + +@dataclass +class CliMeta(CliMetaBase): + docker_lvmpy_stream: str = DEFAULT_DOCKER_LVMPY_STREAM + + def asdict(self) -> dict: + return { + 'version': self.version, + 'config_stream': self.config_stream, + 'docker_lvmpy_stream': self.docker_lvmpy_stream, + 'os_id': self.os_id, + 'os_version': self.os_version, + } + + +@dataclass +class MirageCliMeta(CliMetaBase): + def asdict(self) -> dict: + return { + 'version': self.version, + 'config_stream': self.config_stream, + 'os_id': self.os_id, + 'os_version': self.os_version, + } + +class BaseCliMetaManager(abc.ABC): + def __init__(self, meta_filepath: str = META_FILEPATH) -> None: + self.meta_filepath = meta_filepath -def get_meta_info(raw: bool = False) -> CliMeta: - if not os.path.isfile(META_FILEPATH): - return None - with open(META_FILEPATH) as meta_file: - plain_meta = json.load(meta_file) - if raw: - return plain_meta - return CliMeta(**plain_meta) + def _get_plain_meta(self) -> dict: + if not os.path.isfile(self.meta_filepath): + return {} + with open(self.meta_filepath) as meta_file: + return json.load(meta_file) + @abc.abstractmethod + def get_meta_info(self, raw: bool = False) -> CliMetaBase | dict | None: + pass -def save_meta(meta: CliMeta) -> None: - with open(META_FILEPATH, 'w') as meta_file: - json.dump(meta._asdict(), meta_file) + def save_meta(self, meta: CliMetaBase) -> None: + with open(self.meta_filepath, 'w') as meta_file: + json.dump(meta.asdict(), meta_file) + @abc.abstractmethod + def compose_default_meta(self) -> CliMetaBase: + pass -def compose_default_meta() -> CliMeta: - return CliMeta( - version=DEFAULT_VERSION, - docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, - config_stream=DEFAULT_CONFIG_STREAM, - os_id=DEFAULT_OS_ID, - os_version=DEFAULT_OS_VERSION, - ) + def ensure_meta(self, meta: CliMetaBase | None = None) -> None: + if not self.get_meta_info(): + meta = meta or self.compose_default_meta() + self.save_meta(meta) + @abc.abstractmethod + def update_meta(self, *args, **kwargs) -> None: + pass -def ensure_meta(meta: Optional[CliMeta] = None) -> None: - if not get_meta_info(): - meta = meta or compose_default_meta() - save_meta(meta) +class CliMetaManager(BaseCliMetaManager): + def get_meta_info(self, raw: bool = False) -> CliMeta | dict | None: + plain_meta = self._get_plain_meta() + if not raw and not plain_meta: + return None + allowed_fields = set(CliMeta.__dataclass_fields__.keys()) + clean_plain_meta = {k: v for k, v in plain_meta.items() if k in allowed_fields} + + if raw: + return clean_plain_meta + return CliMeta(**clean_plain_meta) + + def compose_default_meta(self) -> CliMeta: + return CliMeta( + version=DEFAULT_VERSION, + docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, + config_stream=DEFAULT_CONFIG_STREAM, + os_id=DEFAULT_OS_ID, + os_version=DEFAULT_OS_VERSION, + ) + + def update_meta( + self, + version: str, + config_stream: str, + docker_lvmpy_stream: str | None, + os_id: str, + os_version: str, + ) -> None: + self.ensure_meta() + meta = CliMeta( + version, + config_stream, + os_id, + os_version, + docker_lvmpy_stream, + ) + self.save_meta(meta) + + +class MirageCliMetaManager(BaseCliMetaManager): + def get_meta_info(self, raw: bool = False) -> MirageCliMeta | dict | None: + plain_meta = self._get_plain_meta() + if not raw and not plain_meta: + return None + allowed_fields = set(MirageCliMeta.__dataclass_fields__.keys()) + clean_plain_meta = {k: v for k, v in plain_meta.items() if k in allowed_fields} + if raw: + return clean_plain_meta + return MirageCliMeta(**clean_plain_meta) + + def compose_default_meta(self) -> MirageCliMeta: + return MirageCliMeta( + version=DEFAULT_VERSION, + config_stream=DEFAULT_CONFIG_STREAM, + os_id=DEFAULT_OS_ID, + os_version=DEFAULT_OS_VERSION, + ) -def update_meta( - version: str, - config_stream: str, - docker_lvmpy_stream: Optional[str], - os_id: str, - os_version: str, -) -> None: - ensure_meta() - meta = CliMeta(version, config_stream, docker_lvmpy_stream, os_id, os_version) - save_meta(meta) + def update_meta( + self, + version: str, + config_stream: str, + os_id: str, + os_version: str, + ) -> None: + self.ensure_meta() + meta = MirageCliMeta(version, config_stream, os_id, os_version) + self.save_meta(meta) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 97676b9a..efc72c6c 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -8,4 +8,4 @@ LVMPY_LOG_DIR="$PROJECT_DIR/tests/" \ TEST_HOME_DIR="$PROJECT_DIR/tests/" \ GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" \ DOTENV_FILEPATH='tests/test-env' \ - py.test --cov=$PROJECT_DIR/ --ignore=tests/core/nftables_test.py --ignore=tests/core/migration_test.py tests $@ + py.test --cov=$PROJECT_DIR/ --ignore=tests/core/nftables_test.py --ignore=tests/core/migration_test.py tests/ $@ diff --git a/setup.py b/setup.py index ad03a8b4..5ea3e6ce 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ def find_version(*file_paths): 'sh==1.14.2', 'python-crontab==2.6.0', 'requests-mock==1.12.1', - 'redis==6.2.0' + 'redis==6.2.0', ], python_requires='>=3.8,<4', extras_require=extras_require, diff --git a/tests/cli/mirage_cli_test.py b/tests/cli/mirage_cli_test.py index 1a96c22a..5072bd40 100644 --- a/tests/cli/mirage_cli_test.py +++ b/tests/cli/mirage_cli_test.py @@ -9,12 +9,12 @@ init_node as init_node_placeholder, register_node as register_node_placeholder, update_node as update_node_placeholder, + migrate_node, ) from node_cli.cli.mirage_boot import ( init_boot, register_boot, signature_boot, - migrate_boot, ) @@ -149,20 +149,10 @@ def test_mirage_boot_init(mock_init_core, valid_env_file): mock_init_core.assert_called_once_with(valid_env_file) -@mock.patch('node_cli.cli.mirage_boot.migrate') -def test_mirage_boot_migrate(mock_migrate_core, valid_env_file): +@mock.patch('node_cli.cli.mirage_node.migrate_from_boot') +def test_mirage_node_migrate(mock_migrate_core, valid_env_file): runner = CliRunner() - result = runner.invoke(migrate_boot, ['--yes', valid_env_file]) + result = runner.invoke(migrate_node, ['--yes', valid_env_file]) assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' - mock_migrate_core.assert_called_once_with(valid_env_file, None) - - -@mock.patch('node_cli.cli.mirage_boot.migrate') -def test_mirage_boot_migrate_pull_config(mock_migrate_core, valid_env_file): - runner = CliRunner() - schain_name = 'my-schain-config' - result = runner.invoke(migrate_boot, ['--yes', '--pull-config', schain_name, valid_env_file]) - - assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' - mock_migrate_core.assert_called_once_with(valid_env_file, schain_name) + mock_migrate_core.assert_called_once_with(env_filepath=valid_env_file) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index d1dc82b7..60344b71 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -17,31 +17,32 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import logging import pathlib +from unittest.mock import MagicMock, patch import mock -from unittest.mock import MagicMock, patch +import pytest import requests -import logging -from node_cli.configs import SKALE_DIR, G_CONF_HOME from node_cli.cli.node import ( + _set_domain_name, + _turn_off, + _turn_on, + backup_node, node_info, register_node, - signature, - backup_node, + remove_node_from_maintenance, restore_node, set_node_in_maintenance, - remove_node_from_maintenance, + signature, version, - _turn_off, - _turn_on, - _set_domain_name, ) +from node_cli.configs import G_CONF_HOME, SKALE_DIR from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import init_default_logger from node_cli.utils.meta import CliMeta - +from node_cli.utils.node_type import NodeType from tests.helper import ( response_mock, run_command, @@ -321,56 +322,45 @@ def test_backup(): assert 'Backup archive succesfully created ' in result.output -def test_restore(mocked_g_config): +@pytest.mark.parametrize( + 'node_type,test_user_conf', + [ + (NodeType.REGULAR, 'regular_user_conf'), + (NodeType.MIRAGE, 'mirage_user_conf'), + (NodeType.SYNC, 'sync_user_conf'), + ], +) +def test_restore(request, node_type, test_user_conf, mocked_g_config, tmp_path): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) - result = run_command(backup_node, ['/tmp']) + result = run_command(backup_node, [tmp_path]) backup_path = result.output.replace('Backup archive successfully created: ', '').replace( '\n', '' ) with ( + patch('node_cli.cli.node.TYPE', node_type), patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, patch('subprocess.run', new=subprocess_run_mock), patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), patch('node_cli.utils.decorators.is_node_inited', return_value=False), patch( - 'node_cli.core.node.get_meta_info', + 'node_cli.core.node.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.4.0', config_stream='3.0.2'), ), patch('node_cli.operations.base.configure_nftables'), - patch('node_cli.configs.env.validate_env_params'), + patch('node_cli.configs.user.validate_alias_or_address'), ): - result = run_command(restore_node, [backup_path, './tests/test-env']) + user_conf_path = request.getfixturevalue(test_user_conf).as_posix() + + result = run_command(restore_node, [backup_path, user_conf_path]) assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa + assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') == 'True' - assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') == 'True' - - -def test_restore_no_snapshot(mocked_g_config): - pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) - result = run_command(backup_node, ['/tmp']) - backup_path = result.output.replace('Backup archive successfully created: ', '').replace( - '\n', '' - ) - - with ( - patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, - patch('subprocess.run', new=subprocess_run_mock), - patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - patch('node_cli.utils.decorators.is_node_inited', return_value=False), - patch( - 'node_cli.core.node.get_meta_info', - return_value=CliMeta(version='2.4.0', config_stream='3.0.2'), - ), - patch('node_cli.operations.base.configure_nftables'), - patch('node_cli.configs.env.validate_env_params'), - ): - result = run_command(restore_node, [backup_path, './tests/test-env', '--no-snapshot']) + result = run_command(restore_node, [backup_path, user_conf_path, '--no-snapshot']) assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa - - assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') is None + assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') is None def test_maintenance_on(): @@ -397,13 +387,15 @@ def test_maintenance_off(mocked_g_config): ) -def test_turn_off_maintenance_on(mocked_g_config): +def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf): resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with ( mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.core.node.SKALE_DIR_ENV_FILEPATH', regular_user_conf.as_posix()), mock.patch('node_cli.core.node.turn_off_op'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), - patch('node_cli.configs.env.validate_env_params'), + mock.patch('node_cli.configs.user.validate_alias_or_address'), + mock.patch('node_cli.cli.node.TYPE', NodeType.REGULAR), ): result = run_command_mock( 'node_cli.utils.helper.requests.post', @@ -427,7 +419,7 @@ def test_turn_off_maintenance_on(mocked_g_config): assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE -def test_turn_on_maintenance_off(mocked_g_config): +def test_turn_on_maintenance_off(mocked_g_config, regular_user_conf): resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with ( mock.patch('subprocess.run', new=subprocess_run_mock), @@ -435,13 +427,14 @@ def test_turn_on_maintenance_off(mocked_g_config): mock.patch('node_cli.core.node.turn_on_op'), mock.patch('node_cli.core.node.is_base_containers_alive'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), - patch('node_cli.configs.env.validate_env_params'), + mock.patch('node_cli.configs.user.validate_alias_or_address'), + mock.patch('node_cli.cli.node.TYPE', NodeType.REGULAR), ): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, _turn_on, - ['./tests/test-env', '--maintenance-off', '--sync-schains', '--yes'], + [regular_user_conf.as_posix(), '--maintenance-off', '--sync-schains', '--yes'], ) assert result.exit_code == 0 diff --git a/tests/cli/resources_allocation_test.py b/tests/cli/resources_allocation_test.py index b317aad6..4f59daea 100644 --- a/tests/cli/resources_allocation_test.py +++ b/tests/cli/resources_allocation_test.py @@ -17,22 +17,20 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import os import json -import mock -import requests +import os +import mock import pytest +import requests -from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH, NODE_DATA_PATH +from node_cli.cli.resources_allocation import generate, show +from node_cli.configs.resource_allocation import NODE_DATA_PATH, RESOURCE_ALLOCATION_FILEPATH from node_cli.utils.helper import safe_mkdir, write_json +from node_cli.utils.node_type import NodeType from tests.helper import response_mock, run_command_mock - -from node_cli.cli.resources_allocation import show, generate - from tests.resources_test import BIG_DISK_SIZE - TEST_CONFIG = {'test': 1} @@ -51,15 +49,18 @@ def test_show(resource_alloc_config): assert result.exit_code == 0 -def test_generate(): +def test_generate(regular_user_conf): safe_mkdir(NODE_DATA_PATH) resp_mock = response_mock(requests.codes.created) with ( mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - mock.patch('node_cli.configs.env.validate_env_params'), + mock.patch('node_cli.configs.user.validate_alias_or_address'), ): result = run_command_mock( - 'node_cli.utils.helper.post_request', resp_mock, generate, ['./tests/test-env', '--yes'] + 'node_cli.utils.helper.post_request', + resp_mock, + generate, + [regular_user_conf.as_posix(), '--yes'], ) assert result.output == ( f'Resource allocation file generated: {RESOURCE_ALLOCATION_FILEPATH}\n' @@ -67,14 +68,18 @@ def test_generate(): assert result.exit_code == 0 -def test_generate_already_exists(resource_alloc_config): +def test_generate_already_exists(regular_user_conf, resource_alloc_config): resp_mock = response_mock(requests.codes.created) with ( mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - mock.patch('node_cli.configs.env.validate_env_params'), + mock.patch('node_cli.cli.node.TYPE', NodeType.REGULAR), + mock.patch('node_cli.configs.user.validate_alias_or_address'), ): result = run_command_mock( - 'node_cli.utils.helper.post_request', resp_mock, generate, ['./tests/test-env', '--yes'] + 'node_cli.utils.helper.post_request', + resp_mock, + generate, + [regular_user_conf.as_posix(), '--yes'], ) assert result.output == 'Resource allocation file already exists\n' assert result.exit_code == 0 @@ -83,7 +88,7 @@ def test_generate_already_exists(resource_alloc_config): 'node_cli.utils.helper.post_request', resp_mock, generate, - ['./tests/test-env', '--yes', '--force'], + [regular_user_conf.as_posix(), '--yes', '--force'], ) assert result.output == ( f'Resource allocation file generated: {RESOURCE_ALLOCATION_FILEPATH}\n' diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index 9206245a..014510e1 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -17,26 +17,26 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import logging import pathlib import mock -import logging -from node_cli.configs import SKALE_DIR, NODE_DATA_PATH +from node_cli.cli.sync_node import _cleanup_sync, _init_sync, _update_sync +from node_cli.configs import NODE_DATA_PATH, SKALE_DIR from node_cli.core.node_options import NodeOptions -from node_cli.cli.sync_node import _init_sync, _update_sync, _cleanup_sync -from node_cli.utils.meta import CliMeta from node_cli.utils.helper import init_default_logger - +from node_cli.utils.meta import CliMeta +from node_cli.utils.node_type import NodeType +from tests.conftest import set_env_var from tests.helper import run_command, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE -from tests.conftest import set_env_var logger = logging.getLogger(__name__) init_default_logger() -def test_init_sync(mocked_g_config, clean_node_options): +def test_init_sync(mocked_g_config, clean_node_options, sync_user_conf): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), @@ -45,9 +45,9 @@ def test_init_sync(mocked_g_config, clean_node_options): mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), - mock.patch('node_cli.configs.env.validate_env_params'), + mock.patch('node_cli.configs.user.validate_alias_or_address'), ): - result = run_command(_init_sync, ['./tests/test-env']) + result = run_command(_init_sync, [sync_user_conf.as_posix()]) node_options = NodeOptions() assert not node_options.archive @@ -57,9 +57,8 @@ def test_init_sync(mocked_g_config, clean_node_options): assert result.exit_code == 0 -def test_init_sync_archive(mocked_g_config, clean_node_options): +def test_init_sync_archive(mocked_g_config, clean_node_options, sync_user_conf): pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) - # with mock.patch('subprocess.run', new=subprocess_run_mock), \ with ( mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.operations.base.cleanup_volume_artifacts'), @@ -71,16 +70,17 @@ def test_init_sync_archive(mocked_g_config, clean_node_options): mock.patch('node_cli.operations.base.link_env_file'), mock.patch('node_cli.operations.base.generate_nginx_config'), mock.patch('node_cli.operations.base.prepare_block_device'), - mock.patch('node_cli.operations.base.update_meta'), + mock.patch('node_cli.operations.base.CliMetaManager.update_meta'), mock.patch('node_cli.operations.base.update_resource_allocation'), mock.patch('node_cli.operations.base.update_images'), mock.patch('node_cli.operations.base.compose_up'), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), - mock.patch('node_cli.configs.env.validate_env_params'), + mock.patch('node_cli.configs.user.validate_alias_or_address'), + mock.patch('node_cli.cli.node.TYPE', NodeType.SYNC), ): - result = run_command(_init_sync, ['./tests/test-env', '--archive']) + result = run_command(_init_sync, [sync_user_conf.as_posix(), '--archive']) node_options = NodeOptions() assert node_options.archive @@ -107,7 +107,7 @@ def test_init_archive_indexer_fail(mocked_g_config, clean_node_options): assert 'Cannot use both' in result.output -def test_update_sync(mocked_g_config): +def test_update_sync(sync_user_conf, mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( @@ -118,12 +118,12 @@ def test_update_sync(mocked_g_config): mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), mock.patch( - 'node_cli.core.node.get_meta_info', + 'node_cli.core.node.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), - mock.patch('node_cli.configs.env.validate_env_params'), + mock.patch('node_cli.configs.user.validate_alias_or_address'), ): - result = run_command(_update_sync, ['./tests/test-env', '--yes']) + result = run_command(_update_sync, [sync_user_conf.as_posix(), '--yes']) assert result.exit_code == 0 @@ -139,7 +139,7 @@ def test_cleanup_sync(mocked_g_config): mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), mock.patch('node_cli.core.node.compose_node_env', return_value={'SCHAIN_NAME': 'test'}), mock.patch( - 'node_cli.core.node.get_meta_info', + 'node_cli.core.node.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), ): diff --git a/tests/configs/configs_env_validate_test.py b/tests/configs/configs_env_validate_test.py index 03c955e9..a9703b38 100644 --- a/tests/configs/configs_env_validate_test.py +++ b/tests/configs/configs_env_validate_test.py @@ -1,33 +1,27 @@ import os from typing import Optional + import pytest import requests -import mock - -from node_cli.configs.env import ( - absent_required_params, - load_env_file, - build_env_params, - populate_env_params, - get_validated_env_config, - validate_env_params, - validate_env_type, - ALLOWED_ENV_TYPES, - REQUIRED_PARAMS_SKALE, - REQUIRED_PARAMS_SYNC, - REQUIRED_PARAMS_MIRAGE_BOOT, - REQUIRED_PARAMS_MIRAGE, - OPTIONAL_PARAMS, -) + from node_cli.configs.alias_address_validation import ( - validate_env_alias_or_address, - validate_contract_address, - validate_contract_alias, + ContractType, get_chain_id, get_network_metadata, - ContractType, + validate_alias_or_address, + validate_contract_address, + validate_contract_alias, +) +from node_cli.configs.user import ( + ALLOWED_ENV_TYPES, + MirageBootUserConfig, + MirageUserConfig, + SkaleUserConfig, + SyncUserConfig, + get_user_config_class, + get_validated_user_config, + validate_env_type, ) -from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.node_type import NodeType ENDPOINT = 'http://localhost:8545' @@ -42,72 +36,19 @@ def json(self): return self._json_data -def test_absent_required_params_returns_missing_keys(): - params = { - 'A': '', - 'B': 'value', - 'C': '', - 'MONITORING_CONTAINERS': 'optional', - } - missing = absent_required_params(params) - assert 'A' in missing - assert 'C' in missing - assert 'MONITORING_CONTAINERS' not in missing - - -def test_load_env_file_nonexistent(): - with pytest.raises(SystemExit) as excinfo: - load_env_file('nonexistent.env') - assert excinfo.value.code == CLIExitCodes.FAILURE.value - - -def test_populate_env_params_updates_from_environ(monkeypatch): - params = {'FOO': ''} - monkeypatch.setenv('FOO', 'bar') - populate_env_params(params) - assert params['FOO'] == 'bar' - - @pytest.mark.parametrize( - 'node_type, is_mirage_boot, expected_keys, unexpected_keys', + 'node_type, is_mirage_boot, expected_type', [ - ( - NodeType.REGULAR, - False, - REQUIRED_PARAMS_SKALE.keys(), - {'SCHAIN_NAME'}, - ), - ( - NodeType.SYNC, - False, - REQUIRED_PARAMS_SYNC.keys(), - set(), - ), - ( - NodeType.MIRAGE, - True, - REQUIRED_PARAMS_MIRAGE_BOOT.keys(), - {'DOCKER_LVMPY_STREAM', 'SCHAIN_NAME'}, - ), - ( - NodeType.MIRAGE, - False, - REQUIRED_PARAMS_MIRAGE.keys(), - {'IMA_CONTRACTS', 'DOCKER_LVMPY_STREAM', 'SCHAIN_NAME'}, - ), + (NodeType.REGULAR, False, SkaleUserConfig), + (NodeType.SYNC, False, SyncUserConfig), + (NodeType.MIRAGE, True, MirageBootUserConfig), + (NodeType.MIRAGE, False, MirageUserConfig), ], ids=['regular', 'sync', 'mirage_boot', 'mirage_regular'], ) -def test_build_env_params_keys(node_type, is_mirage_boot, expected_keys, unexpected_keys): - params = build_env_params(node_type=node_type, is_mirage_boot=is_mirage_boot) - param_keys = set(params.keys()) - - all_expected = set(expected_keys) | set(OPTIONAL_PARAMS.keys()) - missing_expected = all_expected - param_keys - assert not missing_expected, f'Missing expected keys: {missing_expected}' - - found_unexpected = set(unexpected_keys) & param_keys - assert not found_unexpected, f'Found unexpected keys: {found_unexpected}' +def test_build_env_params_keys(node_type, is_mirage_boot, expected_type): + env_type = get_user_config_class(node_type=node_type, is_mirage_boot=is_mirage_boot) + assert env_type == expected_type @pytest.mark.parametrize( @@ -210,7 +151,7 @@ def test_validate_contract_alias(requests_mock, networks, should_raise): def test_validate_env_alias_or_address_with_address(requests_mock): addr = '0x' + 'b' * 40 requests_mock.post(ENDPOINT, json={'result': '0x1'}) - validate_env_alias_or_address(addr, ContractType.IMA, ENDPOINT) + validate_alias_or_address(addr, ContractType.IMA, ENDPOINT) def test_validate_env_alias_or_address_with_alias(requests_mock): @@ -220,115 +161,12 @@ def test_validate_env_alias_or_address_with_alias(requests_mock): requests_mock.get(metadata_url, json=metadata, status_code=200) alias_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/mainnet/mainnet-ima/test-alias.json' requests_mock.get(alias_url, status_code=200) - validate_env_alias_or_address('test-alias', ContractType.IMA, ENDPOINT) - - -@pytest.mark.parametrize('env_type', ALLOWED_ENV_TYPES) -@pytest.mark.parametrize( - 'required_params, key_to_remove, should_fail', - [ - (REQUIRED_PARAMS_MIRAGE_BOOT, None, False), - (REQUIRED_PARAMS_MIRAGE, None, False), - (REQUIRED_PARAMS_MIRAGE_BOOT, 'IMA_CONTRACTS', True), - (REQUIRED_PARAMS_MIRAGE_BOOT, 'FILEBEAT_HOST', True), - (REQUIRED_PARAMS_MIRAGE, 'FILEBEAT_HOST', True), - ], - ids=[ - 'mirage_boot', - 'mirage_regular', - 'mirage_boot_missing_ima', - 'mirage_boot_missing_filebeat', - 'mirage_regular_missing_filebeat', - ], -) -@mock.patch('node_cli.configs.env.validate_env_alias_or_address') -@mock.patch('node_cli.configs.env.validate_env_type') -def test_validate_env_params_mirage( - mock_validate_type, - mock_validate_alias, - required_params, - key_to_remove, - should_fail, - env_type, -): - params = {k: f'{k}_val' for k in required_params} - params['ENV_TYPE'] = env_type - - if key_to_remove: - params[key_to_remove] = '' - - if should_fail: - with pytest.raises(SystemExit): - validate_env_params(params=params) - else: - validate_env_params(params=params) - - -@pytest.mark.parametrize( - 'node_type, is_boot, required_keys_dict', - [ - (NodeType.MIRAGE, True, REQUIRED_PARAMS_MIRAGE_BOOT), - (NodeType.MIRAGE, False, REQUIRED_PARAMS_MIRAGE), - ], - ids=['mirage_boot', 'mirage_regular'], -) -@mock.patch('node_cli.configs.alias_address_validation.validate_env_alias_or_address') -@mock.patch('node_cli.configs.alias_address_validation.get_chain_id', return_value=1) -@mock.patch( - 'node_cli.configs.alias_address_validation.get_network_metadata', - return_value={'networks': [{'chainId': 1, 'path': 'mainnet'}]}, -) -def test_get_validated_env_config_mirage_success( - mock_meta, - mock_chain, - mock_validate_alias, - tmp_path, - monkeypatch, - node_type, - is_boot, - required_keys_dict, -): - env_file = tmp_path / 'mirage.env' - env_content = '' - expected_config = {} - - for key in {**required_keys_dict, **OPTIONAL_PARAMS}: - env_value = f'{key}_value' - if key == 'ENDPOINT': - env_value = ENDPOINT - if key == 'ENV_TYPE': - env_value = 'devnet' - if key == 'MANAGER_CONTRACTS': - env_value = '0x' + '1' * 40 - if key == 'IMA_CONTRACTS': - env_value = '0x' + '2' * 40 - - if key in required_keys_dict: - env_content += f'{key}={env_value}\n' - monkeypatch.setenv(key, env_value) - expected_config[key] = env_value - - env_file.write_text(env_content) - - with mock.patch('node_cli.configs.alias_address_validation.requests.post') as mock_post: - mock_post.return_value = FakeResponse(200, {'result': '0x123'}) - - config = get_validated_env_config( - node_type=node_type, env_filepath=str(env_file), is_mirage_boot=is_boot - ) - - assert config is not None - assert set(config.keys()) == set(expected_config.keys()) - for key in expected_config: - assert config[key] == expected_config[key] - - for key in {**required_keys_dict, **OPTIONAL_PARAMS}: - monkeypatch.delenv(key, raising=False) + validate_alias_or_address('test-alias', ContractType.IMA, ENDPOINT) def test_get_validated_env_config_missing_file(): with pytest.raises(SystemExit): - get_validated_env_config(env_filepath='nonexistent.env', node_type=NodeType.REGULAR) + get_validated_user_config(env_filepath='nonexistent.env', node_type=NodeType.REGULAR) def test_get_validated_env_config_unreadable_file(tmp_path): @@ -338,6 +176,6 @@ def test_get_validated_env_config_unreadable_file(tmp_path): try: os.chmod(env_file, 0o000) with pytest.raises(PermissionError): - get_validated_env_config(env_filepath=str(env_file), node_type=NodeType.REGULAR) + get_validated_user_config(env_filepath=str(env_file), node_type=NodeType.REGULAR) finally: os.chmod(env_file, original_mode) diff --git a/tests/conftest.py b/tests/conftest.py index 17b43c4a..c6f71fac 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,9 +19,9 @@ import json import os -import tempfile import pathlib import shutil +import tempfile from contextlib import contextmanager import docker @@ -34,18 +34,17 @@ GLOBAL_SKALE_CONF_FILEPATH, GLOBAL_SKALE_DIR, META_FILEPATH, + NGINX_CONFIG_FILEPATH, NGINX_CONTAINER_NAME, + REDIS_URI, REMOVED_CONTAINERS_FOLDER_PATH, SCHAIN_NODE_DATA_PATH, - NGINX_CONFIG_FILEPATH, - REDIS_URI, ) from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH -from node_cli.configs.ssl import SSL_FOLDER_PATH from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH +from node_cli.configs.ssl import SSL_FOLDER_PATH from node_cli.utils.docker_utils import docker_client from node_cli.utils.global_config import generate_g_config_file - from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3, TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN @@ -302,6 +301,91 @@ def set_env_var(name, value): os.environ[name] = old_value +@pytest.fixture +def regular_user_conf(tmp_path): + test_env_path = pathlib.Path(tmp_path / 'test-env') + try: + test_env = """ + ENDPOINT=http://localhost:8545 + CONTAINER_CONFIGS_STREAM='main' + FILEBEAT_HOST=127.0.0.1:3010 + SGX_SERVER_URL=http://127.0.0.1 + DISK_MOUNTPOINT=/dev/sss + DOCKER_LVMPY_STREAM='master' + ENV_TYPE='devnet' + MANAGER_CONTRACTS='test-manager' + IMA_CONTRACTS='test-ima' + """ + with open(test_env_path, 'w') as env_file: + env_file.write(test_env) + yield test_env_path + finally: + test_env_path.unlink() + + +@pytest.fixture +def mirage_user_conf(tmp_path): + test_env_path = pathlib.Path(tmp_path / 'test-env') + try: + test_env = """ + BOOT_ENDPOINT=http://localhost:8545 + CONTAINER_CONFIGS_STREAM='main' + FILEBEAT_HOST=127.0.0.1:3010 + SGX_SERVER_URL=http://127.0.0.1 + DISK_MOUNTPOINT=/dev/sss + ENV_TYPE='devnet' + ENFORCE_BTRFS=False + MIRAGE_CONTRACTS='test-mirage' + """ + with open(test_env_path, 'w') as env_file: + env_file.write(test_env) + yield test_env_path + finally: + test_env_path.unlink() + + +@pytest.fixture +def mirage_boot_user_conf(tmp_path): + test_env_path = pathlib.Path(tmp_path / 'test-env') + try: + test_env = """ + ENDPOINT=http://localhost:8545 + CONTAINER_CONFIGS_STREAM='main' + FILEBEAT_HOST=127.0.0.1:3010 + SGX_SERVER_URL=http://127.0.0.1 + DISK_MOUNTPOINT=/dev/sss + ENV_TYPE='devnet' + MANAGER_CONTRACTS='test-manager' + IMA_CONTRACTS='test-ima' + """ + with open(test_env_path, 'w') as env_file: + env_file.write(test_env) + yield test_env_path + finally: + test_env_path.unlink() + + +@pytest.fixture +def sync_user_conf(tmp_path): + test_env_path = pathlib.Path(tmp_path / 'test-env') + try: + test_env = """ + ENDPOINT=http://localhost:8545 + CONTAINER_CONFIGS_STREAM='main' + FILEBEAT_HOST=127.0.0.1:3010 + DISK_MOUNTPOINT=/dev/sss + ENV_TYPE='devnet' + SCHAIN_NAME='test-schain' + ENFORCE_BTRFS=False + MANAGER_CONTRACTS='test-manager' + """ + with open(test_env_path, 'w') as env_file: + env_file.write(test_env) + yield test_env_path + finally: + test_env_path.unlink() + + @pytest.fixture def redis_client(): cpool = redis.ConnectionPool.from_url(REDIS_URI) diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index c71e4683..fb1484d6 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -12,17 +12,16 @@ from node_cli.configs import NODE_DATA_PATH, SCHAINS_MNT_DIR_REGULAR, SCHAINS_MNT_DIR_SINGLE_CHAIN from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.core.node import ( + compose_node_env, get_expected_container_names, - is_base_containers_alive, init, + is_base_containers_alive, + is_update_safe, pack_dir, update, - is_update_safe, - compose_node_env, ) from node_cli.utils.meta import CliMeta from node_cli.utils.node_type import NodeType - from tests.helper import response_mock, safe_update_api_response, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE @@ -149,15 +148,60 @@ def test_is_base_containers_alive_empty(node_type, is_boot): @pytest.mark.parametrize( ( - 'node_type, is_boot, inited_node, sync_schains, expected_mnt_dir, ' + 'node_type, test_user_conf, is_boot, inited_node, sync_schains, expected_mnt_dir,' 'expect_flask_key, expect_backup_run' ), [ - (NodeType.REGULAR, False, True, False, SCHAINS_MNT_DIR_REGULAR, True, False), - (NodeType.REGULAR, False, True, True, SCHAINS_MNT_DIR_REGULAR, True, True), - (NodeType.SYNC, False, False, False, SCHAINS_MNT_DIR_SINGLE_CHAIN, False, False), - (NodeType.MIRAGE, True, True, False, SCHAINS_MNT_DIR_SINGLE_CHAIN, True, False), - (NodeType.MIRAGE, False, True, False, SCHAINS_MNT_DIR_SINGLE_CHAIN, True, False), + ( + NodeType.REGULAR, + 'regular_user_conf', + False, + True, + False, + SCHAINS_MNT_DIR_REGULAR, + True, + False, + ), + ( + NodeType.REGULAR, + 'regular_user_conf', + False, + True, + True, + SCHAINS_MNT_DIR_REGULAR, + True, + True, + ), + ( + NodeType.SYNC, + 'sync_user_conf', + False, + False, + False, + SCHAINS_MNT_DIR_SINGLE_CHAIN, + False, + False, + ), + ( + NodeType.MIRAGE, + 'mirage_boot_user_conf', + True, + True, + False, + SCHAINS_MNT_DIR_SINGLE_CHAIN, + True, + False, + ), + ( + NodeType.MIRAGE, + 'mirage_user_conf', + False, + True, + False, + SCHAINS_MNT_DIR_SINGLE_CHAIN, + True, + False, + ), ], ids=[ 'regular', @@ -167,42 +211,33 @@ def test_is_base_containers_alive_empty(node_type, is_boot): 'mirage_regular', ], ) -@mock.patch('node_cli.core.node.get_validated_env_config') -@mock.patch('node_cli.core.node.save_env_params') -@mock.patch('node_cli.core.node.get_flask_secret_key', return_value='mock_secret') def test_compose_node_env( - mock_get_secret, - mock_save_params, - mock_get_validated, + request, node_type, + test_user_conf, is_boot, inited_node, sync_schains, expected_mnt_dir, expect_flask_key, expect_backup_run, - valid_env_file, - valid_env_params, ): - mock_get_validated.return_value = valid_env_params.copy() - if node_type == NodeType.SYNC: - mock_get_validated.return_value['ENV_TYPE'] = 'devnet' - else: - mock_get_validated.return_value['ENV_TYPE'] = 'mainnet' - - result_env = compose_node_env( - env_filepath=valid_env_file, - inited_node=inited_node, - sync_schains=sync_schains, - node_type=node_type, - is_mirage_boot=is_boot, - save=True, - ) + user_config_path = request.getfixturevalue(test_user_conf) + + with ( + mock.patch('node_cli.configs.user.validate_alias_or_address'), + mock.patch('node_cli.core.node.save_env_params'), + mock.patch('node_cli.core.node.get_flask_secret_key', return_value='mock_secret'), + ): + result_env = compose_node_env( + env_filepath=user_config_path.as_posix(), + inited_node=inited_node, + sync_schains=sync_schains, + node_type=node_type, + is_mirage_boot=is_boot, + save=True, + ) - mock_save_params.assert_called_once_with(valid_env_file) - mock_get_validated.assert_called_once_with( - env_filepath=valid_env_file, node_type=node_type, is_mirage_boot=is_boot - ) assert result_env['SCHAINS_MNT_DIR'] == expected_mnt_dir assert ( 'FLASK_SECRET_KEY' in result_env and result_env['FLASK_SECRET_KEY'] is not None @@ -211,7 +246,6 @@ def test_compose_node_env( assert result_env['FLASK_SECRET_KEY'] == 'mock_secret' should_have_backup = sync_schains and node_type != NodeType.SYNC assert ('BACKUP_RUN' in result_env and result_env['BACKUP_RUN'] == 'True') == should_have_backup - assert result_env['ENDPOINT'] == valid_env_params['ENDPOINT'] @pytest.fixture @@ -275,10 +309,9 @@ def resource_file(): os.remove(RESOURCE_ALLOCATION_FILEPATH) -def test_init_node(no_resource_file): # todo: write new init node test +def test_init_node(regular_user_conf, no_resource_file): # todo: write new init node test resp_mock = response_mock(requests.codes.created) assert not os.path.isfile(RESOURCE_ALLOCATION_FILEPATH) - env_filepath = './tests/test-env' with ( mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), @@ -288,15 +321,13 @@ def test_init_node(no_resource_file): # todo: write new init node test mock.patch('node_cli.core.node.init_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.utils.helper.post_request', resp_mock), - mock.patch('node_cli.configs.env.validate_env_params'), + mock.patch('node_cli.configs.user.validate_alias_or_address'), ): - init(env_filepath=env_filepath, node_type=NodeType.REGULAR) + init(env_filepath=regular_user_conf.as_posix(), node_type=NodeType.REGULAR) assert os.path.isfile(RESOURCE_ALLOCATION_FILEPATH) -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.SYNC, NodeType.MIRAGE]) -def test_update_node(node_type, mocked_g_config, resource_file, inited_node): - env_filepath = './tests/test-env' +def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_node): resp_mock = response_mock(requests.codes.created) os.makedirs(NODE_DATA_PATH, exist_ok=True) with ( @@ -311,15 +342,19 @@ def test_update_node(node_type, mocked_g_config, resource_file, inited_node): mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.core.host.init_data_dir'), mock.patch( - 'node_cli.core.node.get_meta_info', + 'node_cli.core.node.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), - mock.patch('node_cli.configs.env.validate_env_params'), + mock.patch('node_cli.configs.user.validate_alias_or_address'), ): with mock.patch( 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() ): # noqa - result = update(env_filepath, pull_config_for_schain=None, node_type=node_type) + result = update( + regular_user_conf.as_posix(), + pull_config_for_schain=None, + node_type=NodeType.REGULAR, + ) assert result is None diff --git a/tests/core/core_schains_test.py b/tests/core/core_schains_test.py index 4b8e0fcb..c6ce426b 100644 --- a/tests/core/core_schains_test.py +++ b/tests/core/core_schains_test.py @@ -1,13 +1,12 @@ import os -from unittest import mock from pathlib import Path +from unittest import mock import freezegun -from node_cli.core.schains import cleanup_sync_datadir, toggle_schain_repair_mode +from node_cli.core.schains import cleanup_datadir_for_single_chain_node, toggle_schain_repair_mode from node_cli.utils.helper import read_json - -from tests.helper import CURRENT_TIMESTAMP, CURRENT_DATETIME +from tests.helper import CURRENT_DATETIME, CURRENT_TIMESTAMP @freezegun.freeze_time(CURRENT_DATETIME) @@ -82,5 +81,5 @@ def test_cleanup_sync_datadir(tmp_sync_datadir): hash_path.touch() with mock.patch('node_cli.core.schains.rm_btrfs_subvolume'): - cleanup_sync_datadir(schain_name, base_path=tmp_sync_datadir) + cleanup_datadir_for_single_chain_node(schain_name, base_path=tmp_sync_datadir) assert not os.path.isdir(base_folder) diff --git a/tests/mirage/mirage_node_test.py b/tests/mirage/mirage_node_test.py index 285729f4..f4f9d340 100644 --- a/tests/mirage/mirage_node_test.py +++ b/tests/mirage/mirage_node_test.py @@ -1,10 +1,14 @@ from unittest import mock import freezegun +import pytest from node_cli.configs import SKALE_DIR -from node_cli.mirage.mirage_boot import init as init_boot, migrate, update -from node_cli.mirage.mirage_node import request_repair, restore_mirage +from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH +from node_cli.mirage.mirage_boot import init as init_boot +from node_cli.mirage.mirage_boot import update +from node_cli.mirage.mirage_node import cleanup, migrate_from_boot, request_repair, restore_mirage +from node_cli.operations.mirage import MirageUpdateType from node_cli.utils.node_type import NodeType from tests.helper import CURRENT_DATETIME, CURRENT_TIMESTAMP @@ -65,11 +69,11 @@ def test_init_mirage_boot( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) @mock.patch('node_cli.mirage.mirage_boot.is_base_containers_alive', return_value=True) @mock.patch('node_cli.mirage.mirage_boot.time.sleep') -@mock.patch('node_cli.mirage.mirage_boot.migrate_mirage_boot_op') +@mock.patch('node_cli.mirage.mirage_boot.update_mirage_boot_op') @mock.patch('node_cli.mirage.mirage_boot.compose_node_env') -def test_migrate_mirage_boot( +def test_update_mirage_boot( mock_compose_env, - mock_migrate_op, + mock_update_op, mock_sleep, mock_is_alive, mock_is_user_valid, @@ -80,10 +84,10 @@ def test_migrate_mirage_boot( ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env - mock_migrate_op.return_value = True + mock_update_op.return_value = True pull_config_for_schain = 'mirage' - migrate(valid_env_file, pull_config_for_schain) + update(valid_env_file, pull_config_for_schain) mock_compose_env.assert_called_once_with( valid_env_file, @@ -91,23 +95,20 @@ def test_migrate_mirage_boot( sync_schains=False, pull_config_for_schain=pull_config_for_schain, node_type=NodeType.MIRAGE, + is_mirage_boot=True, ) - mock_migrate_op.assert_called_once_with(valid_env_file, mock_env) + mock_update_op.assert_called_once_with(valid_env_file, mock_env) mock_sleep.assert_called_once() - mock_is_alive.assert_called_once_with(node_type=NodeType.MIRAGE) + mock_is_alive.assert_called_once_with(node_type=NodeType.MIRAGE, is_mirage_boot=True) +@mock.patch('node_cli.mirage.mirage_node.update_mirage_op') +@mock.patch('node_cli.mirage.mirage_node.compose_node_env') @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.mirage.mirage_boot.is_base_containers_alive', return_value=True) -@mock.patch('node_cli.mirage.mirage_boot.time.sleep') -@mock.patch('node_cli.mirage.mirage_boot.update_mirage_boot_op') -@mock.patch('node_cli.mirage.mirage_boot.compose_node_env') -def test_update_mirage_boot( - mock_compose_env, - mock_update_op, - mock_sleep, - mock_is_alive, +def test_migrate_from_boot( mock_is_user_valid, + mock_compose_env, + mock_migrate_op, valid_env_file, inited_node, resource_alloc, @@ -115,33 +116,160 @@ def test_update_mirage_boot( ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env - mock_update_op.return_value = True - pull_config_for_schain = 'mirage' + mock_migrate_op.return_value = True - update(valid_env_file, pull_config_for_schain) + migrate_from_boot(valid_env_file) mock_compose_env.assert_called_once_with( valid_env_file, inited_node=True, sync_schains=False, - pull_config_for_schain=pull_config_for_schain, node_type=NodeType.MIRAGE, - is_mirage_boot=True, ) - mock_update_op.assert_called_once_with(valid_env_file, mock_env) - mock_sleep.assert_called_once() - mock_is_alive.assert_called_once_with(node_type=NodeType.MIRAGE, is_mirage_boot=True) + mock_migrate_op.assert_called_once_with( + valid_env_file, mock_env, update_type=MirageUpdateType.FROM_BOOT + ) @freezegun.freeze_time(CURRENT_DATETIME) @mock.patch('node_cli.mirage.mirage_node.compose_node_env', return_value={'ENV_TYPE': 'devnet'}) -@mock.patch( - 'node_cli.mirage.mirage_node.get_static_params', return_value={'info': {'chain_name': 'test'}} -) -def test_mirage_repair(compsoe_node_env_mock, get_statis_params_mock, redis_client, inited_node): +@mock.patch('node_cli.mirage.record.chain_record.get_mirage_chain_name', return_value='test') +def test_mirage_repair(compose_node_env_mock, get_static_params_mock, redis_client, inited_node): request_repair() assert redis_client.get('test_repair_ts') == f'{CURRENT_TIMESTAMP}'.encode('utf-8') assert redis_client.get('test_snapshot_from') == b'' request_repair(snapshot_from='127.0.0.1') assert redis_client.get('test_repair_ts') == f'{CURRENT_TIMESTAMP}'.encode('utf-8') assert redis_client.get('test_snapshot_from') == b'127.0.0.1' + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.mirage.mirage_node.cleanup_docker_configuration') +@mock.patch('node_cli.mirage.mirage_node.cleanup_mirage_op') +@mock.patch('node_cli.mirage.mirage_node.compose_node_env') +def test_cleanup_success( + mock_compose_env, + mock_cleanup_mirage_op, + mock_cleanup_docker_config, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, +): + mock_env = {'ENV_TYPE': 'devnet'} + mock_compose_env.return_value = mock_env + + cleanup() + + mock_compose_env.assert_called_once_with( + SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.MIRAGE + ) + mock_cleanup_mirage_op.assert_called_once_with(mock_env) + mock_cleanup_docker_config.assert_called_once() + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.mirage.mirage_node.cleanup_docker_configuration') +@mock.patch('node_cli.mirage.mirage_node.cleanup_mirage_op') +@mock.patch('node_cli.mirage.mirage_node.compose_node_env') +def test_cleanup_calls_operations_in_correct_order( + mock_compose_env, + mock_cleanup_mirage_op, + mock_cleanup_docker_config, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, +): + from node_cli.mirage.mirage_node import cleanup + + mock_env = {'ENV_TYPE': 'devnet'} + mock_compose_env.return_value = mock_env + + manager = mock.Mock() + manager.attach_mock(mock_compose_env, 'compose_env') + manager.attach_mock(mock_cleanup_mirage_op, 'cleanup_mirage_op') + manager.attach_mock(mock_cleanup_docker_config, 'cleanup_docker_config') + + cleanup() + + expected_calls = [ + mock.call.compose_env(mock.ANY, save=False, node_type=mock.ANY), + mock.call.cleanup_mirage_op(mock_env), + mock.call.cleanup_docker_config(), + ] + manager.assert_has_calls(expected_calls, any_order=False) + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.mirage.mirage_node.cleanup_docker_configuration') +@mock.patch( + 'node_cli.mirage.mirage_node.cleanup_mirage_op', side_effect=Exception('Cleanup failed') +) +@mock.patch('node_cli.mirage.mirage_node.compose_node_env') +def test_cleanup_continues_after_mirage_op_error( + mock_compose_env, + mock_cleanup_mirage_op, + mock_cleanup_docker_config, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, +): + mock_env = {'ENV_TYPE': 'devnet'} + mock_compose_env.return_value = mock_env + + with pytest.raises(Exception, match='Cleanup failed'): + cleanup() + + mock_compose_env.assert_called_once() + mock_cleanup_mirage_op.assert_called_once_with(mock_env) + mock_cleanup_docker_config.assert_not_called() + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=False) +def test_cleanup_fails_when_user_invalid( + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, +): + """Test that cleanup fails when user validation fails""" + import pytest + + from node_cli.mirage.mirage_node import cleanup + + with pytest.raises(SystemExit): + cleanup() + + +def test_cleanup_fails_when_not_inited(ensure_meta_removed): + import pytest + + with pytest.raises(SystemExit): + cleanup() + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.mirage.mirage_node.cleanup_docker_configuration') +@mock.patch('node_cli.mirage.mirage_node.cleanup_mirage_op') +@mock.patch('node_cli.mirage.mirage_node.compose_node_env') +@mock.patch('node_cli.mirage.mirage_node.logger') +def test_cleanup_logs_success_message( + mock_logger, + mock_compose_env, + mock_cleanup_mirage_op, + mock_cleanup_docker_config, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, +): + mock_env = {'ENV_TYPE': 'devnet'} + mock_compose_env.return_value = mock_env + + cleanup() + + mock_logger.info.assert_called_once_with( + 'Mirage node was cleaned up, all containers and data removed' + ) diff --git a/tests/test-env b/tests/test-env deleted file mode 100644 index 7698a8b8..00000000 --- a/tests/test-env +++ /dev/null @@ -1,15 +0,0 @@ -ENDPOINT=http://localhost:8545 -IMA_ENDPOINT=http://127.0.01 -DB_USER=user -DB_PASSWORD=pass -DB_PORT=3307 -CONTAINER_CONFIGS_STREAM='master' -FILEBEAT_HOST=127.0.0.1:3010 -SGX_SERVER_URL=http://127.0.0.1 -DISK_MOUNTPOINT=/dev/sss -DOCKER_LVMPY_STREAM='master' -ENV_TYPE='devnet' -SCHAIN_NAME='test' -ENFORCE_BTRFS=False -MANAGER_CONTRACTS='test-manager' -IMA_CONTRACTS='test-ima' \ No newline at end of file diff --git a/tests/tools_meta_test.py b/tests/tools_meta_test.py index 9a217233..3c252b9c 100644 --- a/tests/tools_meta_test.py +++ b/tests/tools_meta_test.py @@ -1,35 +1,34 @@ import json +import os from node_cli.configs import META_FILEPATH from node_cli.utils.meta import ( - CliMeta, - compose_default_meta, DEFAULT_CONFIG_STREAM, DEFAULT_VERSION, - ensure_meta, - get_meta_info, - save_meta, - update_meta, + CliMeta, + CliMetaManager, + MirageCliMeta, + MirageCliMetaManager, ) from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3 def test_get_meta_info_v1(meta_file_v1): - meta = get_meta_info() + meta = CliMetaManager().get_meta_info() assert meta.version == TEST_META_V1['version'] assert meta.config_stream == TEST_META_V1['config_stream'] assert meta.docker_lvmpy_stream == '1.0.0' def test_get_meta_info_v2(meta_file_v2): - meta = get_meta_info() + meta = CliMetaManager().get_meta_info() assert meta.version == TEST_META_V2['version'] assert meta.config_stream == TEST_META_V2['config_stream'] assert meta.docker_lvmpy_stream == TEST_META_V2['docker_lvmpy_stream'] def test_get_meta_info_v3(meta_file_v3): - meta = get_meta_info() + meta = CliMetaManager().get_meta_info() assert meta.version == TEST_META_V3['version'] assert meta.config_stream == TEST_META_V3['config_stream'] assert meta.docker_lvmpy_stream == TEST_META_V3['docker_lvmpy_stream'] @@ -38,12 +37,12 @@ def test_get_meta_info_v3(meta_file_v3): def test_get_meta_info_empty(): - meta = get_meta_info() + meta = CliMetaManager().get_meta_info() assert meta is None def test_compose_default_meta(): - meta = compose_default_meta() + meta = CliMetaManager().compose_default_meta() assert meta.version == '1.0.0' assert meta.config_stream == '1.1.0' assert meta.docker_lvmpy_stream == '1.0.0' @@ -53,7 +52,7 @@ def test_compose_default_meta(): def test_save_meta(meta_file_v2): meta = CliMeta(version='1.1.2', config_stream='2.2.2') - save_meta(meta) + CliMetaManager().save_meta(meta) with open(META_FILEPATH) as meta_f: saved_json = json.load(meta_f) assert saved_json == { @@ -66,15 +65,15 @@ def test_save_meta(meta_file_v2): def test_update_meta_from_v2_to_v3(meta_file_v2): - old_meta = get_meta_info() - update_meta( + old_meta = CliMetaManager().get_meta_info() + CliMetaManager().update_meta( version='3.3.3', config_stream='1.1.1', docker_lvmpy_stream='1.2.2', os_id='debian', os_version='11', ) - meta = get_meta_info() + meta = CliMetaManager().get_meta_info() assert meta.version == '3.3.3' assert meta.config_stream == '1.1.1' assert meta.docker_lvmpy_stream == '1.2.2' @@ -84,14 +83,14 @@ def test_update_meta_from_v2_to_v3(meta_file_v2): def test_update_meta_from_v1(meta_file_v1): - update_meta( + CliMetaManager().update_meta( version='4.4.4', config_stream='beta', docker_lvmpy_stream='1.3.3', os_id='debian', os_version='11', ) - meta = get_meta_info() + meta = CliMetaManager().get_meta_info() assert meta.version == '4.4.4' assert meta.config_stream == 'beta' assert meta.docker_lvmpy_stream == '1.3.3' @@ -100,14 +99,14 @@ def test_update_meta_from_v1(meta_file_v1): def test_update_meta_from_v3(meta_file_v3): - update_meta( + CliMetaManager().update_meta( version='5.5.5', config_stream='stable', docker_lvmpy_stream='1.2.3', os_id='ubuntu', os_version='20.04', ) - meta = get_meta_info() + meta = CliMetaManager().get_meta_info() assert meta.version == '5.5.5' assert meta.config_stream == 'stable' assert meta.docker_lvmpy_stream == '1.2.3' @@ -116,7 +115,208 @@ def test_update_meta_from_v3(meta_file_v3): def test_ensure_meta(ensure_meta_removed): - ensure_meta() - assert get_meta_info() == CliMeta(DEFAULT_VERSION, DEFAULT_CONFIG_STREAM) - ensure_meta(CliMeta(version='1.1.1', config_stream='1.1.1')) - assert get_meta_info() == CliMeta(DEFAULT_VERSION, DEFAULT_CONFIG_STREAM) + CliMetaManager().ensure_meta() + assert CliMetaManager().get_meta_info() == CliMeta(DEFAULT_VERSION, DEFAULT_CONFIG_STREAM) + CliMetaManager().ensure_meta(CliMeta(version='1.1.1', config_stream='1.1.1')) + assert CliMetaManager().get_meta_info() == CliMeta(DEFAULT_VERSION, DEFAULT_CONFIG_STREAM) + + +def test_mirage_get_meta_info_v1(meta_file_v1): + meta = MirageCliMetaManager().get_meta_info() + assert meta.version == TEST_META_V1['version'] + assert meta.config_stream == TEST_META_V1['config_stream'] + assert meta.os_id == 'ubuntu' + assert meta.os_version == '18.04' + + +def test_mirage_get_meta_info_v2(meta_file_v2): + meta = MirageCliMetaManager().get_meta_info() + assert meta.version == TEST_META_V2['version'] + assert meta.config_stream == TEST_META_V2['config_stream'] + assert meta.os_id == 'ubuntu' # default value + assert meta.os_version == '18.04' # default value + + +def test_mirage_get_meta_info_v3(meta_file_v3): + meta = MirageCliMetaManager().get_meta_info() + assert meta.version == TEST_META_V3['version'] + assert meta.config_stream == TEST_META_V3['config_stream'] + assert meta.os_id == TEST_META_V3['os_id'] + assert meta.os_version == TEST_META_V3['os_version'] + + +def test_mirage_get_meta_info_empty(): + meta = MirageCliMetaManager().get_meta_info() + assert meta is None + + +def test_mirage_compose_default_meta(): + meta = MirageCliMetaManager().compose_default_meta() + assert meta.version == '1.0.0' + assert meta.config_stream == '1.1.0' + assert meta.os_id == 'ubuntu' + assert meta.os_version == '18.04' + assert not hasattr(meta, 'docker_lvmpy_stream') + + +def test_mirage_save_meta(meta_file_v2): + meta = MirageCliMeta( + version='2.2.2', config_stream='mirage-stable', os_id='debian', os_version='11' + ) + MirageCliMetaManager().save_meta(meta) + with open(META_FILEPATH) as meta_f: + saved_json = json.load(meta_f) + assert saved_json == { + 'version': '2.2.2', + 'config_stream': 'mirage-stable', + 'os_id': 'debian', + 'os_version': '11', + } + assert 'docker_lvmpy_stream' not in saved_json + + +def test_mirage_update_meta_from_v2_to_v3(meta_file_v2): + old_meta = MirageCliMetaManager().get_meta_info() + MirageCliMetaManager().update_meta( + version='3.3.3', + config_stream='mirage-beta', + os_id='debian', + os_version='11', + ) + meta = MirageCliMetaManager().get_meta_info() + assert meta.version == '3.3.3' + assert meta.config_stream == 'mirage-beta' + assert meta.os_id == 'debian' + assert meta.os_version == '11' + assert meta != old_meta + + +def test_mirage_update_meta_from_v1(meta_file_v1): + MirageCliMetaManager().update_meta( + version='4.4.4', + config_stream='mirage-develop', + os_id='centos', + os_version='8', + ) + meta = MirageCliMetaManager().get_meta_info() + assert meta.version == '4.4.4' + assert meta.config_stream == 'mirage-develop' + assert meta.os_id == 'centos' + assert meta.os_version == '8' + + +def test_mirage_update_meta_from_v3(meta_file_v3): + MirageCliMetaManager().update_meta( + version='5.5.5', + config_stream='mirage-stable', + os_id='ubuntu', + os_version='22.04', + ) + meta = MirageCliMetaManager().get_meta_info() + assert meta.version == '5.5.5' + assert meta.config_stream == 'mirage-stable' + assert meta.os_id == 'ubuntu' + assert meta.os_version == '22.04' + + +def test_mirage_ensure_meta(ensure_meta_removed): + MirageCliMetaManager().ensure_meta() + assert MirageCliMetaManager().get_meta_info() == MirageCliMeta( + DEFAULT_VERSION, DEFAULT_CONFIG_STREAM + ) + MirageCliMetaManager().ensure_meta(MirageCliMeta(version='1.1.1', config_stream='1.1.1')) + assert MirageCliMetaManager().get_meta_info() == MirageCliMeta( + DEFAULT_VERSION, DEFAULT_CONFIG_STREAM + ) + + +def test_mirage_get_meta_info_raw(meta_file_v3): + raw_meta = MirageCliMetaManager().get_meta_info(raw=True) + assert isinstance(raw_meta, dict) + assert raw_meta['version'] == TEST_META_V3['version'] + assert raw_meta['config_stream'] == TEST_META_V3['config_stream'] + assert raw_meta['os_id'] == TEST_META_V3['os_id'] + assert raw_meta['os_version'] == TEST_META_V3['os_version'] + assert 'docker_lvmpy_stream' not in raw_meta + + +def test_mirage_get_meta_info_raw_empty(): + raw_meta = MirageCliMetaManager().get_meta_info(raw=True) + assert raw_meta == {} + + +def test_mirage_asdict(): + meta = MirageCliMeta( + version='1.2.3', config_stream='test-stream', os_id='fedora', os_version='35' + ) + meta_dict = meta.asdict() + expected = { + 'version': '1.2.3', + 'config_stream': 'test-stream', + 'os_id': 'fedora', + 'os_version': '35', + } + assert meta_dict == expected + assert 'docker_lvmpy_stream' not in meta_dict + + +def test_mirage_meta_compatibility_with_cli_meta_file(meta_file_v3): + meta = MirageCliMetaManager().get_meta_info() + assert meta.version == TEST_META_V3['version'] + assert meta.config_stream == TEST_META_V3['config_stream'] + assert meta.os_id == TEST_META_V3['os_id'] + assert meta.os_version == TEST_META_V3['os_version'] + # Should not have docker_lvmpy_stream even though it's in the file + assert not hasattr(meta, 'docker_lvmpy_stream') + + +def test_mirage_save_meta_overwrites_cli_meta(meta_file_v3): + with open(META_FILEPATH) as f: + original_data = json.load(f) + assert 'docker_lvmpy_stream' in original_data + + mirage_meta = MirageCliMeta(version='2.0.0', config_stream='mirage-new') + MirageCliMetaManager().save_meta(mirage_meta) + + with open(META_FILEPATH) as f: + saved_data = json.load(f) + assert 'docker_lvmpy_stream' not in saved_data + assert saved_data['version'] == '2.0.0' + assert saved_data['config_stream'] == 'mirage-new' + + +def test_mirage_ensure_meta_with_existing_cli_meta(meta_file_v3): + MirageCliMetaManager().ensure_meta() + meta = MirageCliMetaManager().get_meta_info() + assert meta.version == TEST_META_V3['version'] + assert meta.config_stream == TEST_META_V3['config_stream'] + + +def test_mirage_meta_defaults(): + meta = MirageCliMeta() + assert meta.version == DEFAULT_VERSION + assert meta.config_stream == DEFAULT_CONFIG_STREAM + assert meta.os_id == 'ubuntu' + assert meta.os_version == '18.04' + + +def test_mirage_meta_partial_initialization(): + meta = MirageCliMeta(version='1.5.0', os_id='alpine') + assert meta.version == '1.5.0' + assert meta.config_stream == DEFAULT_CONFIG_STREAM + assert meta.os_id == 'alpine' + assert meta.os_version == '18.04' + + +def test_mirage_update_meta_ensure_called(): + manager = MirageCliMetaManager() + + manager.update_meta(version='1.0.0', config_stream='test', os_id='ubuntu', os_version='20.04') + + meta = manager.get_meta_info() + assert meta is not None + assert meta.version == '1.0.0' + assert meta.config_stream == 'test' + + if os.path.isfile(META_FILEPATH): + os.remove(META_FILEPATH)