diff --git a/.gitignore b/.gitignore index aaed82a4..c6f6d1c9 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,7 @@ pip-wheel-metadata/ /README.rst .idea .venv - venv/ - \.python-version +monkeytype.sqlite3 +annotations.json diff --git a/.travis.yml b/.travis.yml index 0571f245..e3a09948 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_install: # - curl https://raw.githubusercontent.com/dephell/dephell/master/install.py | /opt/python/3.6/bin/python # https://github.com/travis-ci/travis-ci/issues/8589 - > - /opt/python/3.7/bin/python install.py + /opt/python/3.7.1/bin/python install.py --branch=${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH} --slug=${TRAVIS_PULL_REQUEST_SLUG:-dephell/dephell} - dephell inspect self @@ -29,7 +29,7 @@ matrix: include: - python: "3.6.7" env: ENV=pytest - - python: "3.7" + - python: "3.7.1" env: ENV=pytest - python: "3.8" env: ENV=pytest @@ -51,31 +51,37 @@ matrix: # https://github.com/igorshubovych/markdownlint-cli - name: markdown lint - language: generic + language: node_js + node_js: "14" + before_install: + - true install: - npm install -g markdownlint-cli script: - bash -c "markdownlint --config markdownlint.json **/*.md" - name: offline - python: "3.7" + python: "3.7.1" env: ENV=pytest script: - dephell venv run --env=$ENV pytest --no-network tests/ - - python: "3.7" + - python: "3.7.1" env: ENV=flake8 - - python: "3.7" + - python: "3.7.1" + env: ENV=typing + + - python: "3.7.1" env: ENV=isort script: - dephell venv run --env=$ENV flake8 - - python: "3.7" + - python: "3.7.1" env: ENV=docs - name: poetry - python: "3.7" + python: "3.7.1" env: ENV=main before_install: - pip install poetry diff --git a/README.md b/README.md index a0817956..748cdf18 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ +# THE PROJECT IS ARCHIVED + +Forks: https://github.com/orsinium/forks + +--- + # ![DepHell](./assets/logo.png) [![pypi](https://img.shields.io/pypi/v/dephell.svg)](https://pypi.python.org/pypi/dephell/) diff --git a/dephell/__init__.py b/dephell/__init__.py index 638b8b99..d5048b4b 100644 --- a/dephell/__init__.py +++ b/dephell/__init__.py @@ -13,6 +13,6 @@ """ -__version__ = '0.8.2' +__version__ = '0.8.3' __author__ = 'Gram (@orsinium)' __license__ = 'MIT' diff --git a/dephell/actions/_autocomplete.py b/dephell/actions/_autocomplete.py index d930c499..50be1a45 100644 --- a/dephell/actions/_autocomplete.py +++ b/dephell/actions/_autocomplete.py @@ -20,6 +20,7 @@ def make_bash_autocomplete() -> str: + # app from ..commands import COMMANDS template = templates.get_template('autocomplete.sh.j2') @@ -40,6 +41,7 @@ def make_bash_autocomplete() -> str: def make_zsh_autocomplete() -> str: + # app from ..commands import COMMANDS template = templates.get_template('autocomplete-zsh.sh.j2') diff --git a/dephell/actions/_contributing.py b/dephell/actions/_contributing.py index baddb5ba..e7c1b0b2 100644 --- a/dephell/actions/_contributing.py +++ b/dephell/actions/_contributing.py @@ -1,6 +1,6 @@ # built-in from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict # external from jinja2 import Environment, PackageLoader @@ -29,7 +29,7 @@ ) -def make_contributing(config: Dict[str, Dict[str, Any]], project_path: Path) -> Optional[str]: +def make_contributing(config: Dict[str, Dict[str, Any]], project_path: Path) -> str: template = env.get_template('contributing.md.j2') envs = dict() for name, category in KNOWN_SECTIONS: diff --git a/dephell/actions/_dotenv.py b/dephell/actions/_dotenv.py index 37d368b2..e8b9ba51 100644 --- a/dephell/actions/_dotenv.py +++ b/dephell/actions/_dotenv.py @@ -35,7 +35,7 @@ def read_dotenv(path: Path, env_vars: Dict[str, str] = None) -> Dict[str, str]: # clean and substitute value value = ' '.join(shlex.split(value, comments=True)) value = value.replace(r'\$', '$$') # escaping - value = decode(value, 'unicode-escape') + value = decode(value, 'unicode-escape') # type: ignore if '$' in value: value = Template(value).safe_substitute(env_vars) diff --git a/dephell/actions/_downloads.py b/dephell/actions/_downloads.py index d21cd14f..9ee4d8ba 100644 --- a/dephell/actions/_downloads.py +++ b/dephell/actions/_downloads.py @@ -2,7 +2,7 @@ from collections import defaultdict from datetime import date, timedelta from itertools import zip_longest -from typing import Dict, Iterable, Iterator, List +from typing import Any, DefaultDict, Dict, Iterator, List, Sequence # external import attr @@ -20,9 +20,9 @@ @attr.s() class DateList: - start = attr.ib() - end = attr.ib() - _data = attr.ib(factory=dict, repr=False) + start = attr.ib(type=date) + end = attr.ib(type=date) + _data = attr.ib(factory=dict, repr=False, type=Dict[str, int]) def add(self, date: str, value: int): self._data[date] = value @@ -34,7 +34,7 @@ def __iter__(self) -> Iterator[int]: moment += timedelta(1) -def make_chart(values: Iterable[int], group: int = None, ticks: str = '_▁▂▃▄▅▆▇█') -> str: +def make_chart(values: Sequence[int], group: int = None, ticks: str = '_▁▂▃▄▅▆▇█') -> str: peek = max(values) if peek == 0: chart = ticks[-1] * len(values) @@ -62,7 +62,7 @@ def get_total_downloads(name: str) -> Dict[str, int]: ) -def get_downloads_by_category(*, category: str, name: str) -> List[Dict[str, int]]: +def get_downloads_by_category(*, category: str, name: str) -> List[Dict[str, Any]]: url = CATEGORIES_URLS[category].format(name) with requests_session() as session: response = session.get(url) @@ -70,14 +70,15 @@ def get_downloads_by_category(*, category: str, name: str) -> List[Dict[str, int body = response.json()['data'] yesterday = date.today() - timedelta(1) + grouped: DefaultDict[str, DateList] grouped = defaultdict(lambda: DateList(start=yesterday - timedelta(30), end=yesterday)) for line in body: category = line['category'].replace('.', '') grouped[category].add(date=line['date'], value=line['downloads']) result = [] - for category, downloads in grouped.items(): - downloads = list(downloads) + for category, dates in grouped.items(): + downloads = list(dates) if sum(downloads) == 0: continue result.append(dict( diff --git a/dephell/actions/_entrypoints.py b/dephell/actions/_entrypoints.py index ec1bea23..fd255406 100644 --- a/dephell/actions/_entrypoints.py +++ b/dephell/actions/_entrypoints.py @@ -1,5 +1,6 @@ # built-in from logging import getLogger +from pathlib import Path from typing import Iterable, Optional, Tuple # external @@ -15,7 +16,7 @@ logger = getLogger('dephell.actions') -def _get_matching_path(paths: Iterable, name: str) -> Optional[str]: +def _get_matching_path(paths: Iterable[Path], name: str) -> Optional[Path]: name = canonicalize_name(name) for path in paths: package_name = path.stem.split('-')[0] @@ -55,7 +56,7 @@ def get_entrypoints(*, venv: VEnv, name: str) -> Optional[Tuple[EntryPoint, ...] canonicalize_name(name).replace('_', '-'), canonicalize_name(name).replace('-', '').replace('_', ''), } - paths = (venv.bin_path / name for name in names) + paths = tuple(venv.bin_path / name for name in names) if IS_WINDOWS: paths = tuple(p.with_suffix('.exe') for p in paths) diff --git a/dephell/actions/_json.py b/dephell/actions/_json.py index 65a765f8..5627010d 100644 --- a/dephell/actions/_json.py +++ b/dephell/actions/_json.py @@ -2,7 +2,7 @@ import json from collections import defaultdict from functools import reduce -from typing import Optional +from typing import Any, Dict, Optional # app from ..imports import lazy_import @@ -124,19 +124,19 @@ def _beautify(data, *, colors: bool, table: bool) -> str: ) # list of dicts if isinstance(data, list) and data and isinstance(data[0], dict): - table = [] + result = [] for row in data: row = _flatdict(row) keys = tuple(row) row = [v for _, v in sorted(row.items())] - table.append(row) + result.append(row) return tabulate.tabulate( - table, + result, headers=keys, tablefmt='fancy_grid', ) - json_params = dict(indent=2, sort_keys=True, ensure_ascii=False) + json_params = dict(indent=2, sort_keys=True, ensure_ascii=False) # type: Dict[str, Any] dumped = json.dumps(data, **json_params) if not colors: return dumped diff --git a/dephell/actions/_package.py b/dephell/actions/_package.py index 6bf151b5..541ee1f9 100644 --- a/dephell/actions/_package.py +++ b/dephell/actions/_package.py @@ -21,7 +21,6 @@ def get_package(req: str, repo: str = None) -> Dependency: return dep -def get_resolver(reqs: Iterable[str] = None) -> Resolver: - root = PIPConverter(lock=False).loads_resolver('\n'.join(reqs)) - root.name = 'root' - return root +def get_resolver(reqs: Iterable[str]) -> Resolver: + resolver = PIPConverter(lock=False).loads_resolver('\n'.join(reqs)) + return resolver diff --git a/dephell/actions/_python.py b/dephell/actions/_python.py index 9ba0724a..b557dfa8 100644 --- a/dephell/actions/_python.py +++ b/dephell/actions/_python.py @@ -1,7 +1,7 @@ # built-in import subprocess from pathlib import Path -from typing import Optional +from typing import Optional, Union # external from dephell_pythons import Python, Pythons @@ -62,14 +62,14 @@ def get_lib_path(python_path: Path) -> Optional[Path]: """Find site-packages or dist-packages dir for the given python """ # get user site dir path - user_site = None + user_site = None # type: Union[None, Path] cmd = [str(python_path), '-c', r'print(__import__("site").USER_SITE)'] result = subprocess.run(cmd, stdout=subprocess.PIPE) if result.returncode == 0: - user_site = result.stdout.decode().strip() - if user_site: - user_site = Path(user_site) - if not user_site.exists(): + line = result.stdout.decode().strip() + if line: + user_site = Path(line) + if user_site and not user_site.exists(): user_site = None # get sys.path paths @@ -78,8 +78,8 @@ def get_lib_path(python_path: Path) -> Optional[Path]: if result.returncode != 0: return None paths = [] - for path in result.stdout.decode().splitlines(): - path = Path(path) + for line in result.stdout.decode().splitlines(): + path = Path(line) if not path.exists(): continue paths.append(path) diff --git a/dephell/actions/_transform.py b/dephell/actions/_transform.py index 4e9f127d..fb500c11 100644 --- a/dephell/actions/_transform.py +++ b/dephell/actions/_transform.py @@ -11,16 +11,14 @@ if TYPE_CHECKING: - import bowler.helpers as bowler_helpers - import fissix.pytree as fissix_pytree + # external from bowler import LN, Capture, Filename, Query - from fissix import fixer_util modifiers = [] -def _register(modifier): +def _register(modifier: type) -> type: modifiers.append(modifier) return modifier @@ -54,7 +52,7 @@ class ModuleImportModifier: > """ - def __init__(self, old_name, new_name): + def __init__(self, old_name: str, new_name: str) -> None: self.old_name = old_name self.new_name = new_name @@ -91,7 +89,7 @@ class FromImportModifier: > """ - def __init__(self, old_name, new_name): + def __init__(self, old_name: str, new_name: str) -> None: self.old_name = old_name self.new_name = new_name @@ -138,7 +136,7 @@ class ModuleAsImportModifier: > """ - def __init__(self, old_name, new_name): + def __init__(self, old_name: str, new_name: str) -> None: self.old_name = old_name self.new_name = new_name @@ -161,7 +159,7 @@ class StringModifier: string=STRING """ - def __init__(self, old_name, new_name): + def __init__(self, old_name: str, new_name: str) -> None: self.old_name = old_name self.new_name = new_name @@ -198,7 +196,7 @@ class DottedModuleImportModifier: ) """ - def __init__(self, old_name, new_name): + def __init__(self, old_name: str, new_name: str) -> None: self.old_name = old_name self.new_name = new_name @@ -208,7 +206,7 @@ def __call__(self, node: 'LN', capture: 'Capture', filename: 'Filename') -> None else: self._modify_import(capture) - def _modify_import(self, capture): + def _modify_import(self, capture) -> None: new_name_node = build_new_name_node( old_node=capture['module_name'], new_name=self.new_name, @@ -217,7 +215,7 @@ def _modify_import(self, capture): ) capture['module_name'].replace(new_name_node) - def _modify_power(self, node): + def _modify_power(self, node: 'LN') -> None: prefix = node.children[0].prefix # remove old prefix diff --git a/dephell/cache.py b/dephell/cache.py index 6fda56e9..14806cb0 100644 --- a/dephell/cache.py +++ b/dephell/cache.py @@ -3,20 +3,26 @@ import pickle from pathlib import Path from time import time -from typing import List +from typing import TYPE_CHECKING, Any, List, Optional, Union # app from .cached_property import cached_property from .config import config +if TYPE_CHECKING: + # app + from .converters.pip import PIPConverter + from .models.dependency import Dependency # noqa: F401 + + class BaseCache: ext = '' def __init__(self, *keys, ttl: int = -1): self.path = Path(config['cache']['path'], *keys) if self.ext: - self.path = self.path.with_suffix(self.ext) + self.path = self.path.with_name(self.path.name + self.ext) self.ttl = ttl self._check_ttl() @@ -53,7 +59,7 @@ def dump(self, data) -> None: class TextCache(BaseCache): ext = '.txt' - def load(self): + def load(self) -> Optional[Any]: if not self.path.exists(): return None with self.path.open('r') as stream: @@ -68,7 +74,7 @@ def dump(self, data: List[str]) -> None: class JSONCache(BaseCache): ext = '.json' - def load(self): + def load(self) -> Optional[Any]: if not self.path.exists(): return None with self.path.open('r') as stream: @@ -78,7 +84,7 @@ def load(self): return None return None - def dump(self, data): + def dump(self, data: Union[list, dict]) -> None: self.path.parent.mkdir(parents=True, exist_ok=True) with self.path.open('w') as stream: json.dump(data, stream) @@ -88,18 +94,20 @@ class RequirementsCache(BaseCache): ext = '.txt' @cached_property - def converter(self): + def converter(self) -> 'PIPConverter': + # app from .converters import PIPConverter return PIPConverter(lock=False) - def load(self): + def load(self) -> Optional[List['Dependency']]: if not self.path.exists(): return None root = self.converter.load(self.path) return root.dependencies def dump(self, root): + # app from .controllers import Graph from .models import Requirement diff --git a/dephell/cached_property.pyi b/dephell/cached_property.pyi new file mode 100644 index 00000000..452d77d4 --- /dev/null +++ b/dephell/cached_property.pyi @@ -0,0 +1,27 @@ +# built-in +from typing import Any, Callable, Generic, Optional, Type, TypeVar, overload + + +_AnyCallable = Callable[..., Any] + +_T = TypeVar("_T") +_S = TypeVar("_S") + + +# https://github.com/python/typeshed/blob/master/stdlib/3/functools.pyi +class cached_property(Generic[_S, _T]): + func: Callable[[_S], _T] + + def __init__(self, func: Callable[[_S], _T]) -> None: + ... + + @overload + def __get__(self, instance: None, owner: Optional[Type[_S]] = ...) -> 'cached_property[_S, _T]': + ... + + @overload + def __get__(self, instance: _S, owner: Optional[Type[_S]] = ...) -> _T: + ... + + def __set_name__(self, owner: Type[_S], name: str) -> None: + ... diff --git a/dephell/commands/deps_add.py b/dephell/commands/deps_add.py index 9604b509..e3d7dde3 100644 --- a/dephell/commands/deps_add.py +++ b/dephell/commands/deps_add.py @@ -38,6 +38,7 @@ def __call__(self) -> bool: # get new deps new_resolver = get_resolver(reqs=self.args.name) new_root = new_resolver.graph._roots[0] + new_root.name = 'new-root' # set envs for dep in new_root.dependencies: diff --git a/dephell/commands/deps_check.py b/dephell/commands/deps_check.py index 5ea595a4..b1d38aee 100644 --- a/dephell/commands/deps_check.py +++ b/dephell/commands/deps_check.py @@ -42,6 +42,8 @@ def __call__(self) -> bool: data = [] reqs = Requirement.from_graph(graph=resolver.graph, lock=True) for req in reqs: + if not req.version: + continue version = req.version.strip('=') # not installed if req.name not in installed: diff --git a/dephell/commands/deps_convert.py b/dephell/commands/deps_convert.py index fd36bed6..d0711492 100644 --- a/dephell/commands/deps_convert.py +++ b/dephell/commands/deps_convert.py @@ -1,6 +1,7 @@ # built-in from argparse import ArgumentParser from pathlib import Path +from typing import Any, Dict # app from ..actions import attach_deps @@ -66,15 +67,14 @@ def __call__(self) -> bool: # filter out deps by `--envs` if self.config.get('envs'): - if len(resolver.graph._layers) == 1: - for root in resolver.graph._roots: - for dep in root.dependencies: - dep.applied = True - resolver.graph.add(dep) - for root in resolver.graph._roots: - root.applied = True - resolver.apply_envs(set(self.config['envs'])) - resolver.graph._layers = resolver.graph._layers[:1] + if not should_be_resolved: + resolver.graph.fast_apply() + resolver.apply_envs(envs=set(self.config['envs']), deep=should_be_resolved) + # If it's not a lockfile, we should drop all dependencies except direct ones. + # While `fast_apply` doesn't produce such dependencies, we want to be sure + # that we can't ever have them in the output. Trust no one. + if not should_be_resolved: + resolver.graph._layers = resolver.graph._layers[:2] # dump self.logger.debug('dump dependencies...', extra=dict( @@ -82,10 +82,10 @@ def __call__(self) -> bool: path=self.config['to']['path'], )) - dumper_kwargs = { - 'reqs': Requirement.from_graph(resolver.graph, lock=dumper.lock), - 'project': resolver.graph.metainfo, - } + dumper_kwargs = dict( + reqs=Requirement.from_graph(resolver.graph, lock=dumper.lock), + project=resolver.graph.metainfo, + ) # type: Dict[str, Any] if self.config['to']['path'] == 'stdout': print(dumper.dumps(**dumper_kwargs)) else: diff --git a/dephell/commands/deps_install.py b/dephell/commands/deps_install.py index e84b771e..33618c61 100644 --- a/dephell/commands/deps_install.py +++ b/dephell/commands/deps_install.py @@ -82,6 +82,8 @@ def _get_install_remove(self, graph, python) -> Tuple[list, list]: install.append(req) continue # installed the same version, skip + if not req.version: + continue version = req.version.strip('=') if version in installed[req.name]: continue @@ -97,6 +99,6 @@ def _get_install_remove(self, graph, python) -> Tuple[list, list]: # remove packages not in graph if self.sync: names = set(installed) - {req.name for req in reqs} - remove.extend(SimpleNamespace(name=name) for name in names) + remove.extend(SimpleNamespace(name=name) for name in names) # type: ignore return install, remove diff --git a/dephell/commands/deps_tree.py b/dephell/commands/deps_tree.py index 2e24ad7a..dcf9cd4c 100644 --- a/dephell/commands/deps_tree.py +++ b/dephell/commands/deps_tree.py @@ -35,7 +35,10 @@ def __call__(self) -> bool: if self.args.type == 'pretty': for dep in sorted(resolver.graph.get_layer(1)): - print('\n'.join(self._make_tree(dep))) + if not dep.applied: + continue + content = '\n'.join(self._make_tree(dep)) + print(self._colorize(content)) return True if self.args.type == 'json': @@ -61,16 +64,40 @@ def __call__(self) -> bool: self.logger.info('graph saved into .dephell_report/') return True + raise RuntimeError('unreachable') + @classmethod def _make_tree(cls, dep, *, level: int = 0) -> List[str]: - lines = ['{level}- {name} [required: {constraint}, locked: {best}, latest: {latest}]'.format( + best = dep.group.best_release.version + latest = dep.group.best_release.version + if best == latest: + template = '{level}- {name} [required: `{constraint}`, latest: `{latest}`]' + else: + template = '{level}- {name} [required: `{constraint}`, locked: `{best}`, latest: `{latest}`]' + lines = [template.format( level=' ' * level, name=dep.name, constraint=str(dep.constraint) or '*', - best=str(dep.group.best_release.version), - latest=str(dep.groups.releases[0].version), + best=str(best), + latest=str(latest), )] deps = {dep.name: dep for dep in dep.dependencies}.values() # drop duplicates for subdep in sorted(deps): lines.extend(cls._make_tree(subdep, level=level + 1)) return lines + + @staticmethod + def _colorize(content: str) -> str: + try: + # external + import pygments + import pygments.formatters + import pygments.lexers + except ImportError: + return content + content = pygments.highlight( + code=content, + lexer=pygments.lexers.MarkdownLexer(), + formatter=pygments.formatters.TerminalFormatter(), + ) + return content.strip() diff --git a/dephell/commands/generate_config.py b/dephell/commands/generate_config.py index 0d0898b9..3768c09a 100644 --- a/dephell/commands/generate_config.py +++ b/dephell/commands/generate_config.py @@ -5,6 +5,7 @@ # external import tomlkit +from tomlkit.items import Table # app from ..cached_property import cached_property @@ -33,11 +34,11 @@ def config(self) -> Config: config._data['config'] = 'pyproject.toml' return config - def validate(self): + def validate(self) -> bool: return True @staticmethod - def _make_env(from_format, from_path, to_format, to_path): + def _make_env(from_format: str, from_path: str, to_format: str, to_path: str) -> Table: table = tomlkit.table() table['from'] = tomlkit.inline_table() diff --git a/dephell/commands/inspect_project.py b/dephell/commands/inspect_project.py index b5fa79ce..c53ce71b 100644 --- a/dephell/commands/inspect_project.py +++ b/dephell/commands/inspect_project.py @@ -1,6 +1,7 @@ # built-in from argparse import ArgumentParser from pathlib import Path +from typing import Any, Dict # app from ..actions import make_json @@ -34,7 +35,7 @@ def __call__(self) -> bool: name=root.raw_name, version=root.version, description=root.description, - ) + ) # type: Dict[str, Any] if root.python: result['python'] = str(root.python) if root.links: diff --git a/dephell/commands/jail_try.py b/dephell/commands/jail_try.py index 1ae366c7..9e878f60 100644 --- a/dephell/commands/jail_try.py +++ b/dephell/commands/jail_try.py @@ -41,7 +41,7 @@ def __call__(self) -> bool: if isinstance(command, str): command = shlex.split(command) - with TemporaryDirectory() as base_path: + with TemporaryDirectory() as base_path: # type: Path # type: ignore base_path = Path(base_path) # make venv diff --git a/dephell/commands/package_show.py b/dephell/commands/package_show.py index cf2bb31f..4c3a30f5 100644 --- a/dephell/commands/package_show.py +++ b/dephell/commands/package_show.py @@ -1,5 +1,7 @@ # built-in from argparse import ArgumentParser +from pathlib import Path +from typing import List # app from ..actions import format_size, get_package, get_path_size, get_python_env, make_json @@ -30,7 +32,7 @@ def __call__(self) -> bool: root = InstalledConverter().load(paths=python.lib_paths, names={self.args.name}) local_versions = [] - local_places = [] + local_places = [] # type: List[Path] for subdep in root.dependencies: if subdep.name == dep.name: local_versions = str(subdep.constraint).replace('=', '').split(' || ') diff --git a/dephell/commands/project_register.py b/dephell/commands/project_register.py index 4ef2cf88..823e41c9 100644 --- a/dephell/commands/project_register.py +++ b/dephell/commands/project_register.py @@ -69,6 +69,7 @@ def __call__(self) -> bool: return True def _register(self, project_path: Path, lib_path: Path = None) -> bool: + project_path = project_path.resolve() self._make_egg_info( project_path=project_path, from_format=self.config['from']['format'], diff --git a/dephell/commands/project_test.py b/dephell/commands/project_test.py index 607555ff..d68a89c1 100644 --- a/dephell/commands/project_test.py +++ b/dephell/commands/project_test.py @@ -5,9 +5,10 @@ from argparse import REMAINDER, ArgumentParser from pathlib import Path from tempfile import TemporaryDirectory +from typing import Dict, Tuple # external -from dephell_pythons import Pythons +from dephell_pythons import Python, Pythons from dephell_venvs import VEnv # app @@ -70,25 +71,26 @@ def __call__(self) -> bool: # choose pythons self.logger.info('get interpreters') pythons = Pythons() + choosen_pythons: Tuple[Python, ...] if 'python' in self.config: # get from config choosen_pythons = (pythons.get_best(self.config['python']), ) else: # get from project - choosen_pythons = dict() - pyrhon_constraint = resolver.graph.metainfo.python + pythons_by_version = dict() # type: Dict[str, Python] + python_constraint = resolver.graph.metainfo.python for python in pythons: version = str(python.get_short_version()) - if version in choosen_pythons: + if version in pythons_by_version: continue - if python.version not in pyrhon_constraint: + if python.version not in python_constraint: continue - choosen_pythons[version] = python - choosen_pythons = tuple(choosen_pythons.values()) + pythons_by_version[version] = python + choosen_pythons = tuple(pythons_by_version.values()) for python in choosen_pythons: - with TemporaryDirectory() as path: - root_path = Path(path) + with TemporaryDirectory() as root_path: # type: Path # type: ignore + root_path = Path(root_path) # make venv self.logger.info('create venv', extra=dict(python=str(python.version))) @@ -96,7 +98,7 @@ def __call__(self) -> bool: venv.create(python_path=python.path) # copy tests - for path in self.config['tests']: + for path in self.config['tests']: # type: Path self.logger.info('copy files', extra=dict(path=path)) path = Path(path) if not path.exists(): diff --git a/dephell/commands/vendor_download.py b/dephell/commands/vendor_download.py index 14e65bab..0b993124 100644 --- a/dephell/commands/vendor_download.py +++ b/dephell/commands/vendor_download.py @@ -45,7 +45,7 @@ def __call__(self) -> bool: def _download_packages(self, resolver, output_path: Path, exclude: Iterable[str] = None) -> int: exclude = set(exclude or []) - with TemporaryDirectory() as archives_path: + with TemporaryDirectory() as archives_path: # type: Path # type: ignore archives_path = Path(archives_path) loop = asyncio.get_event_loop() @@ -74,14 +74,14 @@ def _download_packages(self, resolver, output_path: Path, def _extract_modules(self, dep, archive_path: Path, output_path: Path) -> bool: # say to shutils that wheel can be parsed as zip - if 'wheel' not in shutil._UNPACK_FORMATS: + if 'wheel' not in shutil._UNPACK_FORMATS: # type: ignore shutil.register_unpack_format( name='wheel', extensions=['.whl'], - function=shutil._unpack_zipfile, + function=shutil._unpack_zipfile, # type: ignore ) - with TemporaryDirectory(suffix=dep.name) as package_path: + with TemporaryDirectory(suffix=dep.name) as package_path: # type: Path # type: ignore package_path = Path(package_path) shutil.unpack_archive(str(archive_path), str(package_path)) if len(list(package_path.iterdir())) == 1: diff --git a/dephell/commands/vendor_import.py b/dephell/commands/vendor_import.py index 7cf683b2..28fa352a 100644 --- a/dephell/commands/vendor_import.py +++ b/dephell/commands/vendor_import.py @@ -1,7 +1,6 @@ # built-in from argparse import ArgumentParser from pathlib import Path -from typing import TYPE_CHECKING # app from ..actions import transform_imports @@ -12,9 +11,6 @@ bowler = lazy_import('bowler') -if TYPE_CHECKING: - import bowler - class VendorImportCommand(BaseCommand): """Patch all imports in project to use vendored dependencies. diff --git a/dephell/commands/venv_run.py b/dephell/commands/venv_run.py index d3adf082..47f9c587 100644 --- a/dephell/commands/venv_run.py +++ b/dephell/commands/venv_run.py @@ -59,13 +59,13 @@ def __call__(self) -> bool: self.logger.warning('executable is not found in venv, trying to install...', extra=dict( executable=command[0], )) - result = install_dep( + installed = install_dep( name=command[0], python_path=venv.python_path, logger=self.logger, silent=self.config['silent'], ) - if not result: + if not installed: return False if not executable.exists(): self.logger.error('package installed, but executable is not found') diff --git a/dephell/config/builders.py b/dephell/config/builders.py index 1339eec9..e5b4c590 100644 --- a/dephell/config/builders.py +++ b/dephell/config/builders.py @@ -1,11 +1,19 @@ -# project +# built-in +from pathlib import Path + # external +from dephell_argparse._parser import Parser from dephell_versioning import get_schemes # app from ..constants import FORMATS, LOG_FORMATTERS, LOG_LEVELS, REPOSITORIES, STRATEGIES +# helper function for path values +def expanded_path(string): + return Path(string).expanduser().resolve().as_posix() + + env_help = ( 'Pipenv has 2 envs in same file: main and dev. ' 'For poetry you can also use main-opt and dev-opt ' @@ -14,45 +22,45 @@ ) -def build_config(parser): +def build_config(parser: Parser) -> None: config_group = parser.add_argument_group('Configuration file') - config_group.add_argument('-c', '--config', help='path to config file.') + config_group.add_argument('-c', '--config', help='path to config file.', type=expanded_path) config_group.add_argument('-e', '--env', help='environment in config.') -def build_from(parser): +def build_from(parser: Parser) -> None: from_group = parser.add_argument_group('Input file') - from_group.add_argument('--from', help='path or format for reading requirements.') + from_group.add_argument('--from', help='path or format for reading requirements.', type=expanded_path) from_group.add_argument('--from-format', choices=FORMATS, help='format for reading requirements.') - from_group.add_argument('--from-path', help='path to input file.') + from_group.add_argument('--from-path', help='path to input file.', type=expanded_path) -def build_to(parser): +def build_to(parser) -> None: to_group = parser.add_argument_group('Output file') - to_group.add_argument('--to', help='path or format for writing requirements.') + to_group.add_argument('--to', help='path or format for writing requirements.', type=expanded_path) to_group.add_argument('--to-format', choices=FORMATS, help='output requirements file format.') - to_group.add_argument('--to-path', help='path to output file.') + to_group.add_argument('--to-path', help='path to output file.', type=expanded_path) to_group.add_argument( '--sdist-ratio', help='ratio of tests and project size after which tests will be excluded from sdist.', ) -def build_resolver(parser): +def build_resolver(parser: Parser) -> None: resolver_group = parser.add_argument_group('Resolver rules') resolver_group.add_argument('--strategy', choices=STRATEGIES, help='Algorithm to select best release.') resolver_group.add_argument('--prereleases', action='store_true', help='Allow prereleases') resolver_group.add_argument('--mutations', type=int, help='Maximum mutations limit') -def build_api(parser): +def build_api(parser: Parser) -> None: api_group = parser.add_argument_group('APIs endpoints') api_group.add_argument('--warehouse', nargs='*', help='warehouse API URL.') api_group.add_argument('--bitbucket', help='bitbucket API URL.') api_group.add_argument('--repo', choices=REPOSITORIES, help='force repository for first-level deps.') -def build_output(parser): +def build_output(parser: Parser) -> None: output_group = parser.add_argument_group('Console output') output_group.add_argument('--format', choices=LOG_FORMATTERS, help='output format.') output_group.add_argument('--level', choices=LOG_LEVELS, help='minimal level for log messages.') @@ -66,11 +74,11 @@ def build_output(parser): output_group.add_argument('--pdb', action='store_true', help='run pdb for critical exceptions.') -def build_venv(parser): +def build_venv(parser: Parser) -> None: venv_group = parser.add_argument_group('Virtual environment') - venv_group.add_argument('--venv', help='path to venv directory for project.') + venv_group.add_argument('--venv', help='path to venv directory for project.', type=expanded_path) venv_group.add_argument('--python', help='python version for venv.') - venv_group.add_argument('--dotenv', help='path to .env file') + venv_group.add_argument('--dotenv', help='path to .env file', type=expanded_path) def build_docker(parser): @@ -80,16 +88,17 @@ def build_docker(parser): docker_group.add_argument('--docker-container', help='container name') -def build_other(parser): +def build_other(parser: Parser) -> None: other_group = parser.add_argument_group('Other') - other_group.add_argument('--cache-path', help='path to dephell cache') + other_group.add_argument('--cache-path', help='path to dephell cache', type=expanded_path) other_group.add_argument('--cache-ttl', type=int, help='Time to live for releases list cache') - other_group.add_argument('--project', help='path to the current project') - other_group.add_argument('--bin', help='path to the dir for installing scripts') + other_group.add_argument('--project', help='path to the current project', type=expanded_path) + other_group.add_argument('--bin', help='path to the dir for installing scripts', type=expanded_path) + other_group.add_argument('--ca', help='path to CA_BUNDLE file for SSL verification.', type=expanded_path) other_group.add_argument('--envs', nargs='*', help='environments (main, dev) or extras to install') - other_group.add_argument('--tests', nargs='*', help='paths to test files') + other_group.add_argument('--tests', nargs='*', help='paths to test files', type=expanded_path) other_group.add_argument('--versioning', choices=sorted(get_schemes()), help='versioning scheme for project') diff --git a/dephell/config/logging_config.py b/dephell/config/logging_config.py index e51193ef..107c3203 100644 --- a/dephell/config/logging_config.py +++ b/dephell/config/logging_config.py @@ -1,4 +1,8 @@ -LOGGING = { +# built-in +from typing import Any, Dict + + +LOGGING: Dict[str, Any] = { 'version': 1, 'disable_existing_loggers': False, 'filters': { diff --git a/dephell/config/manager.py b/dephell/config/manager.py index 9ee1a89f..710c8e91 100644 --- a/dephell/config/manager.py +++ b/dephell/config/manager.py @@ -8,7 +8,7 @@ from logging.config import dictConfig from os import environ from pathlib import Path -from typing import Dict, Optional +from typing import Any, DefaultDict, Dict, Optional # external import tomlkit @@ -36,12 +36,12 @@ class Config: def __init__(self, data: Optional[dict] = None): self._data = data or deepcopy(DEFAULT) - def setup_logging(self, data: Optional[dict] = None) -> None: + def setup_logging(self, data: Optional[Dict[str, Any]] = None) -> None: captureWarnings(True) if data is None: data = LOGGING if self._data: - data['loggers']['dephell']['level'] = self['level'] + data['loggers']['dephell']['level'] = self._data['level'] for formatter in data['formatters'].values(): formatter['colors'] = not self['nocolors'] formatter['traceback'] = self['traceback'] @@ -49,7 +49,11 @@ def setup_logging(self, data: Optional[dict] = None) -> None: handler['formatter'] = self['format'] dictConfig(LOGGING) - def attach(self, data: dict, container: Optional[dict] = None) -> None: + def attach(self, data: Dict[str, Any], container: Optional[Dict[str, Any]] = None) -> None: + """Merge `data` into `container`. + + If no container specified, merge into the current config. + """ if container is None: container = self._data @@ -70,15 +74,24 @@ def attach(self, data: dict, container: Optional[dict] = None) -> None: new_value.append(subvalue) value = new_value + # add a new value into the container if key not in container: container[key] = value - elif isinstance(value, dict): - self.attach(data=value, container=container[key]) - else: - container[key] = value + continue + + # merge dicts + if isinstance(value, dict): + subcontainer = container[key] + if isinstance(subcontainer, dict): + self.attach(data=value, container=subcontainer) + continue + + # overwrite the current value + container[key] = value @staticmethod def _expand_converter(text: str) -> Dict[str, str]: + # app from ..converters import CONVERTERS # passed converter that doesn't require path @@ -128,8 +141,8 @@ def attach_file(self, path: str, env: str, silent: bool = False) -> Optional[dic self.env = env return data - def attach_cli(self, args, sep: str = '_') -> dict: - data = defaultdict(dict) + def attach_cli(self, args, sep: str = '_') -> Dict[str, Any]: + data = defaultdict(dict) # type: DefaultDict[str, Any] for name, value in args._get_kwargs(): if value is None or value is False: continue @@ -144,8 +157,10 @@ def attach_cli(self, args, sep: str = '_') -> dict: self.attach(data) return dict(data) - def attach_env_vars(self, *, env_vars: Dict[str, str] = environ, sep: str = '_') -> dict: - data = defaultdict(dict) + def attach_env_vars(self, *, env_vars: Dict[str, str] = None, sep: str = '_') -> dict: + if env_vars is None: + env_vars = dict(environ) + data = defaultdict(dict) # type: DefaultDict[str, Any] for name, value in env_vars.items(): # drop templated part from name match = ENV_VAR_REX.fullmatch(name) @@ -183,7 +198,7 @@ def validate(self) -> bool: def format_errors(self) -> str: return json.dumps(self.errors, indent=2, sort_keys=True) - def __getattr__(self, name): + def __getattr__(self, name: str): return getattr(self._data, name) def __getitem__(self, name: str): diff --git a/dephell/config/scheme.py b/dephell/config/scheme.py index acfb0946..3f029546 100644 --- a/dephell/config/scheme.py +++ b/dephell/config/scheme.py @@ -1,4 +1,3 @@ -# project # external from dephell_versioning import get_schemes @@ -115,11 +114,18 @@ ), 'project': dict(type='string', required=True), 'bin': dict(type='string', required=True), + 'ca': dict(type='string', required=False), 'envs': dict(type='list', schema=dict(type='string'), required=False, empty=False), 'tests': dict(type='list', schema=dict(type='string'), required=True), 'versioning': dict(type='string', required=True, allowed=get_schemes()), - 'command': dict(type='string', required=False), - 'vendor': dict( + 'command': dict( + anyof=[ + dict(type='string'), + dict(type='list', schema=dict(type='string')), + ], + required=False, + ), + 'vendor': dict( type='dict', required=True, schema={ diff --git a/dephell/context_tools.py b/dephell/context_tools.py index f44b36e1..ae12fbe6 100644 --- a/dephell/context_tools.py +++ b/dephell/context_tools.py @@ -2,11 +2,12 @@ import os import sys from contextlib import contextmanager -from typing import Dict +from pathlib import Path +from typing import Dict, Iterator, Union @contextmanager -def chdir(path): +def chdir(path: Union[Path, str]) -> Iterator: """Context manager for changing dir and restoring previous workdir after exit. """ curdir = os.getcwd() diff --git a/dephell/controllers/__init__.py b/dephell/controllers/__init__.py index 6367fee0..6aff1565 100644 --- a/dephell/controllers/__init__.py +++ b/dephell/controllers/__init__.py @@ -8,7 +8,6 @@ from ._repos import RepositoriesRegistry from ._resolver import Resolver from ._safety import Safety, SafetyVulnInfo -from ._snyk import Snyk, SnykVulnInfo from ._uploader import Uploader @@ -24,7 +23,5 @@ 'Resolver', 'Safety', 'SafetyVulnInfo', - 'Snyk', - 'SnykVulnInfo', 'Uploader', ] diff --git a/dephell/controllers/_conflict.py b/dephell/controllers/_conflict.py index b98a149b..aa66ae5e 100644 --- a/dephell/controllers/_conflict.py +++ b/dephell/controllers/_conflict.py @@ -1,6 +1,7 @@ # built-in import re from logging import getLogger +from typing import Tuple # external from jinja2 import Environment, PackageLoader @@ -47,6 +48,7 @@ def analyze_conflict(resolver, suffix: str = '') -> str: logger.warning(e.args[0]) conflict = resolver.graph.conflict + templates: Tuple[str, ...] if conflict is None: templates = ('state.html.j2', ) elif not resolver.graph.conflict.python_compat: diff --git a/dephell/controllers/_docker.py b/dephell/controllers/_docker.py index b36636fa..824c7c70 100644 --- a/dephell/controllers/_docker.py +++ b/dephell/controllers/_docker.py @@ -13,14 +13,16 @@ from ..networking import requests_session -docker = lazy_import('docker') -dockerpty = lazy_import('dockerpty') - if TYPE_CHECKING: + # external import docker import dockerpty +docker = lazy_import('docker') # noqa: F811 +dockerpty = lazy_import('dockerpty') # noqa: F811 + + DOCKER_PREFIX = 'dephell-' @@ -141,14 +143,18 @@ def create(self, *, pull=True) -> None: ) def stop(self) -> None: - self.container.stop() + if self.container: + self.container.stop() def remove(self) -> None: - self.stop() - self.container.remove(force=True) - for container in self.network.containers: - self.network.disconnect(container, force=True) - self.network.remove() + if self.container: + self.stop() + self.container.remove(force=True) + + if self.network: + for container in self.network.containers: + self.network.disconnect(container, force=True) + self.network.remove() def exists(self) -> bool: if 'container' in self.__dict__: @@ -161,6 +167,8 @@ def activate(self) -> None: def run(self, command: List[str]) -> None: if dockerpty is None: raise RuntimeError('cannot run Docker on Windows') + if not self.container: + raise RuntimeError('cannot find container') self.container.start() dockerpty.exec_command( client=self.client.api, diff --git a/dephell/controllers/_graph.py b/dephell/controllers/_graph.py index 322e2cee..93e45219 100644 --- a/dephell/controllers/_graph.py +++ b/dephell/controllers/_graph.py @@ -1,7 +1,7 @@ # built-in from collections import ChainMap from logging import getLogger -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional # app from ..imports import lazy_import @@ -9,19 +9,22 @@ from ..models.root import RootDependency -graphviz = lazy_import('graphviz') -graphviz_backend = lazy_import('graphviz.backend') - - if TYPE_CHECKING: + # external import graphviz import graphviz.backend as graphviz_backend +graphviz = lazy_import('graphviz') # noqa: F811 +graphviz_backend = lazy_import('graphviz.backend') # noqa: F811 + + logger = getLogger(__name__) class Layer: + _mapping: Dict[str, Any] + def __init__(self, level: int, *deps): self.level = level self._mapping = dict() @@ -63,7 +66,7 @@ def __contains__(self, dep) -> bool: dep = dep.name return dep in self._mapping - def __iter__(self): + def __iter__(self) -> Iterator: return iter(self._mapping.values()) def __repr__(self): @@ -74,9 +77,10 @@ def __repr__(self): class Graph: - conflict = None # type: Optional[Dependency] + conflict: Optional[Dependency] = None + _layers: List[Layer] - def __init__(self, *roots): + def __init__(self, *roots: RootDependency) -> None: for root in roots: if not root.dependencies: logger.warning('empty root passed') @@ -118,7 +122,7 @@ def add(self, dep, *, level: Optional[int] = None) -> None: raise KeyError('cannot find any parent for dependency: ' + str(dep.name)) def get_leafs(self, level: Optional[int] = None) -> tuple: - """Get deps that isn't applied yet + """Get deps that aren't applied yet """ layers = self._layers if level is not None: @@ -193,6 +197,19 @@ def get_parents(self, *deps, avoid: Optional[list] = None) -> dict: )) return parents + def fast_apply(self) -> bool: + """Apply only the first layer. + """ + if len(self._layers) != 1: + return False + for root in self._roots: + for dep in root.dependencies: + dep.applied = True + self.add(dep) + for root in self._roots: + root.applied = True + return True + def draw(self, path: str = '.dephell_report', suffix: str = '') -> None: dot = graphviz.Digraph( name=self._roots[0].name + suffix, @@ -247,7 +264,7 @@ def metainfo(self) -> RootDependency: # magic - def __iter__(self): + def __iter__(self) -> Iterator: return iter(self.deps) def __contains__(self, dep) -> bool: diff --git a/dephell/controllers/_mutator.py b/dephell/controllers/_mutator.py index 221ef0bf..ad2f456c 100644 --- a/dephell/controllers/_mutator.py +++ b/dephell/controllers/_mutator.py @@ -1,7 +1,7 @@ # built-in from itertools import product from logging import getLogger -from typing import Iterable, Iterator, Optional, Sequence, Set, Tuple +from typing import Iterable, Iterator, List, Optional, Sequence, Set, Tuple # external import attr @@ -16,8 +16,8 @@ logger = getLogger('dephell.controllers') -def lazy_product(*all_groups): - slices = [[] for _ in range(len(all_groups))] +def lazy_product(*all_groups) -> Iterator: + slices: List[List[str]] = [[] for _ in range(len(all_groups))] all_groups = [iter(groups) for groups in all_groups] while True: @@ -62,6 +62,7 @@ def mutate(self, graph: Graph) -> Optional[Tuple[Group, ...]]: self._check_not_empty, # self._check_soft, ) + assert graph.conflict # hint for mypy for check in checker: for groups in self.get_mutations(deps=parents): if check(groups=groups, deps=parents, conflict=graph.conflict): diff --git a/dephell/controllers/_repos.py b/dephell/controllers/_repos.py index b59b078e..49e19d7b 100644 --- a/dephell/controllers/_repos.py +++ b/dephell/controllers/_repos.py @@ -1,7 +1,7 @@ # built-in from functools import lru_cache from pathlib import Path -from typing import Dict, Iterable, List, Optional +from typing import Dict, Iterable, List, Optional, Set, Type, Union from urllib.parse import urljoin, urlparse # external @@ -31,14 +31,12 @@ def _has_api(url: str) -> bool: @attr.s() class RepositoriesRegistry(WarehouseBaseRepo): - repos = attr.ib(factory=list) + repos: List[WarehouseBaseRepo] = attr.ib(factory=list) prereleases = attr.ib(type=bool, factory=lambda: global_config['prereleases']) # allow prereleases from_config = attr.ib(type=bool, default=False) - _urls = attr.ib(factory=set) - _names = attr.ib(factory=set) - - propagate = True + _urls: Set[str] = attr.ib(factory=set) + _names: Set[str] = attr.ib(factory=set) def add_repo(self, *, url: str, name: str = None, from_config: bool = False) -> bool: if url in self._urls: @@ -74,8 +72,11 @@ def add_repo(self, *, url: str, name: str = None, from_config: bool = False) -> name = urlparse(url).hostname if name in self._names: return False + if not name: + name = 'pypi' self._names.add(name) + cls: Union[Type[WarehouseAPIRepo], Type[WarehouseSimpleRepo]] if _has_api(url=url): cls = WarehouseAPIRepo else: @@ -132,23 +133,27 @@ def make(self, name: str) -> 'RepositoriesRegistry': return type(self)(repos=repos, prereleases=self.prereleases) def get_releases(self, dep) -> tuple: - first_exception = None + first_exception: Optional[Exception] = None for repo in self.repos: try: return repo.get_releases(dep=dep) except PackageNotFoundError as exc: if first_exception is None: first_exception = exc + if first_exception is None: + raise LookupError('no repositories in registry') raise first_exception async def get_dependencies(self, name: str, version: str, extra: Optional[str] = None) -> tuple: - first_exception = None + first_exception: Optional[Exception] = None for repo in self.repos: try: return await repo.get_dependencies(name=name, version=version, extra=extra) except PackageNotFoundError as exc: if first_exception is None: first_exception = exc + if first_exception is None: + raise LookupError('no repositories in registry') raise first_exception def search(self, query: Iterable[str]) -> List[Dict[str, str]]: @@ -166,17 +171,25 @@ async def download(self, name: str, version: str, path: Path) -> bool: # properties @property - def name(self) -> str: + def name(self) -> str: # type: ignore return self.repos[0].name - @property - def url(self) -> str: + @property # type: ignore + def url(self) -> str: # type: ignore return self.repos[0].url + @url.setter + def url(self, url: str): + self.repos[0].url = url + @property def pretty_url(self) -> str: return self.url + @pretty_url.setter + def pretty_url(self, url: str): + self.repos[0].url = url + @property - def propagate(self) -> bool: + def propagate(self) -> bool: # type: ignore return self.repos[0].propagate diff --git a/dephell/controllers/_resolver.py b/dephell/controllers/_resolver.py index 0bb21240..5d951649 100644 --- a/dephell/controllers/_resolver.py +++ b/dephell/controllers/_resolver.py @@ -1,7 +1,7 @@ # built-in import re from logging import getLogger -from typing import Optional +from typing import TYPE_CHECKING, Optional # external from packaging.markers import Marker @@ -13,18 +13,24 @@ from ._conflict import analyze_conflict +if TYPE_CHECKING: + # project + from dephell.controllers._graph import Graph + from dephell.controllers._mutator import Mutator + + logger = getLogger('dephell.resolver') REX_BASE_VERSION = re.compile(r'[0-9\.]+') class Resolver: - def __init__(self, graph, mutator): + def __init__(self, graph: 'Graph', mutator: 'Mutator') -> None: self.graph = graph self.mutator = mutator - def apply(self, parent): + def apply(self, parent, recursive: bool = False): """ - Returns conflicting (incompatible) dependency + Returns conflicting (incompatible) dependency. """ for new_dep in parent.dependencies: other_dep = self.graph.get(new_dep.name) @@ -45,6 +51,10 @@ def apply(self, parent): other_dep += new_dep except TypeError: # conflict happened return other_dep + # `recursive` used only in re-application of dependencies, + # when the graph already was built before. + if recursive: + self.apply(other_dep, recursive=True) # check if not other_dep.compat: return other_dep @@ -132,8 +142,14 @@ def _resolve(self, debug: bool, silent: bool, level: Optional[int], spinner) -> )) self.unapply(dep) dep.group = group + return None + + def apply_envs(self, envs: set, deep: bool = True) -> None: + """Filter out dependencies from the graph by the given envs. - def apply_envs(self, envs: set) -> None: + deep: Helps to avoid fetching dependencies (hence the network requests). + Set it to False for not resolved graph to make filtering faster. + """ if not any(root.dependencies for root in self.graph.get_layer(0)): logger.debug('no dependencies, nothing to filter') return @@ -152,18 +168,22 @@ def apply_envs(self, envs: set) -> None: # and ignored in Requirement.from_graph. # It's bad behavior because deps of this dep can be required for other # deps that won't be unapplied. - self.unapply(dep, soft=True) + if deep: + self.unapply(dep, soft=True) dep.applied = False # Some child deps can be unapplied from other child deps, but we need them. # For example, if we need A, but don't need B, and A and B depends on C, - # then C will be unapplied from B. Let's return B in the graph by re-applying A. + # then C will be unapplied from B. Let's return B in the graph by reapplying A. for dep in self.graph: if not dep.applied: continue if not (dep.envs | dep.inherited_envs) & envs: continue - self.apply(dep) + logger.debug('reapply', extra=dict(dep=dep.name, envs=envs)) + if deep: + self.apply(dep, recursive=True) + dep.applied = True def apply_markers(self, python) -> None: implementation = python.implementation @@ -172,7 +192,10 @@ def apply_markers(self, python) -> None: # get only base part of python version because `packagings` drops # all markers for python prereleases - python_version = REX_BASE_VERSION.match(str(python.version)).group() + python_version = str(python.version) + match = REX_BASE_VERSION.match(python_version) + if match: + python_version = match.group() for dep in self.graph: if not dep.applied: diff --git a/dephell/controllers/_safety.py b/dephell/controllers/_safety.py index a3aa6c2d..dcc03c62 100644 --- a/dephell/controllers/_safety.py +++ b/dephell/controllers/_safety.py @@ -63,7 +63,7 @@ def vulns(self) -> Dict[str, Tuple[SafetyVulnInfo, ...]]: return vulns def get(self, name: str, version: Union[str, Version]) -> List[SafetyVulnInfo]: - if type(version) is str: + if isinstance(version, str): version = Version(version) vulns = [] for vuln in self.vulns.get(name, []): diff --git a/dephell/controllers/_snyk.py b/dephell/controllers/_snyk.py deleted file mode 100644 index ecd37b1d..00000000 --- a/dephell/controllers/_snyk.py +++ /dev/null @@ -1,103 +0,0 @@ -# built-in -import re -from collections import defaultdict -from typing import Dict, List, Optional, Tuple, Union -from xml.etree import ElementTree - -# external -import attr -from dephell_specifier import RangeSpecifier -from packaging.version import VERSION_PATTERN, Version - -# app -from ..cached_property import cached_property -from ..networking import requests_session - - -RSS_URL = 'https://snyk.io/vuln/feed.xml?type=pip' -REX_VERSION = re.compile(VERSION_PATTERN, re.VERBOSE | re.IGNORECASE) -REX_TAG = re.compile(r'<\/?[a-z]+.*?>') -REX_LINK = re.compile(r'https?\:\/\/[a-z]+[^\s\"]+') - - -@attr.s(slots=True) -class SnykVulnInfo: - name = attr.ib(type=str) - severity = attr.ib(type=str) - description = attr.ib(type=str) - version = attr.ib(type=Optional[Version]) - links = attr.ib(type=Tuple[str, ...]) - - @property - def specifier(self) -> RangeSpecifier: - return RangeSpecifier('<' + str(self.version)) - - -@attr.s() -class Snyk: - """ - https://snyk.io/vuln?type=pip - - RSS was broken! - """ - url = attr.ib(type=str, default=RSS_URL) - - @cached_property - def vulns(self) -> Dict[str, List[SnykVulnInfo]]: - with requests_session() as session: - response = session.get(self.url) - response.raise_for_status() - root = ElementTree.fromstring(response.content) - - result = defaultdict(list) - for item in root[0]: - if item.tag != 'item': - continue - item = {field.tag: field.text for field in item} - desc = self._parse_description(item['description']) - desc['links'].append(item['link']) - - versions = [Version(version) for version in desc['versions'] if version] - version = max(versions) if versions else None - - result[desc['name']].append(SnykVulnInfo( - name=desc['name'], - severity=desc['severity'], - description=item['title'].split('https://snyk.io/')[0], - version=version, - links=tuple(desc['links']), - )) - - return result - - def _parse_description(self, text: str): - result = dict(links=[], versions=[]) - for line in text.splitlines(): - clean = REX_TAG.sub('', line.strip()) - if clean.startswith('Severity:'): - result['severity'] = clean.split()[1] - continue - if clean.startswith('Affects:'): - result['name'] = clean.split()[1] - continue - if ' or higher' in clean: - result['versions'] = self._get_versions(clean) - result['links'].extend(REX_LINK.findall(line)) - return result - - @staticmethod - def _get_versions(text) -> List[str]: - versions = [] - for word in text.split(): - if REX_VERSION.fullmatch(word): - versions.append(word) - return versions - - def get(self, name: str, version: Union[str, Version]) -> List[SnykVulnInfo]: - if type(version) is str: - version = Version(version) - vulns = [] - for vuln in self.vulns.get(name, []): - if vuln.version is None or version < vuln.version: - vulns.append(vuln) - return vulns diff --git a/dephell/controllers/_uploader.py b/dephell/controllers/_uploader.py index f990190f..5c91d5e1 100644 --- a/dephell/controllers/_uploader.py +++ b/dephell/controllers/_uploader.py @@ -3,7 +3,7 @@ import subprocess from io import BytesIO from pathlib import Path -from typing import Dict +from typing import Any, Dict from urllib.parse import urlparse # external @@ -28,7 +28,7 @@ def __attrs_post_init__(self): @cached_property def hostname(self) -> str: - return urlparse(self.url).hostname + return urlparse(self.url).hostname or 'pypi.org' @staticmethod def _fix_url(url: str) -> str: @@ -47,10 +47,11 @@ def _fix_url(url: str) -> str: def _get_hashes(path: Path) -> Dict[str, str]: sha256_manager = hashlib.sha256() md5_manager = hashlib.md5() + + blake2_manager: Any = None if hasattr(hashlib, 'blake2b'): blake2_manager = hashlib.blake2b(digest_size=32) - else: - blake2_manager = None + with path.open('rb') as stream: while True: data = stream.read(65536) @@ -67,8 +68,8 @@ def _get_hashes(path: Path) -> Dict[str, str]: ) @classmethod - def _get_file_info(cls, path: Path) -> dict: - result = dict() + def _get_file_info(cls, path: Path) -> Dict[str, str]: + result: Dict[str, str] = dict() # file type and python version if path.suffix == '.whl': @@ -83,12 +84,12 @@ def _get_file_info(cls, path: Path) -> dict: # gpg signature sign_path = path.with_suffix(path.suffix + '.asc') if sign_path.exists(): - result['gpg_signature'] = sign_path.read_bytes() + result['gpg_signature'] = sign_path.read_text() return result @classmethod - def _get_metadata(cls, root) -> dict: + def _get_metadata(cls, root) -> Dict[str, Any]: meta = dict( # defaults that goes as-is on API comment=None, @@ -137,7 +138,7 @@ def _get_metadata(cls, root) -> dict: for key, url in root.links.items(): key = key[0].upper() + key[1:] - meta['project_urls'].append('{}, {}'.format(key, url)) + meta['project_urls'].append('{}, {}'.format(key, url)) # type: ignore if root.python: meta['requires_python'] = str(root.python.peppify()) @@ -146,6 +147,7 @@ def _get_metadata(cls, root) -> dict: @staticmethod def _get_reqs_info(reqs: list) -> dict: + # app from ..converters import EggInfoConverter extras = set() diff --git a/dephell/converters/conda.py b/dephell/converters/conda.py index ac57d3dd..9df53667 100644 --- a/dephell/converters/conda.py +++ b/dephell/converters/conda.py @@ -4,6 +4,7 @@ from typing import Optional # external +import attr from dephell_discover import Root as PackageRoot from dephell_specifier import RangeSpecifier from packaging.utils import canonicalize_name @@ -16,6 +17,7 @@ from .base import BaseConverter +@attr.s() class CondaConverter(BaseConverter): def can_parse(self, path: Path, content: Optional[str] = None) -> bool: if isinstance(path, str): diff --git a/dephell/converters/egginfo.py b/dephell/converters/egginfo.py index 1fe3f8e4..0640e239 100644 --- a/dephell/converters/egginfo.py +++ b/dephell/converters/egginfo.py @@ -2,14 +2,16 @@ from collections import defaultdict from email.parser import Parser from itertools import chain +from logging import getLogger from pathlib import Path from typing import Dict, Optional # external +import attr from dephell_discover import Root as PackageRoot from dephell_links import parse_link from dephell_markers import Markers -from packaging.requirements import Requirement as PackagingRequirement +from packaging.requirements import InvalidRequirement, Requirement as PackagingRequirement # app from ..constants import DOWNLOAD_FIELD, HOMEPAGE_FIELD @@ -20,6 +22,7 @@ class _Reader: + logger = getLogger('dephell.converters.egginfo') def can_parse(self, path: Path, content: Optional[str] = None) -> bool: if isinstance(path, str): @@ -151,7 +154,15 @@ def parse_info(cls, content: str, root=None, urls: Dict[str, str] = None) -> Roo # dependencies deps = [] for req in cls._get_list(info, 'Requires-Dist'): - req = PackagingRequirement(req) + try: + req = PackagingRequirement(req) + except InvalidRequirement: + cls.logger.warning('invalid requirement', extra=dict( + requirement=req, + package_name=root.name, + package_version=root.version, + )) + continue deps.extend(DependencyMaker.from_requirement( source=root, req=req, @@ -391,9 +402,10 @@ def _format_req(req, with_envs: bool) -> str: return line +@attr.s() class EggInfoConverter(_Reader, _Writer, BaseConverter): """ PEP-314, PEP-345, PEP-566 https://packaging.python.org/specifications/core-metadata/ """ - lock = False + lock = attr.ib(type=bool, default=False) diff --git a/dephell/converters/flit.py b/dephell/converters/flit.py index 0df6147e..3fa77f39 100644 --- a/dephell/converters/flit.py +++ b/dephell/converters/flit.py @@ -4,6 +4,7 @@ from typing import Optional # external +import attr import tomlkit from dephell_discover import Root as PackageRoot from dephell_specifier import RangeSpecifier @@ -17,8 +18,9 @@ from .egginfo import EggInfoConverter +@attr.s() class FlitConverter(BaseConverter): - lock = False + lock = attr.ib(type=bool, default=False) def can_parse(self, path: Path, content: Optional[str] = None) -> bool: if not content: @@ -273,7 +275,7 @@ def dumps(self, reqs, project: RootDependency, content=None) -> str: return tomlkit.dumps(doc) - def _format_req(self, req): + def _format_req(self, req: Requirement) -> str: line = req.name if req.extras: line += '[{extras}]'.format(extras=','.join(req.extras)) diff --git a/dephell/converters/imports.py b/dephell/converters/imports.py index 465565c1..910576c1 100644 --- a/dephell/converters/imports.py +++ b/dephell/converters/imports.py @@ -4,6 +4,7 @@ from typing import Dict, List, Set # external +import attr from dephell_discover import Root as PackageRoot # app @@ -23,8 +24,9 @@ CACHE_TTL = 3600 * 24 * 30 # 30 days +@attr.s() class ImportsConverter(BaseConverter): - lock = True + lock = attr.ib(type=bool, default=True) def can_parse(self, path: Path, content: str = None) -> bool: if isinstance(path, str): diff --git a/dephell/converters/installed.py b/dephell/converters/installed.py index 6c125e78..3c671eff 100644 --- a/dephell/converters/installed.py +++ b/dephell/converters/installed.py @@ -1,12 +1,16 @@ # built-in import sys from pathlib import Path -from typing import Iterable, Union +from typing import Iterable, List, Union # external +import attr from dephell_discover import Root as PackageRoot from packaging.utils import canonicalize_name +# project +from dephell.controllers._resolver import Resolver + # app from ..controllers import DependencyMaker from ..models import RootDependency @@ -15,15 +19,16 @@ from .wheel import WheelConverter +@attr.s() class InstalledConverter(BaseConverter): - lock = True + lock = attr.ib(type=bool, default=True) _blacklist = { 'pkg-resources', # https://bugs.launchpad.net/ubuntu/+source/python-pip/+bug/1635463 'command-not-found', # https://stackoverflow.com/a/22676267 } - def load_resolver(self, path=None, paths=None): + def load_resolver(self, path=None, paths: List[Path] = None) -> Resolver: if path is not None: root = self.load(path=path) else: diff --git a/dephell/converters/pip.py b/dephell/converters/pip.py index d31ca921..6d6181db 100644 --- a/dephell/converters/pip.py +++ b/dephell/converters/pip.py @@ -5,6 +5,7 @@ from typing import Optional # external +import attr from dephell_discover import Root as PackageRoot from dephell_links import DirLink, FileLink from pip._internal.req import parse_requirements @@ -19,20 +20,26 @@ try: # pip<20.0.1 + # external from pip._internal.index import PackageFinder except ImportError: # pip>=20.0.1 + # external from pip._internal.index.package_finder import PackageFinder try: + # external from pip._internal.download import PipSession except ImportError: try: + # external from pip._internal.network import PipSession except ImportError: + # external from pip._internal.network.session import PipSession +@attr.s() class PIPConverter(BaseConverter): sep = ' \\\n ' @@ -158,13 +165,14 @@ def dumps(self, reqs, project: Optional[RootDependency] = None, return '\n'.join(lines) + '\n' @staticmethod - def _get_finder(): + def _get_finder() -> PackageFinder: try: return PackageFinder(find_links=[], index_urls=[], session=PipSession()) except TypeError: pass # pip 19.3 + # external from pip._internal.models.search_scope import SearchScope from pip._internal.models.selection_prefs import SelectionPreferences try: @@ -176,11 +184,14 @@ def _get_finder(): except TypeError: pass + # external from pip._internal.models.target_python import TargetPython try: # pip 19.3.1 + # external from pip._internal.collector import LinkCollector except ImportError: + # external from pip._internal.index.collector import LinkCollector return PackageFinder.create( link_collector=LinkCollector( diff --git a/dephell/converters/pipfile.py b/dephell/converters/pipfile.py index 841b446b..cf49c8e7 100644 --- a/dephell/converters/pipfile.py +++ b/dephell/converters/pipfile.py @@ -1,17 +1,19 @@ # built-in from pathlib import Path -from typing import List, Optional +from typing import List, Optional, Union # external +import attr import tomlkit from dephell_discover import Root as PackageRoot from dephell_pythons import Pythons from dephell_specifier import RangeSpecifier from packaging.utils import canonicalize_name +from tomlkit.items import InlineTable, String # app from ..controllers import DependencyMaker, RepositoriesRegistry -from ..models import Constraint, Dependency, RootDependency +from ..models import Constraint, Dependency, Requirement, RootDependency from ..repositories import WarehouseBaseRepo, WarehouseLocalRepo, get_repo from .base import BaseConverter @@ -19,8 +21,10 @@ VCS_LIST = ('git', 'svn', 'hg', 'bzr') +@attr.s() class PIPFileConverter(BaseConverter): - lock = False + lock = attr.ib(type=bool, default=False) + fields = ( 'version', 'editable', 'extras', 'markers', 'ref', 'vcs', 'index', 'hashes', @@ -194,7 +198,7 @@ def _make_deps(self, root, name: str, content) -> List[Dependency]: editable=content.get('editable', False), ) - def _format_req(self, req): + def _format_req(self, req: Requirement) -> Union[InlineTable, String]: result = tomlkit.inline_table() for name, value in req: if name == 'rev': diff --git a/dephell/converters/pipfilelock.py b/dephell/converters/pipfilelock.py index 0feef518..873dea2e 100644 --- a/dephell/converters/pipfilelock.py +++ b/dephell/converters/pipfilelock.py @@ -3,13 +3,17 @@ from collections import OrderedDict from hashlib import sha256 from pathlib import Path -from typing import Optional +from typing import Any, Dict, Optional # external +import attr from dephell_discover import Root as PackageRoot from dephell_pythons import Pythons from dephell_specifier import RangeSpecifier +# project +from dephell.models.requirement import Requirement + # app from ..controllers import RepositoriesRegistry from ..models import RootDependency @@ -22,8 +26,9 @@ # https://github.com/pypa/pipfile/blob/master/pipfile/api.py +@attr.s() class PIPFileLockConverter(PIPFileConverter): - lock = True + lock = attr.ib(type=bool, default=True) def can_parse(self, path: Path, content: Optional[str] = None) -> bool: if isinstance(path, str): @@ -118,7 +123,7 @@ def _get_hash(data: dict) -> str: content = json.dumps(data, sort_keys=True, separators=(',', ':')) return sha256(content.encode('utf8')).hexdigest() - def _format_req(self, req): + def _format_req(self, req: Requirement) -> Dict[str, Any]: result = dict() for name, value in req: if name == 'rev': diff --git a/dephell/converters/poetry.py b/dephell/converters/poetry.py index 7b5d5573..cf5f5126 100644 --- a/dephell/converters/poetry.py +++ b/dephell/converters/poetry.py @@ -1,23 +1,27 @@ # built-in from collections import defaultdict from pathlib import Path -from typing import List, Optional +from typing import List, Optional, Tuple, Union # external +import attr import tomlkit from dephell_discover import Root as PackageRoot from dephell_specifier import RangeSpecifier from packaging.utils import canonicalize_name +from tomlkit.items import InlineTable, String, Table # app from ..controllers import DependencyMaker, Readme, RepositoriesRegistry -from ..models import Author, Constraint, Dependency, EntryPoint, RootDependency +from ..models import Author, Constraint, Dependency, EntryPoint, Requirement, RootDependency from ..repositories import WarehouseLocalRepo from .base import BaseConverter +@attr.s() class PoetryConverter(BaseConverter): - lock = False + lock = attr.ib(type=bool, default=False) + fields = ( 'version', 'python', 'platform', 'allows-prereleases', 'optional', 'extras', 'develop', @@ -30,7 +34,7 @@ def can_parse(self, path: Path, content: Optional[str] = None) -> bool: if isinstance(path, str): path = Path(path) if content: - return '[tool.poetry]' in content + return '[tool.poetry]' in content or '[tool.poetry.' in content return path.name in ('poetry.toml', 'pyproject.toml') def loads(self, content) -> RootDependency: @@ -236,7 +240,7 @@ def dumps(self, reqs, project: RootDependency, content=None) -> str: return tomlkit.dumps(doc).rstrip() + '\n' @staticmethod - def _add_entrypoints(section, entrypoints): + def _add_entrypoints(section: Table, entrypoints: Tuple[EntryPoint, ...]) -> None: # drop old console_scripts if 'scripts' in section: scripts = {e.name for e in entrypoints if e.group == 'console_scripts'} @@ -367,7 +371,7 @@ def _make_deps(self, root, name: str, content, envs: set) -> List[Dependency]: ) return deps - def _format_req(self, req): + def _format_req(self, req: Requirement) -> Union[InlineTable, String]: result = tomlkit.inline_table() for name, value in req: if name in self.fields: diff --git a/dephell/converters/poetrylock.py b/dephell/converters/poetrylock.py index fd4d47a3..bb267df0 100644 --- a/dephell/converters/poetrylock.py +++ b/dephell/converters/poetrylock.py @@ -4,20 +4,24 @@ from typing import List, Optional # external +import attr import tomlkit from dephell_discover import Root as PackageRoot from dephell_links import DirLink from dephell_specifier import RangeSpecifier +from tomlkit.items import Table # app from ..controllers import DependencyMaker, RepositoriesRegistry -from ..models import Constraint, Dependency, RootDependency +from ..models import Constraint, Dependency, Requirement, RootDependency from ..repositories import WarehouseBaseRepo, WarehouseLocalRepo from .base import BaseConverter +@attr.s() class PoetryLockConverter(BaseConverter): - lock = True + lock = attr.ib(type=bool, default=True) + fields = ( 'category', 'description', 'name', 'marker', 'optional', 'python-versions', 'version', 'dependencies', @@ -114,12 +118,12 @@ def _make_deps(cls, root, content, envs, repo) -> List[Dependency]: # get link url = None if 'source' in content: - if content['source']['type'] == 'legacy': + if content['source'].get('type') == 'legacy': repo = repo.make(content['source']['reference']) else: repo = None url = content['source']['url'] - if content['source']['type'] == 'git': + if content['source'].get('type') == 'git': url = 'git+' + url if 'reference' in content['source']: url += '@' + content['source']['reference'] @@ -188,7 +192,7 @@ def _make_deps(cls, root, content, envs, repo) -> List[Dependency]: return deps - def _format_req(self, req): + def _format_req(self, req: Requirement) -> Table: result = tomlkit.table() for name, value in req: if name in self.fields: diff --git a/dephell/converters/pyproject.py b/dephell/converters/pyproject.py index a22293c4..1e216c3c 100644 --- a/dephell/converters/pyproject.py +++ b/dephell/converters/pyproject.py @@ -3,6 +3,7 @@ from typing import Optional # external +import attr from dephell_discover import Root as PackageRoot from packaging.requirements import Requirement from tomlkit import document, dumps, parse @@ -13,8 +14,9 @@ from .base import BaseConverter +@attr.s() class PyProjectConverter(BaseConverter): - lock = False + lock = attr.ib(type=bool, default=False) def can_parse(self, path: Path, content: Optional[str] = None) -> bool: if isinstance(path, str): diff --git a/dephell/converters/sdist.py b/dephell/converters/sdist.py index 32deef8c..388f026f 100644 --- a/dephell/converters/sdist.py +++ b/dephell/converters/sdist.py @@ -25,7 +25,7 @@ class SDistConverter(BaseConverter): # ratio of tests and project size after which tests will be excluded from sdist ratio = attr.ib(type=Optional[float], default=None) - lock = False + lock = attr.ib(type=bool, default=False) def can_parse(self, path: Path, content: Optional[str] = None) -> bool: if content is not None: @@ -126,7 +126,7 @@ def dump(self, reqs, path: Path, project: RootDependency) -> None: self._write_additional_files(tar=tar, project=project, subdir=subdir) - def _write_additional_files(self, *, tar, project, subdir): + def _write_additional_files(self, *, tar: TarFile, project: RootDependency, subdir: str) -> None: # write readme if project.readme: tar.add( @@ -171,6 +171,7 @@ def _write_additional_files(self, *, tar, project, subdir): ) # write tests + # app from ..actions import get_path_size if self.ratio is None: self.ratio = config['sdist']['ratio'] diff --git a/dephell/converters/setuppy.py b/dephell/converters/setuppy.py index 945b6466..c7dfdb18 100644 --- a/dephell/converters/setuppy.py +++ b/dephell/converters/setuppy.py @@ -6,6 +6,7 @@ from typing import Optional # external +import attr from dephell_discover import Root as PackageRoot from dephell_links import DirLink, FileLink, URLLink, VCSLink, parse_link from dephell_setuptools import read_setup @@ -20,11 +21,13 @@ try: + # external from yapf.yapflib.style import CreateGoogleStyle from yapf.yapflib.yapf_api import FormatCode except ImportError: FormatCode = None try: + # external from autopep8 import fix_code except ImportError: fix_code = None @@ -54,8 +57,9 @@ """ +@attr.s() class SetupPyConverter(BaseConverter): - lock = False + lock = attr.ib(type=bool, default=False) def can_parse(self, path: Path, content: Optional[str] = None) -> bool: if isinstance(path, str): diff --git a/dephell/converters/wheel.py b/dephell/converters/wheel.py index 01a7965d..00f7f5d7 100644 --- a/dephell/converters/wheel.py +++ b/dephell/converters/wheel.py @@ -8,6 +8,7 @@ from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo # external +import attr from dephell_archive import ArchivePath from dephell_discover import Root as PackageRoot @@ -207,9 +208,10 @@ def make_records(self, path: str) -> str: return content + ('\n{},,'.format(path)) +@attr.s() class WheelConverter(_Reader, _Writer, BaseConverter): """ PEP-0427 https://www.python.org/dev/peps/pep-0427/ """ - lock = False + lock = attr.ib(type=bool, default=False) diff --git a/dephell/imports.py b/dephell/imports.py index bce60138..899e439a 100644 --- a/dephell/imports.py +++ b/dephell/imports.py @@ -22,6 +22,7 @@ def _module(self): return import_module(name=self._name) def _install(self): + # app from .package_manager import PackageManager name = self._package or self._name.split('.', maxsplit=1)[0] diff --git a/dephell/logging_helpers.py b/dephell/logging_helpers.py index d6d08dad..b7d4f01f 100644 --- a/dephell/logging_helpers.py +++ b/dephell/logging_helpers.py @@ -8,8 +8,10 @@ try: try: + # external from colorama import Fore, init except ImportError: + # external from pip._vendor.colorama import Fore, init init() except ImportError: diff --git a/dephell/models/auth.py b/dephell/models/auth.py index 9473ef95..2d0d460a 100644 --- a/dephell/models/auth.py +++ b/dephell/models/auth.py @@ -5,6 +5,7 @@ # external import attr from requests.auth import HTTPBasicAuth +from requests.models import PreparedRequest @attr.s(eq=True, order=True, frozen=True) @@ -15,7 +16,7 @@ class Auth(HTTPBasicAuth): encoding = attr.ib(type=str, default='latin1') # for requests - def __call__(self, request): + def __call__(self, request: PreparedRequest) -> PreparedRequest: # additional check to prevent lack of creds if urlparse(request.url).hostname != self.hostname: return request @@ -24,6 +25,6 @@ def __call__(self, request): return request # for aiohttp - def encode(self): + def encode(self) -> str: creds = (self.username + ':' + self.password).encode(self.encoding) return 'Basic ' + b64encode(creds).decode('ascii') diff --git a/dephell/models/constraint.py b/dephell/models/constraint.py index c0f7998e..c017b583 100644 --- a/dephell/models/constraint.py +++ b/dephell/models/constraint.py @@ -1,12 +1,20 @@ # built-in from copy import deepcopy +from typing import TYPE_CHECKING, Any, Union # external from dephell_specifier import RangeSpecifier +if TYPE_CHECKING: + # app + from .dependency import Dependency # noqa: F401 + from .extra_dependency import ExtraDependency # noqa: F401 + from .root import RootDependency # noqa: F401 + + class Constraint: - def __init__(self, source, spec): + def __init__(self, source: Union['Dependency', 'ExtraDependency', 'RootDependency'], spec: Any) -> None: """ source (Dependency) spec (str, LegacySpecifier, Specifier) @@ -76,7 +84,7 @@ def copy(self) -> 'Constraint': def __and__(self, other): return self.copy().__iand__(other) - def __iand__(self, other): + def __iand__(self, other: 'Constraint') -> 'Constraint': if not isinstance(other, Constraint): return NotImplemented for name, group in other._groups.items(): diff --git a/dephell/models/dependency.py b/dephell/models/dependency.py index bc0b3b7a..9ac14793 100644 --- a/dephell/models/dependency.py +++ b/dephell/models/dependency.py @@ -1,6 +1,7 @@ # built-in from copy import deepcopy -from typing import Tuple +from pathlib import Path +from typing import Dict, Set, Tuple # external import attr @@ -27,11 +28,11 @@ class Dependency: applied = attr.ib(type=bool, default=False, repr=False) # optional info - description = attr.ib(type=str, default='', repr=False) # summary - authors = attr.ib(factory=tuple, repr=False) # author{,_email}, maintainer{,_email} - links = attr.ib(factory=dict, repr=False) # project_url{,s}, package_url - classifiers = attr.ib(type=tuple, factory=tuple, repr=False) # classifiers - license = attr.ib(default=None, repr=False) # license + description = attr.ib(type=str, default='', repr=False) # summary + authors = attr.ib(factory=tuple, repr=False) # author{,_email}, maintainer{,_email} + links = attr.ib(factory=dict, type=Dict[str, str], repr=False) # project_url{,s}, package_url + classifiers = attr.ib(type=tuple, factory=tuple, repr=False) # classifiers + license = attr.ib(default=None, repr=False) # license # info from requirements file editable = attr.ib(type=bool, default=False, repr=False) @@ -40,7 +41,7 @@ class Dependency: marker = attr.ib(type=MarkerTracker, factory=MarkerTracker, repr=False) envs = attr.ib(type=set, factory=set, repr=False) # which root extras cause this dep inherited_envs = attr.ib(type=set, factory=set, repr=False) # envs of parents - locations = attr.ib(type=set, factory=set, repr=False) # package places on disk + locations = attr.ib(type=Set[Path], factory=set, repr=False) # package places on disk extra = None @@ -81,6 +82,7 @@ def dependencies(self) -> Tuple['Dependency', ...]: if deps is not None: return deps + # app from ..controllers import DependencyMaker deps = [] for dep in self.group.dependencies: @@ -193,13 +195,14 @@ def __str__(self) -> str: marker = Markers(str(self.marker)) if self.envs - {'main'}: - extra_markers = {'extra == "{}"'.format(env) for env in self.envs - {'main'}} + extra_markers = {'extra == {!r}'.format(env) for env in self.envs - {'main'}} marker &= Markers(' or '.join(extra_markers)) if marker: result += '; ' + str(marker) return result def __iadd__(self, dep: 'Dependency') -> 'Dependency': + # app from ..repositories import GitRepo if not isinstance(dep, type(self)): diff --git a/dephell/models/extra_dependency.py b/dephell/models/extra_dependency.py index d986043f..c79a90ea 100644 --- a/dephell/models/extra_dependency.py +++ b/dephell/models/extra_dependency.py @@ -1,6 +1,10 @@ +# built-in +from typing import Union + # external import attr from packaging.utils import canonicalize_name +from tomlkit.items import String # app from ..cached_property import cached_property @@ -12,11 +16,11 @@ class ExtraDependency(Dependency): extra = attr.ib(type=str, default='') - def __attrs_post_init__(self): + def __attrs_post_init__(self) -> None: assert self.extra != '' @classmethod - def from_dep(cls, dep, extra): + def from_dep(cls, dep: Dependency, extra: Union[str, String]) -> 'ExtraDependency': return cls(**attr.asdict(dep, recurse=False), extra=extra) @cached_property diff --git a/dephell/models/group.py b/dephell/models/group.py index 3b6f8c59..f99ae543 100644 --- a/dephell/models/group.py +++ b/dephell/models/group.py @@ -1,12 +1,19 @@ # built-in from operator import attrgetter -from typing import Optional +from typing import TYPE_CHECKING, Optional, Union # app from ..cached_property import cached_property from ..config import config +if TYPE_CHECKING: + # app + from .dependency import Dependency + from .release import Release + from .root import RootRelease # noqa: F401 + + class Group: def __init__(self, number: int, releases: set, dep=None): """ @@ -19,7 +26,7 @@ def __init__(self, number: int, releases: set, dep=None): # BEST RELEASE PROPERTIES @property - def best_release(self): + def best_release(self) -> 'Release': strategy = max if config['strategy'] == 'max' else min best_time = strategy(release.time for release in self.releases) best_releases = [release for release in self.releases if release.time == best_time] @@ -34,7 +41,7 @@ def time(self): # RANDOM RELEASE PROPERTIES @cached_property - def random(self): + def random(self) -> Union['Release', 'RootRelease']: return next(iter(self.all_releases)) @cached_property @@ -52,7 +59,7 @@ def extra(self) -> Optional[str]: # OTHER PROPERTIES @cached_property - def metadependency(self): + def metadependency(self) -> 'Dependency': """MetaDependency is a dependency on the parent for extra. For example, `requests[security]` extra depends on `requests` Dependency. @@ -63,6 +70,7 @@ def metadependency(self): raise ValueError('dep required for group of extras') min_version = min(self.versions) max_version = max(self.versions) + # app from ..controllers import DependencyMaker return DependencyMaker.from_requirement( source=self.dep, @@ -88,7 +96,7 @@ def versions(self) -> set: def empty(self) -> bool: return not bool(self.releases) - def __str__(self): + def __str__(self) -> str: versions = sorted(release.version for release in self.releases) versions = [str(v) for v in versions] if not versions: diff --git a/dephell/models/groups.py b/dephell/models/groups.py index f69182d0..0ee59019 100644 --- a/dephell/models/groups.py +++ b/dephell/models/groups.py @@ -1,6 +1,6 @@ # built-in import asyncio -from typing import Optional +from typing import Iterator, Optional # external import attr @@ -12,6 +12,9 @@ loop = asyncio.get_event_loop() +# How many latest releases should go in their own group. +# Lesser value -> less network requests per package and more mutations. +ONE_GROUP_RELEASES = 1 def get_key(release) -> str: @@ -158,18 +161,21 @@ def _make_group(self, releases) -> Group: self.actualize(group=group) return group - def __iter__(self): + def __iter__(self) -> Iterator: # return all groups from cache for group in self._loaded_groups: self.actualize(group=group) yield group - # load first group + # load the first two groups if not self._loaded_groups: - release = self.releases[0] - self._load_release_deps(release) - self._loaded_releases_count += 1 - yield self._make_group([release]) + for i in range(ONE_GROUP_RELEASES): + if len(self.releases) <= i: + return + release = self.releases[i] + self._load_release_deps(release) + self._loaded_releases_count += 1 + yield self._make_group([release]) # load new groups prev_key = None diff --git a/dephell/models/marker_tracker.py b/dephell/models/marker_tracker.py index 8dce4b8c..d6927d00 100644 --- a/dephell/models/marker_tracker.py +++ b/dephell/models/marker_tracker.py @@ -1,30 +1,43 @@ +# built-in +from typing import Dict, Union + # external from dephell_markers import Markers, OrMarker class MarkerTracker: - def __init__(self): + _markers: Dict[str, Markers] + + def __init__(self) -> None: self._markers = dict() @property def markers(self) -> Markers: + container = Markers() + + if not self._markers: + return container if len(self._markers) == 1: return next(iter(self._markers.values())) - container = Markers() - markers = [m._marker for m in self._markers.values() if m._marker] + + # Empty marker means "no markers should be applied, always match the dependency". + for marker in self._markers.values(): + if not marker: + return container + + # If no empty markers specified, merge all markers as `or` + markers = [m._marker for m in self._markers.values()] container._marker = OrMarker(*markers) return container - def apply(self, *, source, markers) -> 'MarkerTracker': - if not markers: - return self + def apply(self, *, source, markers: Union[str, Markers]) -> 'MarkerTracker': if type(source) is not str: source = source.name if source in self._markers: raise ValueError('marker for given source already added') if type(markers) is str: markers = Markers(markers) - self._markers[source] = markers + self._markers[source] = markers # type: ignore return self def merge(self, other: 'MarkerTracker') -> None: @@ -37,17 +50,17 @@ def unapply(self, source) -> None: if source in self._markers: del self._markers[source] - def __getattr__(self, name): + def __getattr__(self, name: str): if name not in dir(Markers): raise AttributeError(name) return getattr(self.markers, name) def __bool__(self) -> bool: - return any(marker for marker in self._markers.values()) + return bool(self.markers) def __str__(self) -> str: return str(self.markers) - def __repr__(self): + def __repr__(self) -> str: markers = map(repr, self._markers.values()) return '{}({})'.format(type(self).__name__, ', '.join(markers)) diff --git a/dephell/models/release.py b/dephell/models/release.py index f35db33f..74fbd10a 100644 --- a/dephell/models/release.py +++ b/dephell/models/release.py @@ -1,6 +1,6 @@ # built-in from datetime import datetime -from typing import Optional +from typing import Any, Dict, List, Optional # external import attr @@ -16,20 +16,20 @@ class Release: dependencies = None # type: tuple - raw_name = attr.ib(type=str) - version = attr.ib(converter=parse) # type: ignore + raw_name: str = attr.ib() + version: str = attr.ib(converter=parse) # type: ignore time = attr.ib(repr=False) # upload_time python = attr.ib(default=None, repr=False) # requires_python hashes = attr.ib(factory=tuple, repr=False) # digests/sha256 urls = attr.ib(factory=tuple, repr=False) # url - extra = attr.ib(type=Optional[str], default=None) + extra: Optional[str] = attr.ib(default=None) - def __attrs_post_init__(self): + def __attrs_post_init__(self) -> None: assert '[' not in self.raw_name, self.raw_name @classmethod - def from_response(cls, name, version, info, extra=None): + def from_response(cls, name: str, version: str, info: List[Dict[str, Any]], extra=None) -> 'Release': latest = info[-1] python = latest['requires_python'] if python is not None: diff --git a/dephell/models/requirement.py b/dephell/models/requirement.py index 6f5cf369..8e519e80 100644 --- a/dephell/models/requirement.py +++ b/dephell/models/requirement.py @@ -1,6 +1,6 @@ # built-in from collections import OrderedDict, defaultdict -from typing import Iterable, Optional, Set, Tuple +from typing import TYPE_CHECKING, Any, Iterable, Iterator, Optional, Set, Tuple # external import attr @@ -10,6 +10,11 @@ from ..cached_property import cached_property +if TYPE_CHECKING: + # project + from dephell.models.release import Release # noqa: F401 + + class Requirement: _properties = ( 'name', 'release', 'version', 'extras', 'markers', @@ -29,16 +34,15 @@ def from_graph(cls, graph, *, lock: bool) -> Tuple['Requirement', ...]: extras = defaultdict(list) roots = [root.name for root in graph.get_layer(0)] - # if roots wasn't applied then apply them - if len(graph._layers) == 1: - for root in graph._roots: - for dep in root.dependencies: - graph.add(dep) + # if roots weren't applied, apply them + graph.fast_apply() # get all nodes for layer in reversed(graph._layers[1:]): # skip roots for dep in sorted(layer): - if dep.constraint.empty: + if not dep.used: + continue + if not dep.applied: continue if dep.extra is None: req = cls(dep=dep, lock=lock, roots=roots) @@ -54,7 +58,7 @@ def from_graph(cls, graph, *, lock: bool) -> Tuple['Requirement', ...]: return tuple(result.values()) @cached_property - def release(self): + def release(self) -> Optional['Release']: if self.lock: return self.dep.group.best_release @@ -219,7 +223,10 @@ def optional(self) -> bool: @staticmethod def _get_comparable_dict(dep) -> dict: - excluded = {'constraint', 'repo', 'link', 'marker', 'license', 'inherited_envs', 'locations'} + excluded = { + 'constraint', 'repo', 'link', 'marker', 'license', + 'inherited_envs', 'locations', 'applied', + } result = attr.asdict(dep, recurse=True, filter=lambda x, _: x.name not in excluded) result['constraint'] = str(dep.constraint) if dep.marker: @@ -241,7 +248,7 @@ def same_dep(self, other_dep) -> bool: # magic methods - def __iter__(self): + def __iter__(self) -> Iterator[Any]: for name in self._properties: value = getattr(self, name) if value: diff --git a/dephell/models/root.py b/dephell/models/root.py index c3518a6e..8b2e9c4d 100644 --- a/dephell/models/root.py +++ b/dephell/models/root.py @@ -2,7 +2,7 @@ from contextlib import suppress from itertools import chain from pathlib import Path -from typing import Tuple +from typing import TYPE_CHECKING, List, Tuple # external import attr @@ -17,6 +17,11 @@ from .group import Group +if TYPE_CHECKING: + # app + from .dependency import Dependency # noqa: F401 + + @attr.s() class RootRelease: raw_name = attr.ib() @@ -38,7 +43,7 @@ def __str__(self): @attr.s() class RootDependency: raw_name = attr.ib(default='root') - dependencies = attr.ib(factory=list, repr=False) + dependencies = attr.ib(factory=list, repr=False, type=List['Dependency']) repo = attr.ib(default=None, repr=False) # additional info strings @@ -103,6 +108,7 @@ def python_compat(self) -> bool: @property def warehouses(self) -> tuple: + # app from ..repositories import WarehouseBaseRepo repos = dict() diff --git a/dephell/networking.py b/dephell/networking.py index 9f862be7..faa91d3c 100644 --- a/dephell/networking.py +++ b/dephell/networking.py @@ -1,38 +1,81 @@ # built-in +from functools import partial, update_wrapper +from logging import getLogger from ssl import create_default_context +from time import sleep +from typing import Any, Optional # external import certifi import requests -from aiohttp import ClientSession, TCPConnector +from aiohttp import ClientError, ClientSession, TCPConnector +from requests.sessions import Session + +# project +from dephell.models.auth import Auth # app from . import __version__ +from .config import config USER_AGENT = 'DepHell/{version}'.format(version=__version__) +logger = getLogger('dephell.networking') -def aiohttp_session(*, auth=None, **kwargs): - headers = dict() +def aiohttp_session(*, auth: Optional[Auth] = None, **kwargs: Any) -> ClientSession: + headers = {'User-Agent': USER_AGENT} if auth: headers['Authorization'] = auth.encode() - ssl_context = create_default_context(cafile=certifi.where()) + + # setup SSL + cafile = config.get('ca') + if not cafile: + cafile = certifi.where() + ssl_context = create_default_context(cafile=cafile) try: connector = TCPConnector(ssl=ssl_context) except TypeError: connector = TCPConnector(ssl_context=ssl_context) + return ClientSession(headers=headers, connector=connector, **kwargs) -def requests_session(*, auth=None, headers=None, **kwargs): +def requests_session(*, auth: Optional[Any] = None, headers: Optional[Any] = None, **kwargs: Any) -> Session: session = requests.Session() if auth: session.auth = auth + + # setup SSL + cafile = config.get('ca') + if cafile: + session.verify = cafile + + # set headers if headers is None: headers = dict() headers.setdefault('User-Agent', USER_AGENT) session.headers = headers if kwargs: session.__dict__.update(kwargs) + return session + + +def aiohttp_repeat(func=None, *, count: int = 4): + if func is None: + return partial(func, count=count) + + async def wrapper(*args: Any, **kwargs: Any) -> Optional[Any]: + for pause in range(1, count + 1): + try: + return await func(*args, **kwargs) + except ClientError: + if pause == count: + raise + logger.debug('aiohttp payload error, repeating...', exc_info=True) + sleep(pause) + raise RuntimeError('unreachable') + + wrapper = update_wrapper(wrapper=wrapper, wrapped=func) + return wrapper diff --git a/dephell/repositories/_conda/_composite.py b/dephell/repositories/_conda/_composite.py index 5817e842..3ac64261 100644 --- a/dephell/repositories/_conda/_composite.py +++ b/dephell/repositories/_conda/_composite.py @@ -19,7 +19,7 @@ class CondaRepo(CondaBaseRepo): _requests = attr.ib(default=0) - def __attrs_post_init__(self): + def __attrs_post_init__(self) -> None: if not self.git_repo.channels: self.git_repo.channels = self.channels if not self.cloud_repo.channels: diff --git a/dephell/repositories/_getter.py b/dephell/repositories/_getter.py index 13a288a4..1eba5cec 100644 --- a/dephell/repositories/_getter.py +++ b/dephell/repositories/_getter.py @@ -19,6 +19,7 @@ def get_repo(link=None, *, name: str = None, default=None): + # app from ..controllers import RepositoriesRegistry if name is not None: diff --git a/dephell/repositories/_git/git.py b/dephell/repositories/_git/git.py index 0a6b9820..55a75e70 100644 --- a/dephell/repositories/_git/git.py +++ b/dephell/repositories/_git/git.py @@ -53,9 +53,10 @@ def path(self): return path @cached_property - def metaversion(self): + def metaversion(self) -> Optional[str]: if self.link.rev: return self.get_nearest_version(self.link.rev) + return None # PUBLIC METHODS @@ -153,7 +154,7 @@ def _get_rev_hash(self, rev: str): return data[-1].strip().strip('"') @staticmethod - def _clean_tag(tag): + def _clean_tag(tag: str) -> str: return rex_version.fullmatch(tag).groups()[0] def _version_to_rev(self, version) -> str: diff --git a/dephell/repositories/_local.py b/dephell/repositories/_local.py index 72ad925b..4d4c8590 100644 --- a/dephell/repositories/_local.py +++ b/dephell/repositories/_local.py @@ -50,7 +50,8 @@ async def get_dependencies(self, name: str, version: str, extra: Optional[str] = return deps def get_root(self, name: str, version: str): - from ..converters import EggInfoConverter, SDistConverter, WheelConverter, CONVERTERS + # app + from ..converters import CONVERTERS, EggInfoConverter, SDistConverter, WheelConverter if not self.path.exists(): raise FileNotFoundError(str(self.path)) diff --git a/dephell/repositories/_warehouse/_api.py b/dephell/repositories/_warehouse/_api.py index 5b68d597..cca29fd9 100644 --- a/dephell/repositories/_warehouse/_api.py +++ b/dephell/repositories/_warehouse/_api.py @@ -38,6 +38,8 @@ 'summary', 'version', } +# Do not download too big files +SIZE_LIMIT = 2 * 1024 * 1024 # 2 Mb @attr.s() @@ -54,7 +56,7 @@ class WarehouseAPIRepo(WarehouseBaseRepo): hash = None link = None - def __attrs_post_init__(self): + def __attrs_post_init__(self) -> None: # make name canonical if self.name in ('pypi.org', 'pypi.python.org'): self.name = 'pypi' @@ -270,6 +272,7 @@ async def _get_from_files(self, files_info: List[dict]) -> Tuple[str, ...]: # if file_info['packagetype'] == 'bdist_wheel': # return () + # app from ...converters import SDistConverter, WheelConverter sdist = SDistConverter() @@ -284,8 +287,11 @@ async def _get_from_files(self, files_info: List[dict]) -> Tuple[str, ...]: for converter, checker in rules: for file_info in files_info: + if file_info.get('size') > SIZE_LIMIT: + continue if not checker(file_info): continue + logger.debug('downloading file...', extra=dict(url=file_info['url'])) try: return await self._download_and_parse( url=file_info['url'], diff --git a/dephell/repositories/_warehouse/_base.py b/dephell/repositories/_warehouse/_base.py index 97588f42..b48b64bd 100644 --- a/dephell/repositories/_warehouse/_base.py +++ b/dephell/repositories/_warehouse/_base.py @@ -3,7 +3,7 @@ from logging import getLogger from pathlib import Path from tempfile import TemporaryDirectory -from typing import Tuple +from typing import Any, Tuple from urllib.parse import urlparse, urlunparse # external @@ -13,11 +13,12 @@ # app from ...cached_property import cached_property from ...constants import WAREHOUSE_DOMAINS -from ...networking import aiohttp_session +from ...networking import aiohttp_repeat, aiohttp_session from ..base import Interface try: + # external import aiofiles except ImportError: aiofiles = None @@ -28,10 +29,10 @@ class WarehouseBaseRepo(Interface): - # I'm not sure how to combine `@abstractproperty` and `= attr.ib()` - @cached_property - def prereleases(self) -> bool: - raise NotImplementedError + prereleases: bool + url: str + name: str + auth: Any @cached_property def from_config(self): @@ -99,22 +100,23 @@ def _convert_deps(*, deps, name, version, extra): )) try: - dep_extra = req.marker and Markers(req.marker).extra + dep_extras = req.marker and Markers(req.marker).get_strings('extra') except ValueError: # unsupported operation for version marker python_version: in - dep_extra = None + dep_extras = set() # it's not extra and we want not extra too - if dep_extra is None and extra is None: + if not dep_extras and extra is None: result.append(req) continue # it's extra, but we want not the extra # or it's not the extra, but we want extra. - if dep_extra is None or extra is None: + if not dep_extras or extra is None: continue # it's extra and we want this extra - elif dep_extra == extra: - result.append(req) - continue + for dep_extra in dep_extras: + if dep_extra == extra: + result.append(req) + break return tuple(result) @@ -136,6 +138,7 @@ async def _download_and_parse(self, *, url: str, converter) -> Tuple[str, ...]: deps.append(str(dep)) return tuple(deps) + @aiohttp_repeat async def _download(self, *, url: str, path: Path) -> None: async with aiohttp_session(auth=self.auth) as session: async with session.get(url) as response: diff --git a/dephell/repositories/_warehouse/_local.py b/dephell/repositories/_warehouse/_local.py index 70dbc7d2..f62c9f78 100644 --- a/dephell/repositories/_warehouse/_local.py +++ b/dephell/repositories/_warehouse/_local.py @@ -32,7 +32,7 @@ class WarehouseLocalRepo(WarehouseBaseRepo): from_config = attr.ib(type=bool, default=False) propagate = True # deps of deps will inherit repo - def __attrs_post_init__(self): + def __attrs_post_init__(self) -> None: if isinstance(self.path, str): self.path = Path(self.path) @@ -104,6 +104,7 @@ def _get_hash(path: Path) -> str: return digest.hexdigest() def _get_deps_from_files(self, name, version): + # app from ...converters import SDistConverter, WheelConverter paths = [] diff --git a/dephell/repositories/_warehouse/_simple.py b/dephell/repositories/_warehouse/_simple.py index 51dea3b0..c3fe1d8c 100644 --- a/dephell/repositories/_warehouse/_simple.py +++ b/dephell/repositories/_warehouse/_simple.py @@ -40,7 +40,7 @@ class WarehouseSimpleRepo(WarehouseBaseRepo): from_config = attr.ib(type=bool, default=False) propagate = True # deps of deps will inherit repo - def __attrs_post_init__(self): + def __attrs_post_init__(self) -> None: # make name canonical if self.name in ('pypi.org', 'pypi.python.org'): self.name = 'pypi' @@ -53,9 +53,19 @@ def get_releases(self, dep) -> tuple: releases_info = dict() for link in links: name, version = self._parse_name(link['name']) - if canonicalize_name(name) != dep.name: + if canonicalize_name(name) != canonicalize_name(dep.base_name): + logger.warning('bad dist name', extra=dict( + dist_name=link['name'], + package_name=dep.base_name, + reason='package name does not match', + )) continue if not version: + logger.warning('bad dist name', extra=dict( + dist_name=link['name'], + package_name=dep.base_name, + reason='no version specified', + )) continue if version not in releases_info: @@ -112,6 +122,9 @@ def search(self, query: Iterable[str]) -> List[Dict[str, str]]: raise NotImplementedError async def download(self, name: str, version: str, path: Path) -> bool: + if not isinstance(version, str): + version = str(version) + links = self._get_links(name=name) good_links = [] for link in links: @@ -142,11 +155,13 @@ def _get_links(self, name: str) -> List[Dict[str, str]]: ttl=config['cache']['ttl'], ) links = cache.load() - if links: - return links + if links is not None: + yield from links + return dep_url = posixpath.join(self.url, quote(name)) + '/' with requests_session() as session: + logger.debug('getting dep info from simple repo', extra=dict(url=dep_url)) response = session.get(dep_url, auth=self.auth) if response.status_code == 404: raise PackageNotFoundError(package=name, url=dep_url) @@ -164,17 +179,20 @@ def _get_links(self, name: str) -> List[Dict[str, str]]: python = tag.get('data-requires-python') fragment = parse_qs(parsed.fragment) - yield dict( + link = dict( url=urljoin(dep_url, link), name=parsed.path.strip('/').split('/')[-1], python=html.unescape(python) if python else '*', digest=fragment['sha256'][0] if 'sha256' in fragment else None, ) + links.append(link) + yield link cache.dump(links) return links - async def _get_deps_from_links(self, name, version): + async def _get_deps_from_links(self, name: str, version): + # app from ...converters import SDistConverter, WheelConverter links = self._get_links(name=name) @@ -183,7 +201,7 @@ async def _get_deps_from_links(self, name, version): link_name, link_version = self._parse_name(link['name']) if canonicalize_name(link_name) != name: continue - if link_version != version: + if link_version != str(version): continue good_links.append(link) diff --git a/dephell/repositories/base.py b/dephell/repositories/base.py index d4be6e47..60041e66 100644 --- a/dephell/repositories/base.py +++ b/dephell/repositories/base.py @@ -8,7 +8,7 @@ class Interface(metaclass=abc.ABCMeta): - propagate = False + propagate: bool = False @abc.abstractmethod def get_releases(self, dep) -> tuple: @@ -23,7 +23,7 @@ def pretty_url(self): return self.__dict__.get('pretty_url', self.url) @pretty_url.setter - def pretty_url(self, url): + def pretty_url(self, url: str): self.__dict__['pretty_url'] = url def search(self, query: Iterable[str]) -> List[Dict[str, str]]: diff --git a/dephell/yaml.py b/dephell/yaml.py index a0dfa76e..0b572268 100644 --- a/dephell/yaml.py +++ b/dephell/yaml.py @@ -1,3 +1,6 @@ +# built-in +from typing import Any, Dict, TextIO + # app from .imports import lazy_import @@ -6,7 +9,7 @@ py_yaml = lazy_import('yaml', package='PyYAML') -def yaml_load(stream, *, safe: bool = True): +def yaml_load(stream: TextIO, *, safe: bool = True): if safe: parser = ruamel_yaml.YAML(typ='safe') else: @@ -24,6 +27,6 @@ def yaml_load(stream, *, safe: bool = True): return py_yaml.load(stream) -def yaml_dump(data, stream): +def yaml_dump(data: Dict[str, Any], stream: TextIO): parser = ruamel_yaml.YAML() - return parser.dump(data, stream) + parser.dump(data, stream) diff --git a/docs/changelog.md b/docs/changelog.md index e24a8876..fa846bc4 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,10 @@ Follow [@PythonDepHell](https://twitter.com/PythonDepHell) on Twitter to get updates about new features and releases. +## v.0.8.3 (2020-04-28) + +This release brings a lot of small fixes. See [the milestone](https://github.com/dephell/dephell/milestone/6?closed=1) for the details. The only noticeable change is an ability to provide a custom CA bundle via `--ca` flag. A few next releases also will bring more stability and speed into DepHell. [Stay tuned!](https://twitter.com/PythonDepHell) + ## v.0.8.2 (2020-03-19) New commands: diff --git a/docs/cmd-deps-audit.md b/docs/cmd-deps-audit.md index d70cc2eb..6b3a06cd 100644 --- a/docs/cmd-deps-audit.md +++ b/docs/cmd-deps-audit.md @@ -8,8 +8,6 @@ This command returns non-zero code if some vulnerabilities was found, so you can [pyup.io](https://pyup.io/) provides public repository [safety-db](https://github.com/pyupio/safety-db) with all vulnerabilities in their database. DepHell uses it. This repository automatically updates every month. So, if you want to get actual reports you have to use their official solutions. They provide Safety CI that [free for Open Source](https://pyup.io/pricing/) and $30 for personal usage. If you have "Business" plan you also can get API key and use their [official CLI](https://github.com/pyupio/safety). -We used [snyk.io](https://snyk.io/) before, but now they have removed RSS feed. - ## Dependencies lookup 1. If some package and version explicitly specified then this package will be used. Example: `dephell deps audit jinja2==2.0`. diff --git a/docs/cmd-deps-convert.md b/docs/cmd-deps-convert.md index 4c2ede46..b48c1c2b 100644 --- a/docs/cmd-deps-convert.md +++ b/docs/cmd-deps-convert.md @@ -85,9 +85,9 @@ You can convert anything to anything: 1. Lock Pipfile: `dephell deps convert --from=Pipfile --to=Pipfile.lock` 1. Lock poetry: `dephell deps convert --from=pyproject.toml --to=poetry.lock` 1. Migrate from setup.py to poetry: `dephell deps convert --from=setup.py --to=pyproject.toml` -1. Migrate from pipenv to poetry: `dephell deps convert --from=Pipenv --to=pyproject.toml` +1. Migrate from pipenv to poetry: `dephell deps convert --from=Pipfile --to=pyproject.toml` 1. Generate setup.py for poetry (to make project backward compatible with setuptools): `dephell deps convert --from=pyproject.toml --to=setup.py` -1. Generate requirements.txt from Pipfile to work on a pipenv-based project without pipenv: `dephell deps convert --from=Pipenv --to=requirements.txt` +1. Generate requirements.txt from Pipfile to work on a pipenv-based project without pipenv: `dephell deps convert --from=Pipfile --to=requirements.txt` 1. Generate requirements.txt from poetry to work on a poetry-based project without poetry: `dephell deps convert --from=pyproject.toml --to=requirements.txt` 1. Pipe poetry requirements into pip for installation in a custom environment: `pip install -r <(dephell deps convert --to-path stdout --to-format pip)` diff --git a/docs/conf.py b/docs/conf.py index f1dad500..d669e5e9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,7 +7,6 @@ # external import alabaster -from recommonmark.parser import CommonMarkParser from recommonmark.transform import AutoStructify @@ -20,12 +19,10 @@ 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', + 'recommonmark', ] templates_path = ['_templates'] -source_parsers = { - '.md': CommonMarkParser, -} source_suffix = ['.rst', '.md'] master_doc = 'index' @@ -33,7 +30,7 @@ copyright = '{}, Gram (@orsinium)'.format(date.today().year) author = 'Gram (@orsinium)' -version = '0.8.2' +version = '0.8.3' release = version language = None diff --git a/docs/params.md b/docs/params.md index 538f3167..2dcb80e8 100644 --- a/docs/params.md +++ b/docs/params.md @@ -52,13 +52,27 @@ Commands that accept these parameters: + `{digest}` will be replaced by the short 4-letters digest of the project path to avoid conflicts for the projects with the same name in different locations. + `{env}` will be replaced by current environment (`main` by default). + `--python` -- python version for venv. This can be reloaded in the dependencies file. ++ `--dotenv` -- path to a [.env](https://www.npmjs.com/package/dotenv) file. Will be sourced on a venv activation. + `vars` (config only) -- dict of environment variables to pass in virtual environment. +## Docker + ++ `--docker-repo` -- image name without tag to use for [Docker-based commands](index-docker). ++ `--docker-tag` -- image tag. ++ `--docker-container` -- container name. By default, automatically generated from the project name. + +## Project upload + ++ `--upload-url` -- URL of API endpoint to use to upload dist. ++ `--sign` -- a flag indicates that dists must be signed before uploading. ++ `--identity` -- GPG identity to use to sign dists. + ## Output + `--format` -- output format. + `--level` -- minimal level for log messages. Available levels: `DEBUG`, `INFO`, `WARNING`, `ERROR` and `EXCEPTION`. `INFO` by default. `DEBUG` and `INFO` writes in the stdout, other levels in the stderr. + `--nocolors` -- do not color output. ++ `--table` -- format output as a nice ASCII table. + `--silent` -- suppress any output except errors. Disables progress bar for resolver. + `--filter` -- [filter for JSON output](filters). + `--traceback` -- show traceback for exceptions. @@ -71,9 +85,14 @@ Other: + `--cache-ttl` -- Time to live for releases list cache (in seconds). 1 hour by default. + `--project` -- path to the current project. Current directory by default. + `--bin` -- path to the dir for installing scripts. ++ `--ca` -- path to a custom [CA bundle](https://www.namecheap.com/support/knowledgebase/article.aspx/986/69/what-is-ca-bundle) file. If provided, will be used for both `requests` and `aiohttp`. + `--envs` -- environments (`main`, `dev`) or extras to install or convert. + `--tests` -- path to test files for [dephell project test](cmd-project-test) command. + `--versioning` -- versioning scheme for project. See [dephell project bump](cmd-project-bump) for details. ++ `--tag` -- template for git tag to use in [dephell project bump](cmd-project-bump). ++ `command` (config-only) -- default command to run in [dephell venv run](cmd-venv-run) and [dephell docker run](cmd-docker-run). ++ `--vendor-exclude` -- dependencies that shouldn't be [vendorized](index-vendor). ++ `--vendor-path` -- path to store vendorized dependencies.. ## Default values diff --git a/install.py b/install.py index a5afea9e..1825f22f 100644 --- a/install.py +++ b/install.py @@ -77,19 +77,24 @@ def args(self): def pip_main(self): # install try: + # external import pip # noQA: F401 except ImportError: print('install pip') + # built-in from ensurepip import bootstrap bootstrap() # import try: + # external from pip._internal.main import main except ImportError: try: + # external from pip._internal import main except ImportError: + # external from pip import main return main @@ -97,6 +102,7 @@ def pip_main(self): @cached_property def data_dir(self) -> Path: try: + # external from appdirs import user_data_dir except ImportError: @@ -111,6 +117,7 @@ def data_dir(self) -> Path: return path / 'dephell' self.pip_main(['install', 'appdirs']) + # external from appdirs import user_data_dir return Path(user_data_dir('dephell')) diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..78a98ac3 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +allow_redefinition = True +follow_imports = silent +ignore_missing_imports = True +show_error_codes = True +warn_unused_ignores = True diff --git a/pyproject.toml b/pyproject.toml index 98ac2988..3ab04065 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,11 @@ command = "flake8" from = {format = "poetry", path = "pyproject.toml"} # mypy isn't specified in dependencies, # but `dephell venv run` command will install it automatically -command = "mypy --ignore-missing-imports --allow-redefinition dephell" +command = [ + "mypy", + "dephell/actions", "dephell/commands", "dephell/config", + "dephell/controllers", "dephell/yaml.py" +] # Test project package in the separated venv: # $ dephell project test --env=pytest @@ -70,7 +74,7 @@ command = "sphinx-build -W docs docs/build" [tool.dephell.isort] from = {format = "poetry", path = "pyproject.toml"} envs = ["main", "full", "dev"] -command = "isort -y" +command = "isort ." [tool.dephell.vendorized] from = {format = "poetry", path = "pyproject.toml"} @@ -105,7 +109,7 @@ import_heading_localfolder = "app" [tool.poetry] name = "dephell" -version = "0.8.2" +version = "0.8.3" authors = ["Gram "] description = "Dependency resolution for Python" readme = "README.md" @@ -201,4 +205,8 @@ isort = {extras = ["pyproject"], version = "*"} [tool.poetry.extras] docs = ["alabaster", "pygments-github-lexers", "recommonmark", "sphinx"] tests = ["aioresponses", "pytest", "requests-mock"] -full = ["aiofiles", "appdirs", "autopep8", "bowler", "colorama", "docker", "dockerpty", "fissix", "graphviz", "html5lib", "pygments", "python-gnupg", "tabulate", "yapf"] +full = [ + "aiofiles", "appdirs", "autopep8", "bowler", "colorama", "docker", "dockerpty", + "fissix", "graphviz", "html5lib", "pygments", "python-gnupg", "ruamel.yaml", + "tabulate", "yapf", +] diff --git a/setup.cfg b/setup.cfg index 9b37ff71..b59285e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,3 +11,8 @@ exclude= .pytest_cache build setup.py + +[tool:pytest] +addopts = --strict-markers +markers = + allow_hosts: allow network requests to specified hostnames. diff --git a/setup.py b/setup.py index f7904a7b..bc06fb97 100644 --- a/setup.py +++ b/setup.py @@ -5,10 +5,13 @@ # https://github.com/dephell/dephell try: + # external from setuptools import setup except ImportError: + # built-in from distutils.core import setup +# built-in # built-in import os.path @@ -23,7 +26,7 @@ setup( long_description=readme, name='dephell', - version='0.8.2', + version='0.8.3', description='Dependency resolution for Python', python_requires='>=3.6', project_urls={ @@ -64,7 +67,7 @@ 'dephell-setuptools>=0.2.1', 'dephell-shells>=0.1.3', 'dephell-specifier>=0.1.7', 'dephell-venvs>=0.1.16', 'dephell-versioning', 'jinja2', 'm2r', 'packaging', - 'pip<=19.3.1,>=18.0', 'requests', 'ruamel.yaml', 'tomlkit', 'yaspin' + 'pip<=19.3.1,>=18.0', 'requests', 'tomlkit', 'yaspin' ], extras_require={ "dev": [ @@ -78,7 +81,7 @@ "full": [ "aiofiles", "appdirs", "autopep8", "bowler", "colorama", "docker", "dockerpty", "fissix", "graphviz", "html5lib", "pygments", - "python-gnupg", "tabulate", "yapf" + "python-gnupg", "ruamel.yaml", "tabulate", "yapf" ], "tests": ["aioresponses", "pytest", "requests-mock"] }, diff --git a/tests/conftest.py b/tests/conftest.py index 3e38a1e6..ccc38a01 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -88,6 +88,7 @@ def temp_path(tmp_path: Path): @pytest.fixture() def temp_cache(temp_path): + # project from dephell.config import config config.attach({'cache': {'path': str(temp_path)}}) diff --git a/tests/helpers.py b/tests/helpers.py index 4365cf70..3caed4b3 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -71,7 +71,15 @@ def make_root(root, **releases) -> RootDependency: return root_dep -def check(root, resolved=True, missed=None, **deps): +def set_envs(root: RootDependency, dep_name: str, envs: str) -> None: + for dep in root.dependencies: + if dep.name == dep_name: + dep.envs = envs + return + raise RuntimeError('cannot find dep') + + +def check(root: RootDependency, resolved: bool = True, missed=None, envs: set = None, **deps): resolver = Resolver( graph=Graph(root), mutator=Mutator(), @@ -82,6 +90,9 @@ def check(root, resolved=True, missed=None, **deps): ): result = resolver.resolve(debug=True, silent=True) + if envs is not None: + resolver.apply_envs(envs=envs) + reqs = Requirement.from_graph(resolver.graph, lock=True) reqs = {req.name: req for req in reqs} @@ -104,4 +115,4 @@ def check(root, resolved=True, missed=None, **deps): if missed: for name in missed: - assert name not in reqs + assert name not in reqs, '{} must be missed but it is not'.format(name) diff --git a/tests/requirements/module.py b/tests/requirements/module.py index d836e3e5..0d8f3fa0 100644 --- a/tests/requirements/module.py +++ b/tests/requirements/module.py @@ -7,4 +7,4 @@ 1 / 0 -import pytest # noqa: E402 +import pytest # noqa: E402 # isort:skip diff --git a/tests/requirements/setup.py b/tests/requirements/setup.py index 005ee5fb..f0e84b20 100644 --- a/tests/requirements/setup.py +++ b/tests/requirements/setup.py @@ -5,7 +5,6 @@ # external from setuptools import setup - here = path.abspath(path.dirname(__file__)) root = path.dirname(path.dirname(here)) with open(path.join(root, 'README.md'), encoding='utf-8') as f: diff --git a/tests/test_actions/test_entrypoints.py b/tests/test_actions/test_entrypoints.py index 0bef4490..0a4cd320 100644 --- a/tests/test_actions/test_entrypoints.py +++ b/tests/test_actions/test_entrypoints.py @@ -38,4 +38,5 @@ class FakeVenv: entrypoints = get_entrypoints(venv=FakeVenv, name='pytest') assert len(entrypoints) == 2 assert {e.name for e in entrypoints} == {'pytest', 'py.test'} - assert {e.path for e in entrypoints} == {'pytest:main'} + exp = ({'pytest:main'}, {'pytest:console_main'}) + assert {e.path for e in entrypoints} in exp diff --git a/tests/test_actions/test_transform.py b/tests/test_actions/test_transform.py index 9dcbb778..c7711956 100644 --- a/tests/test_actions/test_transform.py +++ b/tests/test_actions/test_transform.py @@ -50,6 +50,7 @@ # ), ]) def test_transform_imports(code_in: str, code_out: str, old_name: str, new_name: str, temp_path: Path): + # external from bowler import Query code_in += '\n' diff --git a/tests/test_commands/test_deps_add.py b/tests/test_commands/test_deps_add.py new file mode 100644 index 00000000..99b93611 --- /dev/null +++ b/tests/test_commands/test_deps_add.py @@ -0,0 +1,33 @@ +# built-in +from pathlib import Path + +# external +import pytest + +# project +from dephell.commands import DepsAddCommand +from dephell.config import Config + + +@pytest.mark.allow_hosts() +def test_deps_add_command(temp_path: Path, capsys): + reqs_path = temp_path / 'requirements.txt' + reqs_path.write_text('six==1.12.0') + + config = Config() + config.attach({ + 'level': 'WARNING', + 'silent': True, + 'nocolors': True, + 'from': dict(format='pip', path=reqs_path), + }) + + command = DepsAddCommand(argv=['jinja2==2.0'], config=config) + result = command() + + captured = capsys.readouterr() + print(captured.err) + print(captured.out) + assert result is True + + assert set(reqs_path.read_text().split()) == {'six==1.12.0', 'jinja2==2.0'} diff --git a/tests/test_commands/test_generate_license.py b/tests/test_commands/test_generate_license.py index 732581db..2e8c0aec 100644 --- a/tests/test_commands/test_generate_license.py +++ b/tests/test_commands/test_generate_license.py @@ -10,7 +10,7 @@ from dephell.config import Config -@pytest.mark.allow_hosts(['pypi.org', '140.211.169.8']) +@pytest.mark.allow_hosts() def test_generate_license_command(temp_path: Path): config = Config() config.attach({'project': str(temp_path)}) diff --git a/tests/test_controllers/test_safety.py b/tests/test_controllers/test_safety.py index 1464abb0..d7da2a96 100644 --- a/tests/test_controllers/test_safety.py +++ b/tests/test_controllers/test_safety.py @@ -9,5 +9,5 @@ def test_safety(): safety = Safety() vulns = safety.get('django', '1.11.0') - assert len(vulns) == 5 + assert len(vulns) == 13 assert {vuln.name for vuln in vulns} == {'django'} diff --git a/tests/test_controllers/test_snyk.py b/tests/test_controllers/test_snyk.py deleted file mode 100644 index bef2e481..00000000 --- a/tests/test_controllers/test_snyk.py +++ /dev/null @@ -1,15 +0,0 @@ -# external -import pytest - -# project -from dephell.controllers import Snyk - - -@pytest.mark.skipif(True, reason='RSS feed was removed from snyk') -@pytest.mark.allow_hosts() -def test_safety(): - snyk = Snyk() - assert sum(len(vulns) for vulns in snyk.vulns.values()) == 50 - - for name, vulns in snyk.vulns.items(): - assert len(snyk.get(name=name, version='0.0.0')) == len(vulns) diff --git a/tests/test_converters/test_all.py b/tests/test_converters/test_all.py index 62f2ac30..cca78f53 100644 --- a/tests/test_converters/test_all.py +++ b/tests/test_converters/test_all.py @@ -1,7 +1,7 @@ -# external # built-in from pathlib import Path +# external import attr import pytest diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 00000000..871895f1 --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,15 @@ +# built-in +from pathlib import Path + +# project +from dephell.config.scheme import SCHEME + + +def test_params_all_described(): + undocumented = {'and', 'auth', 'vars', 'command'} + path = Path(__file__).parent.parent / 'docs' / 'params.md' + content = path.read_text() + for key in SCHEME: + if key in undocumented: + continue + assert '`--' + key in content diff --git a/tests/test_repositories/test_conda.py b/tests/test_repositories/test_conda.py index 2c4fb048..94aeeac0 100644 --- a/tests/test_repositories/test_conda.py +++ b/tests/test_repositories/test_conda.py @@ -10,6 +10,7 @@ from dephell.repositories import CondaCloudRepo, CondaGitRepo, CondaRepo +@pytest.mark.xfail(reason='conda.anaconda.org is dead') @pytest.mark.allow_hosts() @pytest.mark.parametrize('repo_class', [CondaRepo, CondaCloudRepo]) def test_conda_get_releases(repo_class): @@ -21,6 +22,7 @@ def test_conda_get_releases(repo_class): assert not {'3.0.3', '3.1.0', '4.0.0', '4.1.0'} - versions +@pytest.mark.xfail(reason='conda.anaconda.org is dead') @pytest.mark.allow_hosts() @pytest.mark.skipif('TRAVIS_OS_NAME' in environ, reason='Travis CI usually out of rate for Github') def test_conda_get_releases_git(): @@ -32,6 +34,7 @@ def test_conda_get_releases_git(): assert not {'3.0.3', '3.1.0', '4.0.0', '4.1.0'} - versions +@pytest.mark.xfail(reason='conda.anaconda.org is dead') @pytest.mark.allow_hosts() @pytest.mark.parametrize('repo_class', [CondaRepo, CondaCloudRepo]) def test_conda_deps(repo_class): @@ -43,6 +46,7 @@ def test_conda_deps(repo_class): assert 'prodigal' in deps +@pytest.mark.xfail(reason='conda.anaconda.org is dead') @pytest.mark.allow_hosts() @pytest.mark.skipif('TRAVIS_OS_NAME' in environ, reason='Travis CI usually out of rate for Github') def test_conda_deps_git(): diff --git a/tests/test_repositories/test_warehouse_api.py b/tests/test_repositories/test_warehouse_api.py index ec7f9cb2..844a6357 100644 --- a/tests/test_repositories/test_warehouse_api.py +++ b/tests/test_repositories/test_warehouse_api.py @@ -5,6 +5,7 @@ # external import pytest +from packaging.version import Version # project from dephell.constants import DEFAULT_WAREHOUSE @@ -110,8 +111,9 @@ def test_get_deps_auth(asyncio_mock, temp_cache, fixtures_path: Path): assert client._default_headers['authorization'] == 'Basic Z3JhbTp0ZXN0' +@pytest.mark.parametrize('version', ['0.1.2', Version('0.1.2')]) def test_download(asyncio_mock, temp_cache, fixtures_path: Path, temp_path: Path, - requirements_path: Path): + requirements_path: Path, version): pypi_url = 'https://custom.pypi.org/pypi/' json_response = (fixtures_path / 'warehouse-api-release.json').read_text() json_content = json.loads(json_response) @@ -123,7 +125,7 @@ def test_download(asyncio_mock, temp_cache, fixtures_path: Path, temp_path: Path asyncio_mock.get(file_url, body=file_content) repo = WarehouseAPIRepo(name='pypi', url=pypi_url) - coroutine = repo.download(name='dephell-shells', version='0.1.2', path=temp_path) + coroutine = repo.download(name='dephell-shells', version=version, path=temp_path) result = loop.run_until_complete(asyncio.gather(coroutine))[0] assert result is True assert (temp_path / file_name).exists() diff --git a/tests/test_repositories/test_warehouse_simple.py b/tests/test_repositories/test_warehouse_simple.py index 5950660a..c8277e46 100644 --- a/tests/test_repositories/test_warehouse_simple.py +++ b/tests/test_repositories/test_warehouse_simple.py @@ -6,6 +6,7 @@ # external import pytest +from packaging.version import Version # project from dephell.constants import DEFAULT_WAREHOUSE @@ -128,8 +129,9 @@ def test_get_deps_auth(requests_mock, temp_cache, fixtures_path): assert requests_mock.last_request.headers['Authorization'] == 'Basic Z3JhbTp0ZXN0' +@pytest.mark.parametrize('version', ['0.1.2', Version('0.1.2')]) def test_download(requests_mock, asyncio_mock, temp_cache, fixtures_path: Path, - temp_path: Path, requirements_path: Path): + temp_path: Path, requirements_path: Path, version): pypi_url = 'https://custom.pypi.org/pypi/' text_response = (fixtures_path / 'warehouse-simple.html').read_text() file_url = re.findall( @@ -143,7 +145,7 @@ def test_download(requests_mock, asyncio_mock, temp_cache, fixtures_path: Path, asyncio_mock.get(file_url, body=file_content) repo = WarehouseSimpleRepo(name='pypi', url=pypi_url) - coroutine = repo.download(name='dephell-shells', version='0.1.2', path=temp_path) + coroutine = repo.download(name='dephell-shells', version=version, path=temp_path) result = loop.run_until_complete(asyncio.gather(coroutine))[0] assert result is True assert (temp_path / file_name).exists() diff --git a/tests/test_resolving/test_apply_envs.py b/tests/test_resolving/test_apply_envs.py new file mode 100644 index 00000000..eb9437a9 --- /dev/null +++ b/tests/test_resolving/test_apply_envs.py @@ -0,0 +1,145 @@ +# project +from dephell.controllers import Graph, Mutator, Resolver +from dephell.converters import PIPConverter +from dephell.models import Requirement + +# app +from ..helpers import Fake, check, make_root, set_envs + + +def fast_filter(root, *, deep=True): + resolver = Resolver( + graph=Graph(root), + mutator=Mutator(), + ) + resolver.graph.fast_apply() + resolver.apply_envs(envs={'main'}, deep=deep) + reqs = Requirement.from_graph(resolver.graph, lock=False) + return {req.name: req for req in reqs} + + +def test_direct_dependencies(): + root = make_root( + root=Fake('', 'a', 'b'), + a=(Fake('1.0'), ), + b=(Fake('1.0'), ), + ) + set_envs(root, 'a', {'main'}) + set_envs(root, 'b', {'dev'}) + check(root=root, a='==1.0', missed=['b'], envs={'main'}) + + +def test_subdependencies(): + root = make_root( + root=Fake('', 'a', 'b'), + a=(Fake('1.0'), ), + b=(Fake('1.0', 'c'), ), + c=(Fake('1.0'), ), + ) + set_envs(root, 'a', {'main'}) + set_envs(root, 'b', {'dev'}) + check(root=root, a='==1.0', missed=['b', 'c'], envs={'main'}) + + +def test_reapply(): + root = make_root( + root=Fake('', 'a', 'b'), + a=(Fake('1.0', 'c'), ), + b=(Fake('1.0', 'c'), ), + c=(Fake('1.0'), ), + ) + set_envs(root, 'a', {'main'}) + set_envs(root, 'b', {'dev'}) + check(root=root, a='==1.0', c='==1.0', missed=['b'], envs={'main'}) + + +def test_unapply_twice(): + root = make_root( + root=Fake('', 'a', 'b', 'c'), + a=(Fake('1.0'), ), + b=(Fake('1.0', 'd'), ), + c=(Fake('1.0', 'd'), ), + d=(Fake('1.0'), ), + ) + set_envs(root, 'a', {'main'}) + set_envs(root, 'b', {'dev'}) + set_envs(root, 'c', {'dev'}) + check(root=root, a='==1.0', missed=['b', 'c', 'd'], envs={'main'}) + + +def test_with_real_names(): + root = make_root( + root=Fake('', 'bandit', 'boltons'), + bandit=(Fake('1.0', 'colorama'), ), + boltons=(Fake('1.0'), ), + colorama=(Fake('1.0'), ), + ) + set_envs(root, 'boltons', {'main'}) + set_envs(root, 'bandit', {'dev'}) + check(root=root, boltons='==1.0', missed=['bandit', 'colorama'], envs={'main'}) + + +def test_deep_dependencies(): + root = make_root( + root=Fake('', 'apiwrapper', 'sphinx', 'certifi'), + sphinx=(Fake('1.0', 'requests'), ), + apiwrapper=(Fake('1.0', 'requests'), ), + requests=(Fake('1.0', 'certifi'), ), + certifi=(Fake('1.0'), ), + ) + set_envs(root, 'sphinx', {'main'}) + set_envs(root, 'apiwrapper', {'dev'}) + set_envs(root, 'certifi', {'dev'}) + check(root=root, sphinx='==1.0', certifi='==1.0', requests='==1.0', missed=['apiwrapper'], envs={'main'}) + + +def test_very_deep_dependencies_reapply(): + root = make_root( + root=Fake('', 'a', 'b'), + a=(Fake('1.0', 'c'), ), + b=(Fake('1.0', 'c'), ), + + c=(Fake('1.0', 'd'), ), + d=(Fake('1.0', 'e'), ), + e=(Fake('1.0'), ), + ) + set_envs(root, 'a', {'main'}) + set_envs(root, 'b', {'dev'}) + check(root=root, a='==1.0', c='==1.0', d='==1.0', e='==1.0', missed=['b'], envs={'main'}) + + +def test_dependencies_unapply_twice_and_reapply(): + root = make_root( + root=Fake('', 'requests', 'sphinx', 'certifi'), + requests=(Fake('1.0', 'certifi'), ), + sphinx=(Fake('1.0', 'requests'), ), + certifi=(Fake('1.0'), ), + ) + set_envs(root, 'sphinx', {'main'}) + set_envs(root, 'requests', {'dev'}) + set_envs(root, 'certifi', {'dev'}) + check(root=root, sphinx='==1.0', requests='==1.0', certifi='==1.0', missed=[], envs={'main'}) + + +def test_direct_dependencies_without_resolving(): + root = make_root( + root=Fake('', 'a', 'b'), + a=(Fake('1.0'), ), + b=(Fake('1.0'), ), + ) + set_envs(root, 'a', {'main'}) + set_envs(root, 'b', {'dev'}) + + reqs = fast_filter(root) + assert set(reqs) == {'a'} + + +def test_not_deep(): + """If deep is False, filtering must work and no network requests can be made. + """ + loader = PIPConverter(lock=False) + root = loader.loads(content='sphinx\nrequests') + root.dependencies[0].envs = {'main'} + root.dependencies[1].envs = {'dev'} + reqs = fast_filter(root, deep=False) + assert set(reqs) == {'sphinx'}