diff --git a/CHANGELOG.md b/CHANGELOG.md index 09ccfc280..dcb8d4f93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # **Upcoming release** +- ... +- #516 Autoimport Now automatically detects project dependencies and can read TOML configuration - #730 Match on module aliases for autoimport suggestions # Release 1.12.0 diff --git a/docs/configuration.rst b/docs/configuration.rst index b30bc314f..b77359765 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -18,7 +18,7 @@ Will be used if [tool.rope] is configured. ['dt', 'datetime'], ['mp', 'multiprocessing'], ] - + autoimport.underlined = false config.py --------- @@ -64,6 +64,11 @@ Options ------- .. autopytoolconfigtable:: rope.base.prefs.Prefs +Autoimport Options +------------------ +.. autopytoolconfigtable:: rope.base.prefs.AutoimportPrefs + + Old Configuration File ---------------------- This is a sample config.py. While this config.py works and all options here should be supported, the above documentation reflects the latest version of rope. diff --git a/rope/base/prefs.py b/rope/base/prefs.py index 2e698eba1..491746577 100644 --- a/rope/base/prefs.py +++ b/rope/base/prefs.py @@ -14,13 +14,13 @@ @dataclass class AutoimportPrefs: - # underlined: bool = field( - # default=False, description="Cache underlined (private) modules") + underlined: bool = field( + default=False, description="Cache underlined (private) modules") # memory: bool = field(default=None, description="Cache in memory instead of disk") # parallel: bool = field(default=True, description="Use multiple processes to parse") aliases: List[Tuple[str, str]] = field( - default_factory=lambda : [ + default_factory=lambda: [ ("np", "numpy"), ("pd", "pandas"), ("plt", "matplotlib.pyplot"), diff --git a/rope/contrib/autoimport/sqlite.py b/rope/contrib/autoimport/sqlite.py index 2af7baab6..21139b670 100644 --- a/rope/contrib/autoimport/sqlite.py +++ b/rope/contrib/autoimport/sqlite.py @@ -16,13 +16,16 @@ from threading import local from typing import Generator, Iterable, Iterator, List, Optional, Set, Tuple +from packaging.requirements import Requirement + from rope.base import exceptions, libutils, resourceobserver, taskhandle, versioning +from rope.base.prefs import AutoimportPrefs from rope.base.project import Project from rope.base.resources import Resource from rope.contrib.autoimport import models from rope.contrib.autoimport.defs import ( - ModuleFile, Alias, + ModuleFile, Name, NameType, Package, @@ -54,18 +57,36 @@ def get_future_names( def filter_packages( - packages: Iterable[Package], underlined: bool, existing: List[str] + packages: Iterable[Package], + underlined: bool, + existing: List[str], + dependencies: Optional[List[Requirement]], ) -> Iterable[Package]: """Filter list of packages to parse.""" + parsed_deps = ( + [dep.name for dep in dependencies] if dependencies is not None else None + ) + + def is_dep(package) -> bool: + return ( + parsed_deps is None + or package.name in parsed_deps + or package.source is not Source.SITE_PACKAGE + ) + if underlined: def filter_package(package: Package) -> bool: - return package.name not in existing + return package.name not in existing and is_dep(package) else: def filter_package(package: Package) -> bool: - return package.name not in existing and not package.name.startswith("_") + return ( + package.name not in existing + and not package.name.startswith("_") + and is_dep(package) + ) return filter(filter_package, packages) @@ -85,6 +106,7 @@ class AutoImport: memory: bool project: Project project_package: Package + prefs: AutoimportPrefs underlined: bool def __init__( @@ -114,6 +136,7 @@ def __init__( autoimport = AutoImport(..., memory=True) """ self.project = project + self.prefs = deepcopy(self.project.prefs.autoimport) project_package = get_package_tuple(project.root.pathlib, project) assert project_package is not None assert project_package.path is not None @@ -397,12 +420,13 @@ def generate_modules_cache( """ Generate global name cache for external modules listed in `modules`. - If no modules are provided, it will generate a cache for every module available. + If modules is not specified, uses PEP 621 metadata. + If modules aren't specified and PEP 621 is not present, caches every package This method searches in your sys.path and configured python folders. Do not use this for generating your own project's internal names, use generate_resource_cache for that instead. """ - underlined = self.underlined if underlined is None else underlined + underlined = self.prefs.underlined if underlined is None else underlined packages: List[Package] = ( self._get_available_packages() @@ -411,7 +435,11 @@ def generate_modules_cache( ) existing = self._get_packages_from_cache() - packages = list(filter_packages(packages, underlined, existing)) + packages = list( + filter_packages( + packages, underlined, existing, self.project.prefs.dependencies + ) + ) if not packages: return self._add_packages(packages) @@ -523,7 +551,7 @@ def update_resource( self, resource: Resource, underlined: bool = False, commit: bool = True ): """Update the cache for global names in `resource`.""" - underlined = underlined if underlined else self.underlined + underlined = underlined if underlined else self.prefs.underlined module = self._resource_to_module(resource, underlined) self._del_if_exist(module_name=module.modname, commit=False) for name in get_names(module, self.project_package): @@ -548,7 +576,11 @@ def _del_if_exist(self, module_name, commit: bool = True): def _get_python_folders(self) -> List[Path]: def filter_folders(folder: Path) -> bool: - return folder.is_dir() and folder.as_posix() != "/usr/bin" + return ( + folder.is_dir() + and folder.as_posix() != "/usr/bin" + and str(folder) != self.project.address + ) folders = self.project.get_python_path_folders() folder_paths = filter(filter_folders, map(Path, folders)) diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index d61bac6c2..e4786a353 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -119,7 +119,9 @@ def get_files( yield ModuleFile(package.path, package.path.stem, underlined, False) else: assert package.path - for file in package.path.glob("**/*.py"): + for file in package.path.rglob("*.py"): + if "site-packages" in file.relative_to(package.path).parts: + continue if file.name == "__init__.py": yield ModuleFile( file, diff --git a/ropetest/conftest.py b/ropetest/conftest.py index ce8a1eab5..53e7b4109 100644 --- a/ropetest/conftest.py +++ b/ropetest/conftest.py @@ -7,8 +7,9 @@ @pytest.fixture -def project(): - project = testutils.sample_project() +def project(request): + pyproject = request.param if hasattr(request, "param") else None + project = testutils.sample_project(pyproject=pyproject) yield project testutils.remove_project(project) @@ -39,6 +40,7 @@ def project2(): /pkg1/mod2.py -- mod2 """ + @pytest.fixture def mod1(project) -> resources.File: return testutils.create_module(project, "mod1") diff --git a/ropetest/contrib/autoimport/autoimporttest.py b/ropetest/contrib/autoimport/autoimporttest.py index 072ab1ed8..f9496db60 100644 --- a/ropetest/contrib/autoimport/autoimporttest.py +++ b/ropetest/contrib/autoimport/autoimporttest.py @@ -14,7 +14,7 @@ @pytest.fixture def autoimport(project: Project): - with closing(AutoImport(project)) as ai: + with closing(AutoImport(project, memory=True)) as ai: yield ai @@ -124,9 +124,9 @@ def foo(): def test_connection(project: Project, project2: Project): - ai1 = AutoImport(project) - ai2 = AutoImport(project) - ai3 = AutoImport(project2) + ai1 = AutoImport(project, memory=True) + ai2 = AutoImport(project, memory=True) + ai3 = AutoImport(project2, memory=True) assert ai1.connection is not ai2.connection assert ai1.connection is not ai3.connection diff --git a/ropetest/contrib/autoimport/deptest.py b/ropetest/contrib/autoimport/deptest.py new file mode 100644 index 000000000..7021e3075 --- /dev/null +++ b/ropetest/contrib/autoimport/deptest.py @@ -0,0 +1,40 @@ +from typing import Iterable + +import pytest + +from rope.contrib.autoimport.sqlite import AutoImport + + +@pytest.fixture +def autoimport(project) -> Iterable[AutoImport]: + autoimport = AutoImport(project, memory=True) + autoimport.generate_modules_cache() + yield autoimport + autoimport.close() + + +@pytest.mark.parametrize("project", ((""),), indirect=True) +def test_blank(project, autoimport): + assert project.prefs.dependencies is None + assert autoimport.search("pytoolconfig") + + +@pytest.mark.parametrize("project", (("[project]\n dependencies=[]"),), indirect=True) +def test_empty(project, autoimport): + assert len(project.prefs.dependencies) == 0 + assert [] == autoimport.search("pytoolconfig") + + +FILE = """ +[project] +dependencies = [ + "pytoolconfig", + "bogus" +] +""" + + +@pytest.mark.parametrize("project", ((FILE),), indirect=True) +def test_not_empty(project, autoimport): + assert len(project.prefs.dependencies) == 2 + assert autoimport.search("pytoolconfig") diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index d72f3f23f..de706bc41 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -12,7 +12,7 @@ def setUp(self): self.mod1 = testutils.create_module(self.project, "mod1") self.pkg = testutils.create_package(self.project, "pkg") self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) - self.importer = autoimport.AutoImport(self.project, observe=False) + self.importer = autoimport.AutoImport(self.project, observe=False, memory=True) def tearDown(self): testutils.remove_project(self.project) @@ -87,7 +87,7 @@ def test_not_caching_underlined_names(self): self.assertEqual(["mod1"], self.importer.get_modules("_myvar")) def test_caching_underlined_names_passing_to_the_constructor(self): - importer = autoimport.AutoImport(self.project, False, True) + importer = autoimport.AutoImport(self.project, False, True, memory=True) self.mod1.write("_myvar = None\n") importer.update_resource(self.mod1) self.assertEqual(["mod1"], importer.get_modules("_myvar")) @@ -171,9 +171,10 @@ def test_skipping_directories_not_accessible_because_of_permission_error(self): # The single thread test takes much longer than the multithread test but is easier to debug single_thread = False self.importer.generate_modules_cache(single_thread=single_thread) - + # Create a temporary directory and set permissions to 000 - import tempfile, sys + import sys + import tempfile with tempfile.TemporaryDirectory() as dir: import os os.chmod(dir, 0o000) @@ -190,7 +191,7 @@ def setUp(self): self.mod1 = testutils.create_module(self.project, "mod1") self.pkg = testutils.create_package(self.project, "pkg") self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) - self.importer = autoimport.AutoImport(self.project, observe=True) + self.importer = autoimport.AutoImport(self.project, observe=True, memory=True) def tearDown(self): testutils.remove_project(self.project) diff --git a/ropetest/testutils.py b/ropetest/testutils.py index 11f58ccfe..46087ee56 100644 --- a/ropetest/testutils.py +++ b/ropetest/testutils.py @@ -5,6 +5,7 @@ import tempfile import unittest from pathlib import Path +from typing import Optional import rope.base.project from rope.contrib import generate @@ -14,10 +15,14 @@ RUN_TMP_DIR = tempfile.mkdtemp(prefix="ropetest-run-") -def sample_project(foldername=None, **kwds): +def sample_project(foldername=None, pyproject: Optional[str] = None, **kwds): root = Path(tempfile.mkdtemp(prefix="project-", dir=RUN_TMP_DIR)) root /= foldername if foldername else "sample_project" logging.debug("Using %s as root of the project.", root) + root.mkdir(exist_ok=True) + if pyproject is not None: + file = root / "pyproject.toml" + file.write_text(pyproject, encoding="utf-8") # Using these prefs for faster tests prefs = { "save_objectdb": False,