ansible-galaxy - remove internal path when using AnsibleCollectionConfig.collection_paths (#85596)

* remove internal collections earlier to ignore consistently for different sub-commands

* remove internal collection handling from the dependency resolver

* add a test to ensure ansible._protomatter is not in the output of ansible-galaxy collection list

* fix existing test to ensure an error is given if no valid collection path is configured

* changelog

changelogs/fragments/85596-hide-proto.yml

@@ -0,0 +1,3 @@
+bugfixes:
+- >-
+  ``ansible-galaxy collection list`` - fail when none of the configured collection paths exist.

lib/ansible/cli/galaxy.py

@@ -213,6 +213,18 @@ class GalaxyCLI(CLI):
         self.lazy_role_api = None
         super(GalaxyCLI, self).__init__(args)
 
+    @property
+    def collection_paths(self):
+        """
+        Exclude lib/ansible/_internal/ansible_collections/.
+        """
+        # exclude bundled collections, e.g. ansible._protomatter
+        return [
+            path
+            for path in AnsibleCollectionConfig.collection_paths
+            if path != AnsibleCollectionConfig._internal_collections
+        ]
+
     def init_parser(self):
         """ create an options parser for bin/ansible """
 
@@ -1281,7 +1293,7 @@ class GalaxyCLI(CLI):
         """Compare checksums with the collection(s) found on the server and the installed copy. This does not verify dependencies."""
 
         collections = context.CLIARGS['args']
-        search_paths = AnsibleCollectionConfig.collection_paths
+        search_paths = self.collection_paths
         ignore_errors = context.CLIARGS['ignore_errors']
         local_verify_only = context.CLIARGS['offline']
         requirements_file = context.CLIARGS['requirements']
@@ -1423,7 +1435,7 @@ class GalaxyCLI(CLI):
         collections_path = C.COLLECTIONS_PATHS
 
         managed_paths = set(validate_collection_path(p) for p in C.COLLECTIONS_PATHS)
-        read_req_paths = set(validate_collection_path(p) for p in AnsibleCollectionConfig.collection_paths)
+        read_req_paths = set(validate_collection_path(p) for p in self.collection_paths)
 
         unexpected_path = C.GALAXY_COLLECTIONS_PATH_WARNING and not any(p.startswith(path) for p in managed_paths)
         if unexpected_path and any(p.startswith(path) for p in read_req_paths):
@@ -1639,7 +1651,7 @@ class GalaxyCLI(CLI):
         collection_name = context.CLIARGS['collection']
         default_collections_path = set(C.COLLECTIONS_PATHS)
         collections_search_paths = (
-            set(context.CLIARGS['collections_path'] or []) | default_collections_path | set(AnsibleCollectionConfig.collection_paths)
+            set(context.CLIARGS['collections_path'] or []) | default_collections_path | set(self.collection_paths)
         )
         collections_in_paths = {}
 

lib/ansible/galaxy/collection/__init__.py

@@ -84,7 +84,6 @@ if t.TYPE_CHECKING:
     FileManifestEntryType = t.Dict[FileMetaKeysType, t.Union[str, int, None]]
     FilesManifestType = t.Dict[t.Literal['files', 'format'], t.Union[t.List[FileManifestEntryType], int]]
 
-import ansible
 import ansible.constants as C
 from ansible.errors import AnsibleError
 from ansible.galaxy.api import GalaxyAPI
@@ -144,8 +143,6 @@ ModifiedContent = namedtuple('ModifiedContent', ['filename', 'expected', 'instal
 
 SIGNATURE_COUNT_RE = r"^(?P<strict>\+)?(?:(?P<count>\d+)|(?P<all>all))$"
 
-_ANSIBLE_PACKAGE_PATH = pathlib.Path(ansible.__file__).parent
-
 
 @dataclass
 class ManifestControl:
@@ -1432,12 +1429,9 @@ def find_existing_collections(path_filter, artifacts_manager, namespace_filter=N
     paths = set()
     for path in files('ansible_collections').glob('*/*/'):
         path = _normalize_collection_path(path)
-        if path.is_relative_to(_ANSIBLE_PACKAGE_PATH):
-            # skip internal path, those collections are not for galaxy use
+        if not path.is_dir():
             continue
-        elif not path.is_dir():
-            continue
-        elif path_filter:
+        if path_filter:
             for pf in path_filter:
                 try:
                     path.relative_to(_normalize_collection_path(pf))

lib/ansible/galaxy/dependency_resolution/dataclasses.py

@@ -26,9 +26,6 @@ if t.TYPE_CHECKING:
         '_ComputedReqKindsMixin',
     )
 
-import ansible
-import ansible.release
-
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.galaxy.api import GalaxyAPI
 from ansible.galaxy.collection import HAS_PACKAGING, PkgReq
@@ -42,7 +39,6 @@ _ALLOW_CONCRETE_POINTER_IN_SOURCE = False  # NOTE: This is a feature flag
 _GALAXY_YAML = b'galaxy.yml'
 _MANIFEST_JSON = b'MANIFEST.json'
 _SOURCE_METADATA_FILE = b'GALAXY.yml'
-_ANSIBLE_PACKAGE_PATH = pathlib.Path(ansible.__file__).parent
 
 display = Display()
 
@@ -229,12 +225,6 @@ class _ComputedReqKindsMixin:
             dir_path = dir_path.rstrip(to_bytes(os.path.sep))
         if not _is_collection_dir(dir_path):
             dir_pathlib = pathlib.Path(to_text(dir_path))
-
-            # special handling for bundled collections without manifests, e.g., ansible._protomatter
-            if dir_pathlib.is_relative_to(_ANSIBLE_PACKAGE_PATH):
-                req_name = f'{dir_pathlib.parent.name}.{dir_pathlib.name}'
-                return cls(req_name, ansible.release.__version__, dir_path, 'dir', None)
-
             display.warning(
                 u"Collection at '{path!s}' does not have a {manifest_json!s} "
                 u'file, nor has it {galaxy_yml!s}: cannot detect version.'.

lib/ansible/plugins/loader.py

@@ -1674,7 +1674,7 @@ def _configure_collection_loader(prefix_collections_path=None):
 
     # insert the internal ansible._protomatter collection up front
     paths = [os.path.dirname(_internal.__file__)] + list(prefix_collections_path) + C.COLLECTIONS_PATHS
-    finder = _AnsibleCollectionFinder(paths, C.COLLECTIONS_SCAN_SYS_PATH)
+    finder = _AnsibleCollectionFinder(paths, C.COLLECTIONS_SCAN_SYS_PATH, internal_collections=paths[0])
     finder._install()
 
     # this should succeed now

lib/ansible/utils/collection_loader/_collection_config.py

@@ -62,6 +62,11 @@ class _AnsibleCollectionConfig(type):
         cls._require_finder()
         return [_to_text(p) for p in cls._collection_finder._n_collection_paths]
 
+    @property
+    def _internal_collections(cls):
+        cls._require_finder()
+        return cls._collection_finder._internal_collections
+
     @property
     def default_collection(cls):
         return cls._default_collection

lib/ansible/utils/collection_loader/_collection_finder.py

@@ -182,7 +182,7 @@ class _AnsibleTraversableResources(TraversableResources):
 
 
 class _AnsibleCollectionFinder:
-    def __init__(self, paths=None, scan_sys_paths=True):
+    def __init__(self, paths=None, scan_sys_paths=True, internal_collections=None):
         # TODO: accept metadata loader override
         self._ansible_pkg_path = _to_text(os.path.dirname(_to_bytes(sys.modules['ansible'].__file__)))
 
@@ -209,6 +209,7 @@ class _AnsibleCollectionFinder:
             if p not in good_paths and os.path.isdir(_to_bytes(os.path.join(p, 'ansible_collections'))):
                 good_paths.append(p)
 
+        self._internal_collections = internal_collections
         self._n_configured_paths = good_paths
         self._n_cached_collection_paths = None
         self._n_cached_collection_qualified_paths = None

test/integration/targets/ansible-galaxy-collection/tasks/list.yml

@@ -67,6 +67,7 @@
       - 'list_result.stdout is regex "dev.collection4\s+\*"'
       - 'list_result.stdout is regex "dev.collection5\s+\*"'
       - 'list_result.stdout is regex "dev.collection6\s+\*"'
+      - 'list_result.stdout is not regex "ansible._protomatter\s+.*"'
 
 - name: list collections in human format
   command: ansible-galaxy collection list --format human
@@ -158,9 +159,8 @@
 - name: Ensure we get the expected error
   assert:
     that:
-      # FIXME: This test is currently incorrect, but not a fix to do in this PR, proper test is the commented out one.
-      - "'{}' in list_result_error.stdout"
-      #- "'None of the provided paths were usable' in list_result_error.stderr"
+      - "'{}' not in list_result_error.stdout"
+      - "'None of the provided paths were usable' in list_result_error.stderr"
 
 - name: install an artifact to the second collections path
   command: ansible-galaxy collection install namespace1.name1 -s galaxy_ng {{ galaxy_verbosity }} -p "{{ galaxy_dir }}/prod"