Rewrite _is_role and remove incidental_flatpak_remote (#72105)

* Build explicit list instead of complicated nested loop

* ci_complete ci_coverage

* Remove incidental_flatpak_remote

* ci_complete ci_coverage

lib/ansible/parsing/dataloader.py

@@ -198,20 +198,34 @@ class DataLoader:
         ''' imperfect role detection, roles are still valid w/o tasks|meta/main.yml|yaml|etc '''
 
         b_path = to_bytes(path, errors='surrogate_or_strict')
+        b_path_dirname = os.path.dirname(b_path)
         b_upath = to_bytes(unfrackpath(path, follow=False), errors='surrogate_or_strict')
 
-        for b_finddir in (b'meta', b'tasks'):
+        untasked_paths = (
-            for b_suffix in (b'.yml', b'.yaml', b''):
+            os.path.join(b_path, b'main.yml'),
-                b_main = b'main%s' % (b_suffix)
+            os.path.join(b_path, b'main.yaml'),
-                b_tasked = os.path.join(b_finddir, b_main)
+            os.path.join(b_path, b'main'),
+        )
+        tasked_paths = (
+            os.path.join(b_upath, b'tasks/main.yml'),
+            os.path.join(b_upath, b'tasks/main.yaml'),
+            os.path.join(b_upath, b'tasks/main'),
+            os.path.join(b_upath, b'meta/main.yml'),
+            os.path.join(b_upath, b'meta/main.yaml'),
+            os.path.join(b_upath, b'meta/main'),
+            os.path.join(b_path_dirname, b'tasks/main.yml'),
+            os.path.join(b_path_dirname, b'tasks/main.yaml'),
+            os.path.join(b_path_dirname, b'tasks/main'),
+            os.path.join(b_path_dirname, b'meta/main.yml'),
+            os.path.join(b_path_dirname, b'meta/main.yaml'),
+            os.path.join(b_path_dirname, b'meta/main'),
+        )
+
+        exists_untasked = map(os.path.exists, untasked_paths)
+        exists_tasked = map(os.path.exists, tasked_paths)
+        if RE_TASKS.search(path) and any(exists_untasked) or any(exists_tasked):
+            return True
 
-                if (
-                    RE_TASKS.search(path) and
-                    os.path.exists(os.path.join(b_path, b_main)) or
-                    os.path.exists(os.path.join(b_upath, b_tasked)) or
-                    os.path.exists(os.path.join(os.path.dirname(b_path), b_tasked))
-                ):
-                    return True
         return False
 
     def path_dwim_relative(self, path, dirname, source, is_role=False):

test/integration/targets/incidental_flatpak_remote/aliases

@@ -1,8 +0,0 @@
-shippable/posix/incidental
-destructive
-skip/aix
-skip/freebsd
-skip/osx
-skip/macos
-skip/rhel
-needs/root

test/integration/targets/incidental_flatpak_remote/meta/main.yml

@@ -1,2 +0,0 @@
-dependencies:
-  - incidental_setup_flatpak_remote

test/integration/targets/incidental_flatpak_remote/tasks/check_mode.yml

@@ -1,101 +0,0 @@
-# - Tests with absent flatpak remote -------------------------------------------
-
-# state=present
-
-- name: Test addition of absent flatpak remote (check mode)
-  flatpak_remote:
-    name: flatpak-test
-    flatpakrepo_url: /tmp/flatpak/repo/dummy-repo.flatpakrepo
-    state: present
-  register: addition_result
-  check_mode: true
-
-- name: Verify addition of absent flatpak remote test result (check mode)
-  assert:
-    that:
-      - "addition_result.changed == true"
-    msg: "Adding an absent flatpak remote shall mark module execution as changed"
-
-- name: Test non-existent idempotency of addition of absent flatpak remote (check mode)
-  flatpak_remote:
-    name: flatpak-test
-    flatpakrepo_url: /tmp/flatpak/repo/dummy-repo.flatpakrepo
-    state: present
-  register: double_addition_result
-  check_mode: true
-
-- name: >
-      Verify non-existent idempotency of addition of absent flatpak remote
-      test result (check mode)
-  assert:
-    that:
-      - "double_addition_result.changed == true"
-    msg: |
-        Adding an absent flatpak remote a second time shall still mark module execution
-        as changed in check mode
-
-# state=absent
-
-- name: Test removal of absent flatpak remote not doing anything in check mode
-  flatpak_remote:
-    name: flatpak-test
-    state: absent
-  register: removal_result
-  check_mode: true
-
-- name: Verify removal of absent flatpak remote test result (check mode)
-  assert:
-    that:
-      - "removal_result.changed == false"
-    msg: "Removing an absent flatpak remote shall mark module execution as not changed"
-
-
-# - Tests with present flatpak remote -------------------------------------------
-
-# state=present
-
-- name: Test addition of present flatpak remote (check mode)
-  flatpak_remote:
-    name: check-mode-test-remote
-    flatpakrepo_url: /tmp/flatpak/repo/dummy-repo.flatpakrepo
-    state: present
-  register: addition_result
-  check_mode: true
-
-- name: Verify addition of present flatpak remote test result (check mode)
-  assert:
-    that:
-      - "addition_result.changed == false"
-    msg: "Adding a present flatpak remote shall mark module execution as not changed"
-
-# state=absent
-
-- name: Test removal of present flatpak remote not doing anything in check mode
-  flatpak_remote:
-    name: check-mode-test-remote
-    state: absent
-  register: removal_result
-  check_mode: true
-
-- name: Verify removal of present flatpak remote test result (check mode)
-  assert:
-    that:
-      - "removal_result.changed == true"
-    msg: "Removing a present flatpak remote shall mark module execution as changed"
-
-- name: Test non-existent idempotency of removal of present flatpak remote (check mode)
-  flatpak_remote:
-    name: check-mode-test-remote
-    state: absent
-  register: double_removal_result
-  check_mode: true
-
-- name: >
-      Verify non-existent idempotency of removal of present flatpak remote
-      test result (check mode)
-  assert:
-    that:
-      - "double_removal_result.changed == true"
-    msg: |
-        Removing a present flatpak remote a second time shall still mark module execution
-        as changed in check mode

test/integration/targets/incidental_flatpak_remote/tasks/main.yml

@@ -1,57 +0,0 @@
-# (c) 2018, Alexander Bethke <oolongbrothers@gmx.net>
-# (c) 2018, Ansible Project
-
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-- block:
-
-  - import_tasks: setup.yml
-    become: true
-
-  # executable override
-
-  - name: Test executable override
-    flatpak_remote:
-      name: irrelevant
-      remote: irrelevant
-      state: present
-      executable: nothing-that-exists
-    ignore_errors: true
-    register: executable_override_result
-
-  - name: Verify executable override test result
-    assert:
-      that:
-        - "executable_override_result.failed == true"
-        - "executable_override_result.changed == false"
-      msg: "Specifying non-existing executable shall fail module execution"
-
-  - import_tasks: check_mode.yml
-    become: false
-
-  - import_tasks: test.yml
-    become: false
-    vars:
-      method: user
-
-  - import_tasks: test.yml
-    become: true
-    vars:
-      method: system
-
-  when: |
-      ansible_distribution == 'Fedora' or
-      ansible_distribution == 'Ubuntu' and not ansible_distribution_major_version | int < 16

test/integration/targets/incidental_flatpak_remote/tasks/setup.yml

@@ -1,27 +0,0 @@
-- name: Install flatpak on Fedora
-  dnf:
-    name: flatpak
-    state: present
-
-  when: ansible_distribution == 'Fedora'
-
-- block:
-  - name: Activate flatpak ppa on Ubuntu versions older than 18.04/bionic
-    apt_repository:
-      repo: "ppa:alexlarsson/flatpak"
-      state: present
-      mode: 0644
-    when: ansible_lsb.major_release | int < 18
-
-  - name: Install flatpak package on Ubuntu
-    apt:
-      name: flatpak
-      state: present
-
-  when: ansible_distribution == 'Ubuntu'
-
-- name: Install flatpak remote for testing check mode
-  flatpak_remote:
-    name: check-mode-test-remote
-    flatpakrepo_url: /tmp/flatpak/repo/dummy-repo.flatpakrepo
-    state: present

test/integration/targets/incidental_flatpak_remote/tasks/test.yml

@@ -1,72 +0,0 @@
-# state=present
-
-- name: Test addition - {{ method }}
-  flatpak_remote:
-    name: flatpak-test
-    flatpakrepo_url: /tmp/flatpak/repo/dummy-repo.flatpakrepo
-    state: present
-    method: "{{ method }}"
-  register: addition_result
-
-- name: Verify addition test result - {{ method }}
-  assert:
-    that:
-      - "addition_result.changed == true"
-    msg: "state=preset shall add flatpak when absent"
-
-- name: Test idempotency of addition - {{ method }}
-  flatpak_remote:
-    name: flatpak-test
-    flatpakrepo_url: /tmp/flatpak/repo/dummy-repo.flatpakrepo
-    state: present
-    method: "{{ method }}"
-  register: double_addition_result
-
-- name: Verify idempotency of addition test result - {{ method }}
-  assert:
-    that:
-      - "double_addition_result.changed == false"
-    msg: "state=present shall not do anything when flatpak is already present"
-
-- name: Test updating remote url does not do anything - {{ method }}
-  flatpak_remote:
-    name: flatpak-test
-    flatpakrepo_url: https://a.different/repo.flatpakrepo
-    state: present
-    method: "{{ method }}"
-  register: url_update_result
-
-- name: Verify updating remote url does not do anything - {{ method }}
-  assert:
-    that:
-      - "url_update_result.changed == false"
-    msg: "Trying to update the URL of an existing flatpak remote shall not do anything"
-
-
-# state=absent
-
-- name: Test removal - {{ method }}
-  flatpak_remote:
-    name: flatpak-test
-    state: absent
-    method: "{{ method }}"
-  register: removal_result
-
-- name: Verify removal test result - {{ method }}
-  assert:
-    that:
-      - "removal_result.changed == true"
-    msg: "state=absent shall remove flatpak when present"
-
-- name: Test idempotency of removal - {{ method }}
-  flatpak_remote:
-    name: flatpak-test
-    state: absent
-    method: "{{ method }}"
-  register: double_removal_result
-
-- name: Verify idempotency of removal test result - {{ method }}
-  assert:
-    that:
-      - "double_removal_result.changed == false"
-    msg: "state=absent shall not do anything when flatpak is not present"

test/integration/targets/incidental_setup_flatpak_remote/README.md

@@ -1,138 +0,0 @@
-# Create a dummy flatpak repository remote
-
-This document describes how to create a local flatpak dummy repo. Just like the one contained in the `files/repo.tar.gxz` archive.
-
-
-## Create a hello world app
-
-Prerequisites:
-
- - flathub
-
-Prepare the environment:
-
-```
-flatpak install --system flathub org.freedesktop.Platform//1.6 org.freedesktop.Sdk//1.6
-```
-
-Create a hello world executable:
-
-```
-echo $'#!/bin/sh\necho hello world' > hello.sh
-```
-
-To create dummy flatpaks, run this (defining a unique NUM for every flatpak to add):
-
-```
-export NUM=1
-flatpak build-init appdir$NUM com.dummy.App$NUM org.freedesktop.Sdk org.freedesktop.Platform 1.6;
-flatpak build appdir$NUM mkdir /app/bin;
-flatpak build appdir$NUM install --mode=750 hello.sh /app/bin;
-flatpak build-finish --command=hello.sh appdir$NUM
-```
-
-## Create a repo and/or add the app to it
-
-Create a repo and add the file to it in one command:
-
-```
-flatpak build-export repo appdir$NUM stable
-```
-
-## Create flatpak*-files
-
-Put a flatpakref file under the repo folder (`repo/com.dummy.App1.flatpakref`):
-
-```
-[Flatpak Ref]
-Title=Dummy App$NUM
-Name=com.dummy.App$NUM
-Branch=stable
-Url=file:///tmp/flatpak/repo
-GPGKey={{ base64-encoded public KEY }}
-IsRuntime=false
-RuntimeRepo=https://flathub.org/repo/flathub.flatpakrepo
-```
-
-Add a `.flatpakrepo` file to the `repo` folder (`repo/dummy-repo.flatpakrepo`):
-
-```
-[Flatpak Repo]
-Title=Dummy Repo
-Url=file:///tmp/flatpak/repo
-Comment=Dummy repo for ansible module integration testing
-Description=Dummy repo for ansible module integration testing
-GPGKey={{ base64-encoded public KEY }}
-```
-
-## Sign the repo
-
-Create a new key in a new gpg home folder (On RedHat systems, the executable needs to addressed as gpg2):
-
-```
-mkdir gpg
-gpg --homedir gpg --quick-gen-key test@dummy.com
-```
-
-Sign the repo and summary file, you need to redo this when you update the repository:
-
-```
-flatpak build-sign repo --gpg-sign=KEY_ID --gpg-homedir=gpg
-flatpak build-update-repo repo --gpg-sign=KEY_ID --gpg-homedir=gpg
-```
-
-Export the public key as a file:
-
-```
-gpg --homedir=gpg --export KEY_ID > dummy-repo.gpg
-```
-
-Create base64-encoded string from gpg-file for `GPGKey=` property in flatpak*-files:
-
-```
-base64 dummy-repo.gpg | tr -d '\n'
-```
-
-## How to use the repo
-
-Now you can add the `repo` folder as a local repo:
-
-```
-flatpak --system remote-add --gpg-import=/tmp/flatpak/repo/dummy-repo.gpg dummy-repo /tmp/flatpak/repo
-```
-
-Or, via `.flatpakrepo` file:
-
-```
-flatpak --system remote-add dummy-repo /tmp/flatpak/repo/dummy-repo.flatpakrepo
-```
-
-And install the hello world flatpaks like this:
-
-```
-flatpak --system install dummy-repo com.dummy.App$NUM
-```
-
-Or from flatpakref:
-
-```
-flatpak --system install --from /tmp/flatpak/repo/com.dummy.App$NUM.flatpakref
-```
-
-Run the app:
-
-```
-flatpak run com.dummy.App$NUM
-```
-
-To install an app without any runtime dependencies (the app will be broken, but it is enough to test flatpak installation):
-
-```
-flatpak --system install --no-deps dummy-repo com.dummy.App$NUM
-```
-
-## Sources:
-
-* https://blogs.gnome.org/alexl/2017/02/10/maintaining-a-flatpak-repository/
-
-* http://docs.flatpak.org/en/latest/first-build.html

test/integration/targets/incidental_setup_flatpak_remote/aliases

@@ -1 +0,0 @@
-hidden

test/integration/targets/incidental_setup_flatpak_remote/handlers/main.yaml

@@ -1,4 +0,0 @@
-- name: remove temporary flatpak link
-  file:
-    state: absent
-    path: /tmp/flatpak

test/integration/targets/incidental_setup_flatpak_remote/meta/main.yaml

@@ -1,2 +0,0 @@
-dependencies:
-   - setup_remote_tmp_dir

test/integration/targets/incidental_setup_flatpak_remote/tasks/main.yaml

@@ -1,22 +0,0 @@
-- name: Set up dummy flatpak repository remote
-  block:
-
-    - name: Copy repo into place
-      unarchive:
-        src: repo.tar.xz
-        dest: "{{ remote_tmp_dir }}"
-        owner: root
-        group: root
-        mode: 0644
-
-    - name: Create deterministic link to temp directory
-      file:
-        state: link
-        src: "{{ remote_tmp_dir }}/"
-        path: "/tmp/flatpak"
-        owner: root
-        group: root
-        mode: 0644
-      notify: remove temporary flatpak link
-
-  become: true

test/support/integration/plugins/modules/flatpak_remote.py

@@ -1,243 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-# Copyright: (c) 2017 John Kwiatkoski (@JayKayy) <jkwiat40@gmail.com>
-# Copyright: (c) 2018 Alexander Bethke (@oolongbrothers) <oolongbrothers@gmx.net>
-# Copyright: (c) 2017 Ansible Project
-# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-
-
-# ATTENTION CONTRIBUTORS!
-#
-# TL;DR: Run this module's integration tests manually before opening a pull request
-#
-# Long explanation:
-# The integration tests for this module are currently NOT run on the Ansible project's continuous
-# delivery pipeline. So please: When you make changes to this module, make sure that you run the
-# included integration tests manually for both Python 2 and Python 3:
-#
-#   Python 2:
-#       ansible-test integration -v --docker fedora28 --docker-privileged --allow-unsupported --python 2.7 flatpak_remote
-#   Python 3:
-#       ansible-test integration -v --docker fedora28 --docker-privileged --allow-unsupported --python 3.6 flatpak_remote
-#
-# Because of external dependencies, the current integration tests are somewhat too slow and brittle
-# to be included right now. I have plans to rewrite the integration tests based on a local flatpak
-# repository so that they can be included into the normal CI pipeline.
-# //oolongbrothers
-
-
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-ANSIBLE_METADATA = {'metadata_version': '1.1',
-                    'status': ['preview'],
-                    'supported_by': 'community'}
-
-DOCUMENTATION = r'''
----
-module: flatpak_remote
-version_added: '2.6'
-short_description: Manage flatpak repository remotes
-description:
-- Allows users to add or remove flatpak remotes.
-- The flatpak remotes concept is comparable to what is called repositories in other packaging
-  formats.
-- Currently, remote addition is only supported via I(flatpakrepo) file URLs.
-- Existing remotes will not be updated.
-- See the M(flatpak) module for managing flatpaks.
-author:
-- John Kwiatkoski (@JayKayy)
-- Alexander Bethke (@oolongbrothers)
-requirements:
-- flatpak
-options:
-  executable:
-    description:
-    - The path to the C(flatpak) executable to use.
-    - By default, this module looks for the C(flatpak) executable on the path.
-    default: flatpak
-  flatpakrepo_url:
-    description:
-    - The URL to the I(flatpakrepo) file representing the repository remote to add.
-    - When used with I(state=present), the flatpak remote specified under the I(flatpakrepo_url)
-      is added using the specified installation C(method).
-    - When used with I(state=absent), this is not required.
-    - Required when I(state=present).
-  method:
-    description:
-    - The installation method to use.
-    - Defines if the I(flatpak) is supposed to be installed globally for the whole C(system)
-      or only for the current C(user).
-    choices: [ system, user ]
-    default: system
-  name:
-    description:
-    - The desired name for the flatpak remote to be registered under on the managed host.
-    - When used with I(state=present), the remote will be added to the managed host under
-      the specified I(name).
-    - When used with I(state=absent) the remote with that name will be removed.
-    required: true
-  state:
-    description:
-    - Indicates the desired package state.
-    choices: [ absent, present ]
-    default: present
-'''
-
-EXAMPLES = r'''
-- name: Add the Gnome flatpak remote to the system installation
-  flatpak_remote:
-    name: gnome
-    state: present
-    flatpakrepo_url: https://sdk.gnome.org/gnome-apps.flatpakrepo
-
-- name: Add the flathub flatpak repository remote to the user installation
-  flatpak_remote:
-    name: flathub
-    state: present
-    flatpakrepo_url: https://dl.flathub.org/repo/flathub.flatpakrepo
-    method: user
-
-- name: Remove the Gnome flatpak remote from the user installation
-  flatpak_remote:
-    name: gnome
-    state: absent
-    method: user
-
-- name: Remove the flathub remote from the system installation
-  flatpak_remote:
-    name: flathub
-    state: absent
-'''
-
-RETURN = r'''
-command:
-  description: The exact flatpak command that was executed
-  returned: When a flatpak command has been executed
-  type: str
-  sample: "/usr/bin/flatpak remote-add --system flatpak-test https://dl.flathub.org/repo/flathub.flatpakrepo"
-msg:
-  description: Module error message
-  returned: failure
-  type: str
-  sample: "Executable '/usr/local/bin/flatpak' was not found on the system."
-rc:
-  description: Return code from flatpak binary
-  returned: When a flatpak command has been executed
-  type: int
-  sample: 0
-stderr:
-  description: Error output from flatpak binary
-  returned: When a flatpak command has been executed
-  type: str
-  sample: "error: GPG verification enabled, but no summary found (check that the configured URL in remote config is correct)\n"
-stdout:
-  description: Output from flatpak binary
-  returned: When a flatpak command has been executed
-  type: str
-  sample: "flathub\tFlathub\thttps://dl.flathub.org/repo/\t1\t\n"
-'''
-
-import subprocess
-from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils._text import to_bytes, to_native
-
-
-def add_remote(module, binary, name, flatpakrepo_url, method):
-    """Add a new remote."""
-    global result
-    command = "{0} remote-add --{1} {2} {3}".format(
-        binary, method, name, flatpakrepo_url)
-    _flatpak_command(module, module.check_mode, command)
-    result['changed'] = True
-
-
-def remove_remote(module, binary, name, method):
-    """Remove an existing remote."""
-    global result
-    command = "{0} remote-delete --{1} --force {2} ".format(
-        binary, method, name)
-    _flatpak_command(module, module.check_mode, command)
-    result['changed'] = True
-
-
-def remote_exists(module, binary, name, method):
-    """Check if the remote exists."""
-    command = "{0} remote-list -d --{1}".format(binary, method)
-    # The query operation for the remote needs to be run even in check mode
-    output = _flatpak_command(module, False, command)
-    for line in output.splitlines():
-        listed_remote = line.split()
-        if len(listed_remote) == 0:
-            continue
-        if listed_remote[0] == to_native(name):
-            return True
-    return False
-
-
-def _flatpak_command(module, noop, command):
-    global result
-    if noop:
-        result['rc'] = 0
-        result['command'] = command
-        return ""
-
-    process = subprocess.Popen(
-        command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout_data, stderr_data = process.communicate()
-    result['rc'] = process.returncode
-    result['command'] = command
-    result['stdout'] = stdout_data
-    result['stderr'] = stderr_data
-    if result['rc'] != 0:
-        module.fail_json(msg="Failed to execute flatpak command", **result)
-    return to_native(stdout_data)
-
-
-def main():
-    module = AnsibleModule(
-        argument_spec=dict(
-            name=dict(type='str', required=True),
-            flatpakrepo_url=dict(type='str'),
-            method=dict(type='str', default='system',
-                        choices=['user', 'system']),
-            state=dict(type='str', default="present",
-                       choices=['absent', 'present']),
-            executable=dict(type='str', default="flatpak")
-        ),
-        # This module supports check mode
-        supports_check_mode=True,
-    )
-
-    name = module.params['name']
-    flatpakrepo_url = module.params['flatpakrepo_url']
-    method = module.params['method']
-    state = module.params['state']
-    executable = module.params['executable']
-    binary = module.get_bin_path(executable, None)
-
-    if flatpakrepo_url is None:
-        flatpakrepo_url = ''
-
-    global result
-    result = dict(
-        changed=False
-    )
-
-    # If the binary was not found, fail the operation
-    if not binary:
-        module.fail_json(msg="Executable '%s' was not found on the system." % executable, **result)
-
-    remote_already_exists = remote_exists(module, binary, to_bytes(name), method)
-
-    if state == 'present' and not remote_already_exists:
-        add_remote(module, binary, name, flatpakrepo_url, method)
-    elif state == 'absent' and remote_already_exists:
-        remove_remote(module, binary, name, method)
-
-    module.exit_json(**result)
-
-
-if __name__ == '__main__':
-    main()