path: root/debian/tests/ansible-test-integration.py

#!/usr/bin/python3

import argparse
import glob
import os
import subprocess
import sys

# helper function to print log messages depending on verbosity
def log (level, *msg):
    if args.verbose >= level:
        print(' '.join([str(x) for x in msg]), flush=True)

sys.dont_write_bytecode = True

# helper function to run and log processes
def runprog (name, progargs, exit_on_failure=True):
    log(1, 'Running', name)
    proc = subprocess.run(
        progargs,
        timeout = 300,
        capture_output = True,
        text = True,
    )
    if proc.returncode != 0:
        log(0,"#"*72)
        log(0,'Running', name, 'failed!')
        log(0,"Returncode:", proc.returncode)
        log(1,"#### STDOUT ####")
        log(1, proc.stdout)
        log(1, "#### STDERR ####")
        log(1, proc.stderr)
        log(0,"#"*72)
        if exit_on_failure == True:
            exit(proc.returncode)
    return proc

def locale_debug(name):
    if args.verbose >= 3:
        log(2, name)
        log(2, 'output of /usr/bin/locale:')
        subprocess.run(['/usr/bin/locale'])
        prog = runprog('cat /etc/default/locale', ['cat', '/etc/default/locale'])
        log(2, prog.stdout, prog.stderr)
        # grep will exit 1 when it doesn't return anything, so don't fail on it.
        prog = runprog('grep -v -P \'^#|^$\' /etc/locale.gen', ['grep', '-v', '-P', '^#|^$', '/etc/locale.gen'], exit_on_failure=False)
        log(2, prog.stdout, prog.stderr)

parser = argparse.ArgumentParser(
    prog='ansible-test-integration.py',
    description='python script to run ansible-test integration against the Debian source package',
)

# Whether to run the default tests not in any other list
parser.add_argument(
    '--default-tests',
    action=argparse.BooleanOptionalAction,
    default=True,
    help='Run the default tests not listed anywhere else. (default: yes)',
)

parser.add_argument(
    '--requires-root',
    action=argparse.BooleanOptionalAction,
    default=False,
    help='Run tests that require root. (default: no)',
)

parser.add_argument(
    '--requires-ssh',
    action=argparse.BooleanOptionalAction,
    default=False,
    help='Run tests that require a specially configured SSH server. (default: no)'
)

parser.add_argument(
    '--requires-apt-mark-manual',
    action=argparse.BooleanOptionalAction,
    default=True,
    help='Run tests that do "apt-mark manual" on certain packages. (default: yes)',
)

parser.add_argument(
    '--fails-on-pip',
    action=argparse.BooleanOptionalAction,
    default=False,
    help='Run tests that run "pip3 install" on certain modules. (default: no)',
)

parser.add_argument(
    '--failing',
    action=argparse.BooleanOptionalAction,
    default=False,
    help='Run tests that fail on other reasons. (default: no)',
)

parser.add_argument(
    '--setup',
    action=argparse.BooleanOptionalAction,
    default=True,
    help='Setup testbed via sudo. (default: yes)',
)

parser.add_argument(
    '--dry-run',
    action=argparse.BooleanOptionalAction,
    default=False,
    help='Print the list of targets without actually running them',
)

parser.add_argument(
    '--verbose', '-v',
    action='count',
    default=1,
    help='verbosity between 0 and 5. 0 will only emit errors. 1=INFO. More for increasing levels of debug info. Defaults to 1.',
)

args = parser.parse_args()

locale_debug('locale before setting os.environ:')
os.environ['LANG'] = 'en_US.UTF-8'
locale_debug('locale after setting os.environ:')

if args.setup == True:
    proc = runprog('testbed-setup.sh', ['sudo', './debian/tests/testbed-setup.sh'])
    log(2,"#### STDOUT ####")
    log(2, proc.stdout)
    log(2, "#### STDERR ####")
    log(2, proc.stderr)
    locale_debug('locale after running testbed-setup.sh:')

# integration tests requiring root in some form
integration_requires_root = {
    'ansible-vault',    # ignores --local and tries to use venv
    'apt_key',          # add/removes apt keys
    'blockinfile',      # setup_remote_tmp_dir handler fails (hidden output)
    'callback_default', # checks for an error that has root's homedir
    'debconf',          # Writes to debconf database
    'gathering',        # writes to /etc/ansible/facts.d/
    'group',            # wants to add/remove systemd groups
    'keyword_inheritance', # requires sudo
    #'module_defaults',  # requires sudo
    'noexec',           # calls mount
    #'omit',             # requires sudo
    'systemd',          # disables/enables services
}

# integration tests requiring a running ssh server
integration_requires_ssh = {
    'become_unprivileged',
    'cli',
    'connection_paramiko_ssh',
    'connection_ssh',
    'delegate_to',
    'fetch',
    'module_tracebacks',
}

# integration tests requiring root because the apt module is used to
# install missing packages, or to mark packages as manually installed
integration_requires_apt_mark_manual = {
    'ansible-galaxy-collection-scm',   # apt-mark manual git
    'ansible-pull',                    # apt-mark manual git
    #'debconf',                         # apt-mark manual debconf-utils
    'iptables',                        # apt-mark manual iptables
    'git',                             # apt-mark manual git
}

integration_fails_on_pip = {
    'ansible-galaxy-collection-cli',   # fails on pip
    'ansible-inventory',               # pip error: externally-managed-environment ## upstream fix
    'builtin_vars_prompt',             # passlib: pip error: externally-managed-environment
    'debugger',                        # pip installs pexpect
    'pause',                           # pip installs pexpect
}

integration_failing = {
    'ansible-galaxy-role':            'dict object has no attribute lnk_source',  ## needs upstream fix?
    'ansible-test-docker':            "pwsh doesn't exist in Debian yet",
    'ansible-test':                   'installs and runs python libs from remote',
    'ansible-test-sanity':            'checks are only valid for the source tree',
    'ansible-test-units-forked':      '?????',
    'facts_d':                        'seems to read an unreadable problem without error', ## needs upstream fix
    'infra':                          'requires hacking/test-module.py not present',
    'interpreter_discovery_python':   'detects /usr/bin/python3.11, expect python3, detects os_version 12.6, expects it to compare > 10',
#    'preflight_encoding':             'fails due to missing en_US.UTF-8', # workaround in testbed-setup.sh
    'remote_tmp':                     'Will often show false positive on: "Test tempdir is removed", needs upstream fixing',
    'service_facts':                  "Version comparison failed: '<' not supported between instances of 'str' and 'int'",    # writes to /usr/sbin/
#    'tags':                           'fails due to missing en_US.UTF-8', # workaround in testbed-setup.sh
    'template_jinja2_non_native':     'no need to test against latest jinja2',
}

# work around autopkgtest providing the source tree owned by a different user
runprog('git config hack', ['git', 'config', '--global', '--add', 'safe.directory', '*'])

pyver = str(sys.version_info.major) + '.' + str(sys.version_info.minor)

overall_test_rc = 0
failed_tests = []
succeeded_tests = []

# retrieve a list of all integration tests
all_targets_cmd = runprog(
    'ansible-test to retrieve list of targets',
    ['./bin/ansible-test', 'integration', '--list-targets'],
)

# Compile list of all targets
all_targets = set(all_targets_cmd.stdout.splitlines())

default_targets = list(
    all_targets
    - integration_requires_root
    - integration_requires_ssh
    - integration_requires_apt_mark_manual
    - integration_fails_on_pip
    - set(integration_failing.keys())
)

# compile a list of targets to run, depending on CLI parameters
targets = []
skipped = []

if args.default_tests == True:
    targets.extend(default_targets)
else:
    skipped.extend(default_targets)

if args.requires_root == True:
    targets.extend(integration_requires_root)
else:
    skipped.extend(integration_requires_root)

if args.requires_ssh == True:
    targets.extend(integration_requires_ssh)
else:
    skipped.extend(integration_requires_ssh)

if args.requires_apt_mark_manual == True:
    targets.extend(integration_requires_apt_mark_manual)
else:
    skipped.extend(integration_requires_apt_mark_manual)

if args.fails_on_pip == True:
    targets.extend(integration_fails_on_pip)
else:
    skipped.extend(integration_fails_on_pip)

if args.failing == True:
    targets.extend(integration_failing)
else:
    skipped.extend(integration_failing)

targets.sort()
skipped.sort()

for i in targets:

    if args.dry_run == True:
        log(1, 'Would run ansible-test in', i)
        skipped.append(i)
        continue

    print ("\n" + "#"*72, flush=True)
    print ("#### Running integration tests in", i, flush=True)
    print ("#"*72, flush=True)

    proc = subprocess.run([
        './bin/ansible-test',
        'integration',
        '--python-interpreter',
        '/usr/bin/python3',
        '--python',
        pyver,
        '--local',
        '--color', 'yes',
        i
    ])


    if proc.returncode != 0:
        failed_tests.append(i)
        overall_test_rc = proc.returncode
    else:
        succeeded_tests.append(i)

if overall_test_rc != 0:
    print ("#"*72, flush=True)
    print ("#### failed tests are:", flush=True)
    for i in failed_tests:
        print ("####", i, flush=True)
    print ("#"*72, flush=True)

log(1, '### TEST SUMMARY ###')
log(1, 'succeeded tests:', len(succeeded_tests))
log(1, 'failed tests:', len(failed_tests))
log(1, 'skipped tests:', len(skipped))

log(2, '### succeed test list:')
for i in succeeded_tests:
    log(2, i)
log(2, '### failed test list:')
for i in failed_tests:
    log(2, i)
log(2, '### skipped test list:')
for i in skipped:
    log(2, i)

exit(overall_test_rc)