Google
Edit File: module_common.py
# (c) 2013-2014, Michael DeHaan <michael.dehaan@gmail.com> # (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com> # # This file is part of Ansible # # Ansible is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Ansible is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. # Make coding more python3-ish from __future__ import (absolute_import, division, print_function) __metaclass__ = type import ast import base64 import datetime import json import os import shlex import zipfile import re import pkgutil from ast import AST, Import, ImportFrom from io import BytesIO from ansible.release import __version__, __author__ from ansible import constants as C from ansible.errors import AnsibleError from ansible.executor.interpreter_discovery import InterpreterDiscoveryRequiredError from ansible.executor.powershell import module_manifest as ps_manifest from ansible.module_utils.common.json import AnsibleJSONEncoder from ansible.module_utils.common.text.converters import to_bytes, to_text, to_native from ansible.plugins.loader import module_utils_loader from ansible.utils.collection_loader._collection_finder import _get_collection_metadata, _nested_dict_get # Must import strategy and use write_locks from there # If we import write_locks directly then we end up binding a # variable to the object and then it never gets updated. from ansible.executor import action_write_locks from ansible.utils.display import Display from collections import namedtuple import importlib.util import importlib.machinery display = Display() ModuleUtilsProcessEntry = namedtuple('ModuleUtilsProcessEntry', ['name_parts', 'is_ambiguous', 'has_redirected_child', 'is_optional']) REPLACER = b"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>" REPLACER_VERSION = b"\"<<ANSIBLE_VERSION>>\"" REPLACER_COMPLEX = b"\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\"" REPLACER_WINDOWS = b"# POWERSHELL_COMMON" REPLACER_JSONARGS = b"<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>" REPLACER_SELINUX = b"<<SELINUX_SPECIAL_FILESYSTEMS>>" # We could end up writing out parameters with unicode characters so we need to # specify an encoding for the python source file ENCODING_STRING = u'# -*- coding: utf-8 -*-' b_ENCODING_STRING = b'# -*- coding: utf-8 -*-' # module_common is relative to module_utils, so fix the path _MODULE_UTILS_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils') # ****************************************************************************** ANSIBALLZ_TEMPLATE = u'''%(shebang)s %(coding)s _ANSIBALLZ_WRAPPER = True # For test-module.py script to tell this is a ANSIBALLZ_WRAPPER # This code is part of Ansible, but is an independent component. # The code in this particular templatable string, and this templatable string # only, is BSD licensed. Modules which end up using this snippet, which is # dynamically combined together by Ansible still belong to the author of the # module, and they may assign their own license to the complete work. # # Copyright (c), James Cammarata, 2016 # Copyright (c), Toshio Kuratomi, 2016 # # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. def _ansiballz_main(): import os import os.path # Access to the working directory is required by Python when using pipelining, as well as for the coverage module. # Some platforms, such as macOS, may not allow querying the working directory when using become to drop privileges. try: os.getcwd() except OSError: try: os.chdir(os.path.expanduser('~')) except OSError: os.chdir('/') %(rlimit)s import sys import __main__ # For some distros and python versions we pick up this script in the temporary # directory. This leads to problems when the ansible module masks a python # library that another import needs. We have not figured out what about the # specific distros and python versions causes this to behave differently. # # Tested distros: # Fedora23 with python3.4 Works # Ubuntu15.10 with python2.7 Works # Ubuntu15.10 with python3.4 Fails without this # Ubuntu16.04.1 with python3.5 Fails without this # To test on another platform: # * use the copy module (since this shadows the stdlib copy module) # * Turn off pipelining # * Make sure that the destination file does not exist # * ansible ubuntu16-test -m copy -a 'src=/etc/motd dest=/var/tmp/m' # This will traceback in shutil. Looking at the complete traceback will show # that shutil is importing copy which finds the ansible module instead of the # stdlib module scriptdir = None try: scriptdir = os.path.dirname(os.path.realpath(__main__.__file__)) except (AttributeError, OSError): # Some platforms don't set __file__ when reading from stdin # OSX raises OSError if using abspath() in a directory we don't have # permission to read (realpath calls abspath) pass # Strip cwd from sys.path to avoid potential permissions issues excludes = set(('', '.', scriptdir)) sys.path = [p for p in sys.path if p not in excludes] import base64 import runpy import shutil import tempfile import zipfile if sys.version_info < (3,): PY3 = False else: PY3 = True ZIPDATA = """%(zipdata)s""" # Note: temp_path isn't needed once we switch to zipimport def invoke_module(modlib_path, temp_path, json_params): # When installed via setuptools (including python setup.py install), # ansible may be installed with an easy-install.pth file. That file # may load the system-wide install of ansible rather than the one in # the module. sitecustomize is the only way to override that setting. z = zipfile.ZipFile(modlib_path, mode='a') # py3: modlib_path will be text, py2: it's bytes. Need bytes at the end sitecustomize = u'import sys\\nsys.path.insert(0,"%%s")\\n' %% modlib_path sitecustomize = sitecustomize.encode('utf-8') # Use a ZipInfo to work around zipfile limitation on hosts with # clocks set to a pre-1980 year (for instance, Raspberry Pi) zinfo = zipfile.ZipInfo() zinfo.filename = 'sitecustomize.py' zinfo.date_time = ( %(year)i, %(month)i, %(day)i, %(hour)i, %(minute)i, %(second)i) z.writestr(zinfo, sitecustomize) z.close() # Put the zipped up module_utils we got from the controller first in the python path so that we # can monkeypatch the right basic sys.path.insert(0, modlib_path) # Monkeypatch the parameters into basic from ansible.module_utils import basic basic._ANSIBLE_ARGS = json_params %(coverage)s # Run the module! By importing it as '__main__', it thinks it is executing as a script runpy.run_module(mod_name='%(module_fqn)s', init_globals=dict(_module_fqn='%(module_fqn)s', _modlib_path=modlib_path), run_name='__main__', alter_sys=True) # Ansible modules must exit themselves print('{"msg": "New-style module did not handle its own exit", "failed": true}') sys.exit(1) def debug(command, zipped_mod, json_params): # The code here normally doesn't run. It's only used for debugging on the # remote machine. # # The subcommands in this function make it easier to debug ansiballz # modules. Here's the basic steps: # # Run ansible with the environment variable: ANSIBLE_KEEP_REMOTE_FILES=1 and -vvv # to save the module file remotely:: # $ ANSIBLE_KEEP_REMOTE_FILES=1 ansible host1 -m ping -a 'data=october' -vvv # # Part of the verbose output will tell you where on the remote machine the # module was written to:: # [...] # <host1> SSH: EXEC ssh -C -q -o ControlMaster=auto -o ControlPersist=60s -o KbdInteractiveAuthentication=no -o # PreferredAuthentications=gssapi-with-mic,gssapi-keyex,hostbased,publickey -o PasswordAuthentication=no -o ConnectTimeout=10 -o # ControlPath=/home/badger/.ansible/cp/ansible-ssh-%%h-%%p-%%r -tt rhel7 '/bin/sh -c '"'"'LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 # LC_MESSAGES=en_US.UTF-8 /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping'"'"'' # [...] # # Login to the remote machine and run the module file via from the previous # step with the explode subcommand to extract the module payload into # source files:: # $ ssh host1 # $ /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping explode # Module expanded into: # /home/badger/.ansible/tmp/ansible-tmp-1461173408.08-279692652635227/ansible # # You can now edit the source files to instrument the code or experiment with # different parameter values. When you're ready to run the code you've modified # (instead of the code from the actual zipped module), use the execute subcommand like this:: # $ /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping execute # Okay to use __file__ here because we're running from a kept file basedir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'debug_dir') args_path = os.path.join(basedir, 'args') if command == 'explode': # transform the ZIPDATA into an exploded directory of code and then # print the path to the code. This is an easy way for people to look # at the code on the remote machine for debugging it in that # environment z = zipfile.ZipFile(zipped_mod) for filename in z.namelist(): if filename.startswith('/'): raise Exception('Something wrong with this module zip file: should not contain absolute paths') dest_filename = os.path.join(basedir, filename) if dest_filename.endswith(os.path.sep) and not os.path.exists(dest_filename): os.makedirs(dest_filename) else: directory = os.path.dirname(dest_filename) if not os.path.exists(directory): os.makedirs(directory) f = open(dest_filename, 'wb') f.write(z.read(filename)) f.close() # write the args file f = open(args_path, 'wb') f.write(json_params) f.close() print('Module expanded into:') print('%%s' %% basedir) exitcode = 0 elif command == 'execute': # Execute the exploded code instead of executing the module from the # embedded ZIPDATA. This allows people to easily run their modified # code on the remote machine to see how changes will affect it. # Set pythonpath to the debug dir sys.path.insert(0, basedir) # read in the args file which the user may have modified with open(args_path, 'rb') as f: json_params = f.read() # Monkeypatch the parameters into basic from ansible.module_utils import basic basic._ANSIBLE_ARGS = json_params # Run the module! By importing it as '__main__', it thinks it is executing as a script runpy.run_module(mod_name='%(module_fqn)s', init_globals=None, run_name='__main__', alter_sys=True) # Ansible modules must exit themselves print('{"msg": "New-style module did not handle its own exit", "failed": true}') sys.exit(1) else: print('WARNING: Unknown debug command. Doing nothing.') exitcode = 0 return exitcode # # See comments in the debug() method for information on debugging # ANSIBALLZ_PARAMS = %(params)s if PY3: ANSIBALLZ_PARAMS = ANSIBALLZ_PARAMS.encode('utf-8') try: # There's a race condition with the controller removing the # remote_tmpdir and this module executing under async. So we cannot # store this in remote_tmpdir (use system tempdir instead) # Only need to use [ansible_module]_payload_ in the temp_path until we move to zipimport # (this helps ansible-test produce coverage stats) temp_path = tempfile.mkdtemp(prefix='ansible_%(ansible_module)s_payload_') zipped_mod = os.path.join(temp_path, 'ansible_%(ansible_module)s_payload.zip') with open(zipped_mod, 'wb') as modlib: modlib.write(base64.b64decode(ZIPDATA)) if len(sys.argv) == 2: exitcode = debug(sys.argv[1], zipped_mod, ANSIBALLZ_PARAMS) else: # Note: temp_path isn't needed once we switch to zipimport invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS) finally: try: shutil.rmtree(temp_path) except (NameError, OSError): # tempdir creation probably failed pass sys.exit(exitcode) if __name__ == '__main__': _ansiballz_main() ''' ANSIBALLZ_COVERAGE_TEMPLATE = ''' os.environ['COVERAGE_FILE'] = '%(coverage_output)s=python-%%s=coverage' %% '.'.join(str(v) for v in sys.version_info[:2]) import atexit try: import coverage except ImportError: print('{"msg": "Could not import `coverage` module.", "failed": true}') sys.exit(1) cov = coverage.Coverage(config_file='%(coverage_config)s') def atexit_coverage(): cov.stop() cov.save() atexit.register(atexit_coverage) cov.start() ''' ANSIBALLZ_COVERAGE_CHECK_TEMPLATE = ''' try: if PY3: import importlib.util if importlib.util.find_spec('coverage') is None: raise ImportError else: import imp imp.find_module('coverage') except ImportError: print('{"msg": "Could not find `coverage` module.", "failed": true}') sys.exit(1) ''' ANSIBALLZ_RLIMIT_TEMPLATE = ''' import resource existing_soft, existing_hard = resource.getrlimit(resource.RLIMIT_NOFILE) # adjust soft limit subject to existing hard limit requested_soft = min(existing_hard, %(rlimit_nofile)d) if requested_soft != existing_soft: try: resource.setrlimit(resource.RLIMIT_NOFILE, (requested_soft, existing_hard)) except ValueError: # some platforms (eg macOS) lie about their hard limit pass ''' def _strip_comments(source): # Strip comments and blank lines from the wrapper buf = [] for line in source.splitlines(): l = line.strip() if not l or l.startswith(u'#'): continue buf.append(line) return u'\n'.join(buf) if C.DEFAULT_KEEP_REMOTE_FILES: # Keep comments when KEEP_REMOTE_FILES is set. That way users will see # the comments with some nice usage instructions ACTIVE_ANSIBALLZ_TEMPLATE = ANSIBALLZ_TEMPLATE else: # ANSIBALLZ_TEMPLATE stripped of comments for smaller over the wire size ACTIVE_ANSIBALLZ_TEMPLATE = _strip_comments(ANSIBALLZ_TEMPLATE) # dirname(dirname(dirname(site-packages/ansible/executor/module_common.py) == site-packages # Do this instead of getting site-packages from distutils.sysconfig so we work when we # haven't been installed site_packages = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) CORE_LIBRARY_PATH_RE = re.compile(r'%s/(?P<path>ansible/modules/.*)\.(py|ps1)$' % re.escape(site_packages)) COLLECTION_PATH_RE = re.compile(r'/(?P<path>ansible_collections/[^/]+/[^/]+/plugins/modules/.*)\.(py|ps1)$') # Detect new-style Python modules by looking for required imports: # import ansible_collections.[my_ns.my_col.plugins.module_utils.my_module_util] # from ansible_collections.[my_ns.my_col.plugins.module_utils import my_module_util] # import ansible.module_utils[.basic] # from ansible.module_utils[ import basic] # from ansible.module_utils[.basic import AnsibleModule] # from ..module_utils[ import basic] # from ..module_utils[.basic import AnsibleModule] NEW_STYLE_PYTHON_MODULE_RE = re.compile( # Relative imports br'(?:from +\.{2,} *module_utils.* +import |' # Collection absolute imports: br'from +ansible_collections\.[^.]+\.[^.]+\.plugins\.module_utils.* +import |' br'import +ansible_collections\.[^.]+\.[^.]+\.plugins\.module_utils.*|' # Core absolute imports br'from +ansible\.module_utils.* +import |' br'import +ansible\.module_utils\.)' ) class ModuleDepFinder(ast.NodeVisitor): def __init__(self, module_fqn, tree, is_pkg_init=False, *args, **kwargs): """ Walk the ast tree for the python module. :arg module_fqn: The fully qualified name to reach this module in dotted notation. example: ansible.module_utils.basic :arg is_pkg_init: Inform the finder it's looking at a package init (eg __init__.py) to allow relative import expansion to use the proper package level without having imported it locally first. Save submodule[.submoduleN][.identifier] into self.submodules when they are from ansible.module_utils or ansible_collections packages self.submodules will end up with tuples like: - ('ansible', 'module_utils', 'basic',) - ('ansible', 'module_utils', 'urls', 'fetch_url') - ('ansible', 'module_utils', 'database', 'postgres') - ('ansible', 'module_utils', 'database', 'postgres', 'quote') - ('ansible', 'module_utils', 'database', 'postgres', 'quote') - ('ansible_collections', 'my_ns', 'my_col', 'plugins', 'module_utils', 'foo') It's up to calling code to determine whether the final element of the tuple are module names or something else (function, class, or variable names) .. seealso:: :python3:class:`ast.NodeVisitor` """ super(ModuleDepFinder, self).__init__(*args, **kwargs) self._tree = tree # squirrel this away so we can compare node parents to it self.submodules = set() self.optional_imports = set() self.module_fqn = module_fqn self.is_pkg_init = is_pkg_init self._visit_map = { Import: self.visit_Import, ImportFrom: self.visit_ImportFrom, } self.visit(tree) def generic_visit(self, node): """Overridden ``generic_visit`` that makes some assumptions about our use case, and improves performance by calling visitors directly instead of calling ``visit`` to offload calling visitors. """ generic_visit = self.generic_visit visit_map = self._visit_map for field, value in ast.iter_fields(node): if isinstance(value, list): for item in value: if isinstance(item, (Import, ImportFrom)): item.parent = node visit_map[item.__class__](item) elif isinstance(item, AST): generic_visit(item) visit = generic_visit def visit_Import(self, node): """ Handle import ansible.module_utils.MODLIB[.MODLIBn] [as asname] We save these as interesting submodules when the imported library is in ansible.module_utils or ansible.collections """ for alias in node.names: if (alias.name.startswith('ansible.module_utils.') or alias.name.startswith('ansible_collections.')): py_mod = tuple(alias.name.split('.')) self.submodules.add(py_mod) # if the import's parent is the root document, it's a required import, otherwise it's optional if node.parent != self._tree: self.optional_imports.add(py_mod) self.generic_visit(node) def visit_ImportFrom(self, node): """ Handle from ansible.module_utils.MODLIB import [.MODLIBn] [as asname] Also has to handle relative imports We save these as interesting submodules when the imported library is in ansible.module_utils or ansible.collections """ # FIXME: These should all get skipped: # from ansible.executor import module_common # from ...executor import module_common # from ... import executor (Currently it gives a non-helpful error) if node.level > 0: # if we're in a package init, we have to add one to the node level (and make it none if 0 to preserve the right slicing behavior) level_slice_offset = -node.level + 1 or None if self.is_pkg_init else -node.level if self.module_fqn: parts = tuple(self.module_fqn.split('.')) if node.module: # relative import: from .module import x node_module = '.'.join(parts[:level_slice_offset] + (node.module,)) else: # relative import: from . import x node_module = '.'.join(parts[:level_slice_offset]) else: # fall back to an absolute import node_module = node.module else: # absolute import: from module import x node_module = node.module # Specialcase: six is a special case because of its # import logic py_mod = None if node.names[0].name == '_six': self.submodules.add(('_six',)) elif node_module.startswith('ansible.module_utils'): # from ansible.module_utils.MODULE1[.MODULEn] import IDENTIFIER [as asname] # from ansible.module_utils.MODULE1[.MODULEn] import MODULEn+1 [as asname] # from ansible.module_utils.MODULE1[.MODULEn] import MODULEn+1 [,IDENTIFIER] [as asname] # from ansible.module_utils import MODULE1 [,MODULEn] [as asname] py_mod = tuple(node_module.split('.')) elif node_module.startswith('ansible_collections.'): if node_module.endswith('plugins.module_utils') or '.plugins.module_utils.' in node_module: # from ansible_collections.ns.coll.plugins.module_utils import MODULE [as aname] [,MODULE2] [as aname] # from ansible_collections.ns.coll.plugins.module_utils.MODULE import IDENTIFIER [as aname] # FIXME: Unhandled cornercase (needs to be ignored): # from ansible_collections.ns.coll.plugins.[!module_utils].[FOO].plugins.module_utils import IDENTIFIER py_mod = tuple(node_module.split('.')) else: # Not from module_utils so ignore. for instance: # from ansible_collections.ns.coll.plugins.lookup import IDENTIFIER pass if py_mod: for alias in node.names: self.submodules.add(py_mod + (alias.name,)) # if the import's parent is the root document, it's a required import, otherwise it's optional if node.parent != self._tree: self.optional_imports.add(py_mod + (alias.name,)) self.generic_visit(node) def _slurp(path): if not os.path.exists(path): raise AnsibleError("imported module support code does not exist at %s" % os.path.abspath(path)) with open(path, 'rb') as fd: data = fd.read() return data def _get_shebang(interpreter, task_vars, templar, args=tuple(), remote_is_local=False): """ Handles the different ways ansible allows overriding the shebang target for a module. """ # FUTURE: add logical equivalence for python3 in the case of py3-only modules interpreter_name = os.path.basename(interpreter).strip() # name for interpreter var interpreter_config = u'ansible_%s_interpreter' % interpreter_name # key for config interpreter_config_key = "INTERPRETER_%s" % interpreter_name.upper() interpreter_out = None # looking for python, rest rely on matching vars if interpreter_name == 'python': # skip detection for network os execution, use playbook supplied one if possible if remote_is_local: interpreter_out = task_vars['ansible_playbook_python'] # a config def exists for this interpreter type; consult config for the value elif C.config.get_configuration_definition(interpreter_config_key): interpreter_from_config = C.config.get_config_value(interpreter_config_key, variables=task_vars) interpreter_out = templar.template(interpreter_from_config.strip()) # handle interpreter discovery if requested or empty interpreter was provided if not interpreter_out or interpreter_out in ['auto', 'auto_legacy', 'auto_silent', 'auto_legacy_silent']: discovered_interpreter_config = u'discovered_interpreter_%s' % interpreter_name facts_from_task_vars = task_vars.get('ansible_facts', {}) if discovered_interpreter_config not in facts_from_task_vars: # interpreter discovery is desired, but has not been run for this host raise InterpreterDiscoveryRequiredError("interpreter discovery needed", interpreter_name=interpreter_name, discovery_mode=interpreter_out) else: interpreter_out = facts_from_task_vars[discovered_interpreter_config] else: raise InterpreterDiscoveryRequiredError("interpreter discovery required", interpreter_name=interpreter_name, discovery_mode='auto_legacy') elif interpreter_config in task_vars: # for non python we consult vars for a possible direct override interpreter_out = templar.template(task_vars.get(interpreter_config).strip()) if not interpreter_out: # nothing matched(None) or in case someone configures empty string or empty intepreter interpreter_out = interpreter # set shebang shebang = u'#!{0}'.format(interpreter_out) if args: shebang = shebang + u' ' + u' '.join(args) return shebang, interpreter_out class ModuleUtilLocatorBase: def __init__(self, fq_name_parts, is_ambiguous=False, child_is_redirected=False, is_optional=False): self._is_ambiguous = is_ambiguous # a child package redirection could cause intermediate package levels to be missing, eg # from ansible.module_utils.x.y.z import foo; if x.y.z.foo is redirected, we may not have packages on disk for # the intermediate packages x.y.z, so we'll need to supply empty packages for those self._child_is_redirected = child_is_redirected self._is_optional = is_optional self.found = False self.redirected = False self.fq_name_parts = fq_name_parts self.source_code = '' self.output_path = '' self.is_package = False self._collection_name = None # for ambiguous imports, we should only test for things more than one level below module_utils # this lets us detect erroneous imports and redirections earlier if is_ambiguous and len(self._get_module_utils_remainder_parts(fq_name_parts)) > 1: self.candidate_names = [fq_name_parts, fq_name_parts[:-1]] else: self.candidate_names = [fq_name_parts] @property def candidate_names_joined(self): return ['.'.join(n) for n in self.candidate_names] def _handle_redirect(self, name_parts): module_utils_relative_parts = self._get_module_utils_remainder_parts(name_parts) # only allow redirects from below module_utils- if above that, bail out (eg, parent package names) if not module_utils_relative_parts: return False try: collection_metadata = _get_collection_metadata(self._collection_name) except ValueError as ve: # collection not found or some other error related to collection load if self._is_optional: return False raise AnsibleError('error processing module_util {0} loading redirected collection {1}: {2}' .format('.'.join(name_parts), self._collection_name, to_native(ve))) routing_entry = _nested_dict_get(collection_metadata, ['plugin_routing', 'module_utils', '.'.join(module_utils_relative_parts)]) if not routing_entry: return False # FIXME: add deprecation warning support dep_or_ts = routing_entry.get('tombstone') removed = dep_or_ts is not None if not removed: dep_or_ts = routing_entry.get('deprecation') if dep_or_ts: removal_date = dep_or_ts.get('removal_date') removal_version = dep_or_ts.get('removal_version') warning_text = dep_or_ts.get('warning_text') msg = 'module_util {0} has been removed'.format('.'.join(name_parts)) if warning_text: msg += ' ({0})'.format(warning_text) else: msg += '.' display.deprecated(msg, removal_version, removed, removal_date, self._collection_name) if 'redirect' in routing_entry: self.redirected = True source_pkg = '.'.join(name_parts) self.is_package = True # treat all redirects as packages redirect_target_pkg = routing_entry['redirect'] # expand FQCN redirects if not redirect_target_pkg.startswith('ansible_collections'): split_fqcn = redirect_target_pkg.split('.') if len(split_fqcn) < 3: raise Exception('invalid redirect for {0}: {1}'.format(source_pkg, redirect_target_pkg)) # assume it's an FQCN, expand it redirect_target_pkg = 'ansible_collections.{0}.{1}.plugins.module_utils.{2}'.format( split_fqcn[0], # ns split_fqcn[1], # coll '.'.join(split_fqcn[2:]) # sub-module_utils remainder ) display.vvv('redirecting module_util {0} to {1}'.format(source_pkg, redirect_target_pkg)) self.source_code = self._generate_redirect_shim_source(source_pkg, redirect_target_pkg) return True return False def _get_module_utils_remainder_parts(self, name_parts): # subclasses should override to return the name parts after module_utils return [] def _get_module_utils_remainder(self, name_parts): # return the remainder parts as a package string return '.'.join(self._get_module_utils_remainder_parts(name_parts)) def _find_module(self, name_parts): return False def _locate(self, redirect_first=True): for candidate_name_parts in self.candidate_names: if redirect_first and self._handle_redirect(candidate_name_parts): break if self._find_module(candidate_name_parts): break if not redirect_first and self._handle_redirect(candidate_name_parts): break else: # didn't find what we were looking for- last chance for packages whose parents were redirected if self._child_is_redirected: # make fake packages self.is_package = True self.source_code = '' else: # nope, just bail return if self.is_package: path_parts = candidate_name_parts + ('__init__',) else: path_parts = candidate_name_parts self.found = True self.output_path = os.path.join(*path_parts) + '.py' self.fq_name_parts = candidate_name_parts def _generate_redirect_shim_source(self, fq_source_module, fq_target_module): return """ import sys import {1} as mod sys.modules['{0}'] = mod """.format(fq_source_module, fq_target_module) # FIXME: add __repr__ impl class LegacyModuleUtilLocator(ModuleUtilLocatorBase): def __init__(self, fq_name_parts, is_ambiguous=False, mu_paths=None, child_is_redirected=False): super(LegacyModuleUtilLocator, self).__init__(fq_name_parts, is_ambiguous, child_is_redirected) if fq_name_parts[0:2] != ('ansible', 'module_utils'): raise Exception('this class can only locate from ansible.module_utils, got {0}'.format(fq_name_parts)) if fq_name_parts[2] == 'six': # FIXME: handle the ansible.module_utils.six._six case with a redirect or an internal _six attr on six itself? # six creates its submodules at runtime; convert all these to just 'ansible.module_utils.six' fq_name_parts = ('ansible', 'module_utils', 'six') self.candidate_names = [fq_name_parts] self._mu_paths = mu_paths self._collection_name = 'ansible.builtin' # legacy module utils always look in ansible.builtin for redirects self._locate(redirect_first=False) # let local stuff override redirects for legacy def _get_module_utils_remainder_parts(self, name_parts): return name_parts[2:] # eg, foo.bar for ansible.module_utils.foo.bar def _find_module(self, name_parts): rel_name_parts = self._get_module_utils_remainder_parts(name_parts) # no redirection; try to find the module if len(rel_name_parts) == 1: # direct child of module_utils, just search the top-level dirs we were given paths = self._mu_paths else: # a nested submodule of module_utils, extend the paths given with the intermediate package names paths = [os.path.join(p, *rel_name_parts[:-1]) for p in self._mu_paths] # extend the MU paths with the relative bit # find_spec needs the full module name self._info = info = importlib.machinery.PathFinder.find_spec('.'.join(name_parts), paths) if info is not None and os.path.splitext(info.origin)[1] in importlib.machinery.SOURCE_SUFFIXES: self.is_package = info.origin.endswith('/__init__.py') path = info.origin else: return False self.source_code = _slurp(path) return True class CollectionModuleUtilLocator(ModuleUtilLocatorBase): def __init__(self, fq_name_parts, is_ambiguous=False, child_is_redirected=False, is_optional=False): super(CollectionModuleUtilLocator, self).__init__(fq_name_parts, is_ambiguous, child_is_redirected, is_optional) if fq_name_parts[0] != 'ansible_collections': raise Exception('CollectionModuleUtilLocator can only locate from ansible_collections, got {0}'.format(fq_name_parts)) elif len(fq_name_parts) >= 6 and fq_name_parts[3:5] != ('plugins', 'module_utils'): raise Exception('CollectionModuleUtilLocator can only locate below ansible_collections.(ns).(coll).plugins.module_utils, got {0}' .format(fq_name_parts)) self._collection_name = '.'.join(fq_name_parts[1:3]) self._locate() def _find_module(self, name_parts): # synthesize empty inits for packages down through module_utils- we don't want to allow those to be shipped over, but the # package hierarchy needs to exist if len(name_parts) < 6: self.source_code = '' self.is_package = True return True # NB: we can't use pkgutil.get_data safely here, since we don't want to import/execute package/module code on # the controller while analyzing/assembling the module, so we'll have to manually import the collection's # Python package to locate it (import root collection, reassemble resource path beneath, fetch source) collection_pkg_name = '.'.join(name_parts[0:3]) resource_base_path = os.path.join(*name_parts[3:]) src = None # look for package_dir first, then module try: src = pkgutil.get_data(collection_pkg_name, to_native(os.path.join(resource_base_path, '__init__.py'))) except ImportError: pass # TODO: we might want to synthesize fake inits for py3-style packages, for now they're required beneath module_utils if src is not None: # empty string is OK self.is_package = True else: try: src = pkgutil.get_data(collection_pkg_name, to_native(resource_base_path + '.py')) except ImportError: pass if src is None: # empty string is OK return False self.source_code = src return True def _get_module_utils_remainder_parts(self, name_parts): return name_parts[5:] # eg, foo.bar for ansible_collections.ns.coll.plugins.module_utils.foo.bar def recursive_finder(name, module_fqn, module_data, zf): """ Using ModuleDepFinder, make sure we have all of the module_utils files that the module and its module_utils files needs. (no longer actually recursive) :arg name: Name of the python module we're examining :arg module_fqn: Fully qualified name of the python module we're scanning :arg module_data: string Python code of the module we're scanning :arg zf: An open :python:class:`zipfile.ZipFile` object that holds the Ansible module payload which we're assembling """ # py_module_cache maps python module names to a tuple of the code in the module # and the pathname to the module. # Here we pre-load it with modules which we create without bothering to # read from actual files (In some cases, these need to differ from what ansible # ships because they're namespace packages in the module) # FIXME: do we actually want ns pkg behavior for these? Seems like they should just be forced to emptyish pkg stubs py_module_cache = { ('ansible',): ( b'from pkgutil import extend_path\n' b'__path__=extend_path(__path__,__name__)\n' b'__version__="' + to_bytes(__version__) + b'"\n__author__="' + to_bytes(__author__) + b'"\n', 'ansible/__init__.py'), ('ansible', 'module_utils'): ( b'from pkgutil import extend_path\n' b'__path__=extend_path(__path__,__name__)\n', 'ansible/module_utils/__init__.py')} module_utils_paths = [p for p in module_utils_loader._get_paths(subdirs=False) if os.path.isdir(p)] module_utils_paths.append(_MODULE_UTILS_PATH) # Parse the module code and find the imports of ansible.module_utils try: tree = compile(module_data, '<unknown>', 'exec', ast.PyCF_ONLY_AST) except (SyntaxError, IndentationError) as e: raise AnsibleError("Unable to import %s due to %s" % (name, e.msg)) finder = ModuleDepFinder(module_fqn, tree) # the format of this set is a tuple of the module name and whether or not the import is ambiguous as a module name # or an attribute of a module (eg from x.y import z <-- is z a module or an attribute of x.y?) modules_to_process = [ModuleUtilsProcessEntry(m, True, False, is_optional=m in finder.optional_imports) for m in finder.submodules] # HACK: basic is currently always required since module global init is currently tied up with AnsiballZ arg input modules_to_process.append(ModuleUtilsProcessEntry(('ansible', 'module_utils', 'basic'), False, False, is_optional=False)) # we'll be adding new modules inline as we discover them, so just keep going til we've processed them all while modules_to_process: modules_to_process.sort() # not strictly necessary, but nice to process things in predictable and repeatable order py_module_name, is_ambiguous, child_is_redirected, is_optional = modules_to_process.pop(0) if py_module_name in py_module_cache: # this is normal; we'll often see the same module imported many times, but we only need to process it once continue if py_module_name[0:2] == ('ansible', 'module_utils'): module_info = LegacyModuleUtilLocator(py_module_name, is_ambiguous=is_ambiguous, mu_paths=module_utils_paths, child_is_redirected=child_is_redirected) elif py_module_name[0] == 'ansible_collections': module_info = CollectionModuleUtilLocator(py_module_name, is_ambiguous=is_ambiguous, child_is_redirected=child_is_redirected, is_optional=is_optional) else: # FIXME: dot-joined result display.warning('ModuleDepFinder improperly found a non-module_utils import %s' % [py_module_name]) continue # Could not find the module. Construct a helpful error message. if not module_info.found: if is_optional: # this was a best-effort optional import that we couldn't find, oh well, move along... continue # FIXME: use dot-joined candidate names msg = 'Could not find imported module support code for {0}. Looked for ({1})'.format(module_fqn, module_info.candidate_names_joined) raise AnsibleError(msg) # check the cache one more time with the module we actually found, since the name could be different than the input # eg, imported name vs module if module_info.fq_name_parts in py_module_cache: continue # compile the source, process all relevant imported modules try: tree = compile(module_info.source_code, '<unknown>', 'exec', ast.PyCF_ONLY_AST) except (SyntaxError, IndentationError) as e: raise AnsibleError("Unable to import %s due to %s" % (module_info.fq_name_parts, e.msg)) finder = ModuleDepFinder('.'.join(module_info.fq_name_parts), tree, module_info.is_package) modules_to_process.extend(ModuleUtilsProcessEntry(m, True, False, is_optional=m in finder.optional_imports) for m in finder.submodules if m not in py_module_cache) # we've processed this item, add it to the output list py_module_cache[module_info.fq_name_parts] = (module_info.source_code, module_info.output_path) # ensure we process all ancestor package inits accumulated_pkg_name = [] for pkg in module_info.fq_name_parts[:-1]: accumulated_pkg_name.append(pkg) # we're accumulating this across iterations normalized_name = tuple(accumulated_pkg_name) # extra machinations to get a hashable type (list is not) if normalized_name not in py_module_cache: modules_to_process.append(ModuleUtilsProcessEntry(normalized_name, False, module_info.redirected, is_optional=is_optional)) for py_module_name in py_module_cache: py_module_file_name = py_module_cache[py_module_name][1] zf.writestr(py_module_file_name, py_module_cache[py_module_name][0]) mu_file = to_text(py_module_file_name, errors='surrogate_or_strict') display.vvvvv("Including module_utils file %s" % mu_file) def _is_binary(b_module_data): textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f])) start = b_module_data[:1024] return bool(start.translate(None, textchars)) def _get_ansible_module_fqn(module_path): """ Get the fully qualified name for an ansible module based on its pathname remote_module_fqn is the fully qualified name. Like ansible.modules.system.ping Or ansible_collections.Namespace.Collection_name.plugins.modules.ping .. warning:: This function is for ansible modules only. It won't work for other things (non-module plugins, etc) """ remote_module_fqn = None # Is this a core module? match = CORE_LIBRARY_PATH_RE.search(module_path) if not match: # Is this a module in a collection? match = COLLECTION_PATH_RE.search(module_path) # We can tell the FQN for core modules and collection modules if match: path = match.group('path') if '.' in path: # FQNs must be valid as python identifiers. This sanity check has failed. # we could check other things as well raise ValueError('Module name (or path) was not a valid python identifier') remote_module_fqn = '.'.join(path.split('/')) else: # Currently we do not handle modules in roles so we can end up here for that reason raise ValueError("Unable to determine module's fully qualified name") return remote_module_fqn def _add_module_to_zip(zf, remote_module_fqn, b_module_data): """Add a module from ansible or from an ansible collection into the module zip""" module_path_parts = remote_module_fqn.split('.') # Write the module module_path = '/'.join(module_path_parts) + '.py' zf.writestr(module_path, b_module_data) # Write the __init__.py's necessary to get there if module_path_parts[0] == 'ansible': # The ansible namespace is setup as part of the module_utils setup... start = 2 existing_paths = frozenset() else: # ... but ansible_collections and other toplevels are not start = 1 existing_paths = frozenset(zf.namelist()) for idx in range(start, len(module_path_parts)): package_path = '/'.join(module_path_parts[:idx]) + '/__init__.py' # If a collections module uses module_utils from a collection then most packages will have already been added by recursive_finder. if package_path in existing_paths: continue # Note: We don't want to include more than one ansible module in a payload at this time # so no need to fill the __init__.py with namespace code zf.writestr(package_path, b'') def _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, templar, module_compression, async_timeout, become, become_method, become_user, become_password, become_flags, environment, remote_is_local=False): """ Given the source of the module, convert it to a Jinja2 template to insert module code and return whether it's a new or old style module. """ module_substyle = module_style = 'old' # module_style is something important to calling code (ActionBase). It # determines how arguments are formatted (json vs k=v) and whether # a separate arguments file needs to be sent over the wire. # module_substyle is extra information that's useful internally. It tells # us what we have to look to substitute in the module files and whether # we're using module replacer or ansiballz to format the module itself. if _is_binary(b_module_data): module_substyle = module_style = 'binary' elif REPLACER in b_module_data: # Do REPLACER before from ansible.module_utils because we need make sure # we substitute "from ansible.module_utils basic" for REPLACER module_style = 'new' module_substyle = 'python' b_module_data = b_module_data.replace(REPLACER, b'from ansible.module_utils.basic import *') elif NEW_STYLE_PYTHON_MODULE_RE.search(b_module_data): module_style = 'new' module_substyle = 'python' elif REPLACER_WINDOWS in b_module_data: module_style = 'new' module_substyle = 'powershell' b_module_data = b_module_data.replace(REPLACER_WINDOWS, b'#Requires -Module Ansible.ModuleUtils.Legacy') elif re.search(b'#Requires -Module', b_module_data, re.IGNORECASE) \ or re.search(b'#Requires -Version', b_module_data, re.IGNORECASE)\ or re.search(b'#AnsibleRequires -OSVersion', b_module_data, re.IGNORECASE) \ or re.search(b'#AnsibleRequires -Powershell', b_module_data, re.IGNORECASE) \ or re.search(b'#AnsibleRequires -CSharpUtil', b_module_data, re.IGNORECASE): module_style = 'new' module_substyle = 'powershell' elif REPLACER_JSONARGS in b_module_data: module_style = 'new' module_substyle = 'jsonargs' elif b'WANT_JSON' in b_module_data: module_substyle = module_style = 'non_native_want_json' shebang = None # Neither old-style, non_native_want_json nor binary modules should be modified # except for the shebang line (Done by modify_module) if module_style in ('old', 'non_native_want_json', 'binary'): return b_module_data, module_style, shebang output = BytesIO() try: remote_module_fqn = _get_ansible_module_fqn(module_path) except ValueError: # Modules in roles currently are not found by the fqn heuristic so we # fallback to this. This means that relative imports inside a module from # a role may fail. Absolute imports should be used for future-proofness. # People should start writing collections instead of modules in roles so we # may never fix this display.debug('ANSIBALLZ: Could not determine module FQN') remote_module_fqn = 'ansible.modules.%s' % module_name if module_substyle == 'python': params = dict(ANSIBLE_MODULE_ARGS=module_args,) try: python_repred_params = repr(json.dumps(params, cls=AnsibleJSONEncoder, vault_to_text=True)) except TypeError as e: raise AnsibleError("Unable to pass options to module, they must be JSON serializable: %s" % to_native(e)) try: compression_method = getattr(zipfile, module_compression) except AttributeError: display.warning(u'Bad module compression string specified: %s. Using ZIP_STORED (no compression)' % module_compression) compression_method = zipfile.ZIP_STORED lookup_path = os.path.join(C.DEFAULT_LOCAL_TMP, 'ansiballz_cache') cached_module_filename = os.path.join(lookup_path, "%s-%s" % (remote_module_fqn, module_compression)) zipdata = None # Optimization -- don't lock if the module has already been cached if os.path.exists(cached_module_filename): display.debug('ANSIBALLZ: using cached module: %s' % cached_module_filename) with open(cached_module_filename, 'rb') as module_data: zipdata = module_data.read() else: if module_name in action_write_locks.action_write_locks: display.debug('ANSIBALLZ: Using lock for %s' % module_name) lock = action_write_locks.action_write_locks[module_name] else: # If the action plugin directly invokes the module (instead of # going through a strategy) then we don't have a cross-process # Lock specifically for this module. Use the "unexpected # module" lock instead display.debug('ANSIBALLZ: Using generic lock for %s' % module_name) lock = action_write_locks.action_write_locks[None] display.debug('ANSIBALLZ: Acquiring lock') with lock: display.debug('ANSIBALLZ: Lock acquired: %s' % id(lock)) # Check that no other process has created this while we were # waiting for the lock if not os.path.exists(cached_module_filename): display.debug('ANSIBALLZ: Creating module') # Create the module zip data zipoutput = BytesIO() zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method) # walk the module imports, looking for module_utils to send- they'll be added to the zipfile recursive_finder(module_name, remote_module_fqn, b_module_data, zf) display.debug('ANSIBALLZ: Writing module into payload') _add_module_to_zip(zf, remote_module_fqn, b_module_data) zf.close() zipdata = base64.b64encode(zipoutput.getvalue()) # Write the assembled module to a temp file (write to temp # so that no one looking for the file reads a partially # written file) # # FIXME: Once split controller/remote is merged, this can be simplified to # os.makedirs(lookup_path, exist_ok=True) if not os.path.exists(lookup_path): try: # Note -- if we have a global function to setup, that would # be a better place to run this os.makedirs(lookup_path) except OSError: # Multiple processes tried to create the directory. If it still does not # exist, raise the original exception. if not os.path.exists(lookup_path): raise display.debug('ANSIBALLZ: Writing module') with open(cached_module_filename + '-part', 'wb') as f: f.write(zipdata) # Rename the file into its final position in the cache so # future users of this module can read it off the # filesystem instead of constructing from scratch. display.debug('ANSIBALLZ: Renaming module') os.rename(cached_module_filename + '-part', cached_module_filename) display.debug('ANSIBALLZ: Done creating module') if zipdata is None: display.debug('ANSIBALLZ: Reading module after lock') # Another process wrote the file while we were waiting for # the write lock. Go ahead and read the data from disk # instead of re-creating it. try: with open(cached_module_filename, 'rb') as f: zipdata = f.read() except IOError: raise AnsibleError('A different worker process failed to create module file. ' 'Look at traceback for that process for debugging information.') zipdata = to_text(zipdata, errors='surrogate_or_strict') o_interpreter, o_args = _extract_interpreter(b_module_data) if o_interpreter is None: o_interpreter = u'/usr/bin/python' shebang, interpreter = _get_shebang(o_interpreter, task_vars, templar, o_args, remote_is_local=remote_is_local) # FUTURE: the module cache entry should be invalidated if we got this value from a host-dependent source rlimit_nofile = C.config.get_config_value('PYTHON_MODULE_RLIMIT_NOFILE', variables=task_vars) if not isinstance(rlimit_nofile, int): rlimit_nofile = int(templar.template(rlimit_nofile)) if rlimit_nofile: rlimit = ANSIBALLZ_RLIMIT_TEMPLATE % dict( rlimit_nofile=rlimit_nofile, ) else: rlimit = '' coverage_config = os.environ.get('_ANSIBLE_COVERAGE_CONFIG') if coverage_config: coverage_output = os.environ['_ANSIBLE_COVERAGE_OUTPUT'] if coverage_output: # Enable code coverage analysis of the module. # This feature is for internal testing and may change without notice. coverage = ANSIBALLZ_COVERAGE_TEMPLATE % dict( coverage_config=coverage_config, coverage_output=coverage_output, ) else: # Verify coverage is available without importing it. # This will detect when a module would fail with coverage enabled with minimal overhead. coverage = ANSIBALLZ_COVERAGE_CHECK_TEMPLATE else: coverage = '' now = datetime.datetime.utcnow() output.write(to_bytes(ACTIVE_ANSIBALLZ_TEMPLATE % dict( zipdata=zipdata, ansible_module=module_name, module_fqn=remote_module_fqn, params=python_repred_params, shebang=shebang, coding=ENCODING_STRING, year=now.year, month=now.month, day=now.day, hour=now.hour, minute=now.minute, second=now.second, coverage=coverage, rlimit=rlimit, ))) b_module_data = output.getvalue() elif module_substyle == 'powershell': # Powershell/winrm don't actually make use of shebang so we can # safely set this here. If we let the fallback code handle this # it can fail in the presence of the UTF8 BOM commonly added by # Windows text editors shebang = u'#!powershell' # create the common exec wrapper payload and set that as the module_data # bytes b_module_data = ps_manifest._create_powershell_wrapper( b_module_data, module_path, module_args, environment, async_timeout, become, become_method, become_user, become_password, become_flags, module_substyle, task_vars, remote_module_fqn ) elif module_substyle == 'jsonargs': module_args_json = to_bytes(json.dumps(module_args, cls=AnsibleJSONEncoder, vault_to_text=True)) # these strings could be included in a third-party module but # officially they were included in the 'basic' snippet for new-style # python modules (which has been replaced with something else in # ansiballz) If we remove them from jsonargs-style module replacer # then we can remove them everywhere. python_repred_args = to_bytes(repr(module_args_json)) b_module_data = b_module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__))) b_module_data = b_module_data.replace(REPLACER_COMPLEX, python_repred_args) b_module_data = b_module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS))) # The main event -- substitute the JSON args string into the module b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json) facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict') b_module_data = b_module_data.replace(b'syslog.LOG_USER', facility) return (b_module_data, module_style, shebang) def _extract_interpreter(b_module_data): """ Used to extract shebang expression from binary module data and return a text string with the shebang, or None if no shebang is detected. """ interpreter = None args = [] b_lines = b_module_data.split(b"\n", 1) if b_lines[0].startswith(b"#!"): b_shebang = b_lines[0].strip() # shlex.split needs text on Python 3 cli_split = shlex.split(to_text(b_shebang[2:], errors='surrogate_or_strict')) # convert args to text cli_split = [to_text(a, errors='surrogate_or_strict') for a in cli_split] interpreter = cli_split[0] args = cli_split[1:] return interpreter, args def modify_module(module_name, module_path, module_args, templar, task_vars=None, module_compression='ZIP_STORED', async_timeout=0, become=False, become_method=None, become_user=None, become_password=None, become_flags=None, environment=None, remote_is_local=False): """ Used to insert chunks of code into modules before transfer rather than doing regular python imports. This allows for more efficient transfer in a non-bootstrapping scenario by not moving extra files over the wire and also takes care of embedding arguments in the transferred modules. This version is done in such a way that local imports can still be used in the module code, so IDEs don't have to be aware of what is going on. Example: from ansible.module_utils.basic import * ... will result in the insertion of basic.py into the module from the module_utils/ directory in the source tree. For powershell, this code effectively no-ops, as the exec wrapper requires access to a number of properties not available here. """ task_vars = {} if task_vars is None else task_vars environment = {} if environment is None else environment with open(module_path, 'rb') as f: # read in the module source b_module_data = f.read() (b_module_data, module_style, shebang) = _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, templar, module_compression, async_timeout=async_timeout, become=become, become_method=become_method, become_user=become_user, become_password=become_password, become_flags=become_flags, environment=environment, remote_is_local=remote_is_local) if module_style == 'binary': return (b_module_data, module_style, to_text(shebang, nonstring='passthru')) elif shebang is None: interpreter, args = _extract_interpreter(b_module_data) # No interpreter/shebang, assume a binary module? if interpreter is not None: shebang, new_interpreter = _get_shebang(interpreter, task_vars, templar, args, remote_is_local=remote_is_local) # update shebang b_lines = b_module_data.split(b"\n", 1) if interpreter != new_interpreter: b_lines[0] = to_bytes(shebang, errors='surrogate_or_strict', nonstring='passthru') if os.path.basename(interpreter).startswith(u'python'): b_lines.insert(1, b_ENCODING_STRING) b_module_data = b"\n".join(b_lines) return (b_module_data, module_style, shebang) def get_action_args_with_defaults(action, args, defaults, templar, redirected_names=None, action_groups=None): if redirected_names: resolved_action_name = redirected_names[-1] else: resolved_action_name = action if redirected_names is not None: msg = ( "Finding module_defaults for the action %s. " "The caller passed a list of redirected action names, which is deprecated. " "The task's resolved action should be provided as the first argument instead." ) display.deprecated(msg % resolved_action_name, version='2.16') # Get the list of groups that contain this action if action_groups is None: msg = ( "Finding module_defaults for action %s. " "The caller has not passed the action_groups, so any " "that may include this action will be ignored." ) display.warning(msg=msg) group_names = [] else: group_names = action_groups.get(resolved_action_name, []) tmp_args = {} module_defaults = {} # Merge latest defaults into dict, since they are a list of dicts if isinstance(defaults, list): for default in defaults: module_defaults.update(default) # module_defaults keys are static, but the values may be templated module_defaults = templar.template(module_defaults) for default in module_defaults: if default.startswith('group/'): group_name = default.split('group/')[-1] if group_name in group_names: tmp_args.update((module_defaults.get('group/%s' % group_name) or {}).copy()) # handle specific action defaults tmp_args.update(module_defaults.get(resolved_action_name, {}).copy()) # direct args override all tmp_args.update(args) return tmp_args