Function exec_pip_install

infra/python/bootstrap_virtualenv.py:139–193

Executes "pip install" with the provided command line arguments. If 'cc' is set, it is used as the C compiler. Otherwise compilation of C/C++ code is disabled by setting the CC environment variable to a bogus value. Other environment vars can optionally be set with the 'env' argument. By defau

(venv_dir, args, cc="no-cc-available", env=None)

def exec_pip_install(venv_dir, args, cc="no-cc-available", env=None):
  '''Executes "pip install" with the provided command line arguments. If 'cc' is set,
  it is used as the C compiler. Otherwise compilation of C/C++ code is disabled by
  setting the CC environment variable to a bogus value.
  Other environment vars can optionally be set with the 'env' argument. By default the
  current process's command line arguments are inherited.'''
  if not env: env = dict(os.environ)
  env["CC"] = cc
  # Since gcc is now built with toolchain binutils which may be newer than the
  # system binutils, we need to include the toolchain binutils on the PATH.
  toolchain_binutils_dir = toolchain_pkg_dir("binutils")
  binutils_bin_dir = os.path.join(toolchain_binutils_dir, "bin")
  env["PATH"] = "{0}:{1}".format(binutils_bin_dir, env["PATH"])
  # Sometimes pip install invokes gcc directly without using the CC environment
  # variable. If system GCC is too new, then it will fail, because it needs symbols
  # that are not in Impala's libstdc++. To avoid this, we add GCC to the PATH,
  # so any direct reference will use our GCC rather than the system GCC.
  toolchain_gcc_dir = toolchain_pkg_dir("gcc")
  gcc_bin_dir = os.path.join(toolchain_gcc_dir, "bin")
  env["PATH"] = "{0}:{1}".format(gcc_bin_dir, env["PATH"])

  # Parallelize the slow numpy build.
  # Use getconf instead of nproc because it is supported more widely, e.g. on older
  # linux distributions.
  env["NPY_NUM_BUILD_JOBS"] = exec_cmd(["getconf", "_NPROCESSORS_ONLN"]).strip()

  # Don't call the virtualenv pip directly, it uses a hashbang to to call the python
  # virtualenv using an absolute path. If the path to the virtualenv is very long, the
  # hashbang won't work.
  impala_pip_base_cmd = [os.path.join(venv_dir, "bin", "python3"),
                         os.path.join(venv_dir, "bin", "pip3"), "install", "-v"]

  # Passes --no-binary for IMPALA-3767: without this, Cython (and
  # several other packages) fail download.
  #
  # --no-cache-dir is used to prevent caching of compiled artifacts, which may be built
  # with different compilers or settings.
  third_party_pkg_install_cmd = \
      impala_pip_base_cmd[:] + ["--no-binary", ":all:", "--no-cache-dir"]

  # Prevent fetching additional packages from the index. If we forget to add a package
  # to one of the requirements.txt files, this should trigger an error. However, we will
  # still access the index for version/dependency resolution, hence we need to change it
  # when using a private mirror.
  third_party_pkg_install_cmd.append("--no-index")

  third_party_pkg_install_cmd.extend(["--find-links",
      "file://%s" % pathname2url(os.path.abspath(DEPS_DIR))])
  third_party_pkg_install_cmd.extend(args)
  exec_cmd(third_party_pkg_install_cmd, env=env)

  # Finally, we want to install the packages from our own internal python lib
  local_package_install_cmd = impala_pip_base_cmd + \
      ['-e', os.path.join(os.getenv('IMPALA_HOME'), 'lib', 'python')]
  exec_cmd(local_package_install_cmd)


def find_file(*paths):