Merge pull request #1348 from gavindsouza/virtualenv-shenanigans

fix: Virtualenv shenanigans

bench/app.py

@@ -1,5 +1,5 @@
 # imports - standard imports
-import functools
+from functools import lru_cache
 import json
 import logging
 import os
@@ -149,7 +149,7 @@ class AppMeta:
 		return f"git@{self.remote_server}:{self.org}/{self.repo}.git"
 
 
-@functools.lru_cache(maxsize=None)
+@lru_cache(maxsize=None)
 class App(AppMeta):
 	def __init__(
 		self,

bench/bench.py

@@ -1,6 +1,6 @@
 # imports - standard imports
 import subprocess
-import functools
+from functools import lru_cache
 import os
 import shutil
 import json
@@ -30,7 +30,6 @@ from bench.utils.bench import (
 	restart_process_manager,
 	remove_backups_crontab,
 	get_venv_path,
-	get_virtualenv_path,
 	get_env_cmd,
 )
 from bench.utils.render import job, step
@@ -55,7 +54,7 @@ class Validator:
 		validate_app_installed_on_sites(app, bench_path=self.name)
 
 
-@functools.lru_cache(maxsize=None)
+@lru_cache(maxsize=None)
 class Bench(Base, Validator):
 	def __init__(self, path):
 		self.name = path
@@ -71,7 +70,7 @@ class Bench(Base, Validator):
 
 	@property
 	def python(self) -> str:
-		return get_env_cmd("python", bench_path=self.name)
+		return get_env_cmd("python*", bench_path=self.name)
 
 	@property
 	def shallow_clone(self) -> bool:
@@ -347,13 +346,9 @@ class BenchSetup(Base):
 		click.secho("Setting Up Environment", fg="yellow")
 
 		frappe = os.path.join(self.bench.name, "apps", "frappe")
-		virtualenv = get_virtualenv_path(verbose=verbose)
 		quiet_flag = "" if verbose else "--quiet"
 
 		if not os.path.exists(self.bench.python):
-			if virtualenv:
-				self.run(f"{virtualenv} {quiet_flag} env -p {python}", cwd=self.bench.name)
-			else:
 			venv = get_venv_path(verbose=verbose, python=python)
 			self.run(f"{venv} env", cwd=self.bench.name)
 

bench/cli.py

@@ -190,13 +190,13 @@ def change_uid():
 
 
 def app_cmd(bench_path="."):
-	f = get_env_cmd("python", bench_path=bench_path)
+	f = get_env_cmd("python*", bench_path=bench_path)
 	os.chdir(os.path.join(bench_path, "sites"))
 	os.execv(f, [f] + ["-m", "frappe.utils.bench_helper"] + sys.argv[1:])
 
 
 def frappe_cmd(bench_path="."):
-	f = get_env_cmd("python", bench_path=bench_path)
+	f = get_env_cmd("python*", bench_path=bench_path)
 	os.chdir(os.path.join(bench_path, "sites"))
 	os.execv(f, [f] + ["-m", "frappe.utils.bench_helper", "frappe"] + sys.argv[1:])
 
@@ -216,7 +216,7 @@ def get_frappe_commands():
 
 
 def get_frappe_help(bench_path="."):
-	python = get_env_cmd("python", bench_path=bench_path)
+	python = get_env_cmd("python*", bench_path=bench_path)
 	sites_path = os.path.join(bench_path, "sites")
 	try:
 		out = get_cmd_output(
@@ -245,6 +245,7 @@ def setup_clear_cache():
 
 	def _chdir(*args, **kwargs):
 		Bench.cache_clear()
+		get_env_cmd.cache_clear()
 		return f(*args, **kwargs)
 
 	os.chdir = _chdir

bench/commands/make.py

@@ -227,5 +227,5 @@ def pip(ctx, args):
 
 	from bench.utils.bench import get_env_cmd
 
-	env_py = get_env_cmd("python")
+	env_py = get_env_cmd("python*")
 	os.execv(env_py, (env_py, "-m", "pip") + args)

bench/commands/setup.py

@@ -96,7 +96,7 @@ def setup_backups():
 	Bench(".").setup.backups()
 
 
-@click.command("env", help="Setup virtualenv for bench")
+@click.command("env", help="Setup Python environment for bench")
 @click.option(
 	"--python", type=str, default="python3", help="Path to Python Executable."
 )

bench/utils/__init__.py

@@ -231,10 +231,10 @@ def run_frappe_cmd(*args, **kwargs):
 	from bench.utils.bench import get_env_cmd
 
 	bench_path = kwargs.get("bench_path", ".")
-	f = get_env_cmd("python", bench_path=bench_path)
+	f = get_env_cmd("python*", bench_path=bench_path)
 	sites_dir = os.path.join(bench_path, "sites")
 
-	is_async = False if from_command_line else True
+	is_async = not from_command_line
 	if is_async:
 		stderr = stdout = subprocess.PIPE
 	else:
@@ -247,11 +247,7 @@ def run_frappe_cmd(*args, **kwargs):
 		stderr=stderr,
 	)
 
-	if is_async:
+	return_code = print_output(p) if is_async else p.wait()
-		return_code = print_output(p)
-	else:
-		return_code = p.wait()
-
 	if return_code > 0:
 		sys.exit(return_code)
 
@@ -390,7 +386,7 @@ def generate_command_cache(bench_path=".") -> List:
 	"""
 	from bench.utils.bench import get_env_cmd
 
-	python = get_env_cmd("python", bench_path=bench_path)
+	python = get_env_cmd("python*", bench_path=bench_path)
 	sites_path = os.path.join(bench_path, "sites")
 
 	if os.path.exists(bench_cache_file):

bench/utils/bench.py

@@ -6,6 +6,8 @@ import os
 import re
 import subprocess
 import sys
+from functools import lru_cache
+from glob import glob
 from json.decoder import JSONDecodeError
 
 # imports - third party imports
@@ -19,19 +21,21 @@ from bench.utils import exec_cmd, get_bench_name, get_cmd_output, log, which
 logger = logging.getLogger(bench.PROJECT_NAME)
 
 
-def get_env_cmd(cmd, bench_path="."):
+@lru_cache(maxsize=None)
+def get_env_cmd(cmd: str, bench_path: str = ".") -> str:
+	# this supports envs' generated by patched virtualenv or venv (which may cause an extra 'local' folder to be created)
+
+	existing_python_bins = glob(
+		os.path.join(bench_path, "env", "**", "bin", cmd), recursive=True
+	)
+
+	if existing_python_bins:
+		return os.path.abspath(existing_python_bins[0])
+
+	cmd = cmd.strip("*")
 	return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd))
 
 
-def get_virtualenv_path(verbose=False):
-	virtualenv_path = which("virtualenv")
-
-	if not virtualenv_path and verbose:
-		log("virtualenv cannot be found", level=2)
-
-	return virtualenv_path
-
-
 def get_venv_path(verbose=False, python="python3"):
 	with open(os.devnull, "wb") as devnull:
 		is_venv_installed = not subprocess.call(
@@ -172,6 +176,9 @@ def migrate_env(python, backup=False):
 	path = os.getcwd()
 	python = which(python)
 	virtualenv = which("virtualenv")
+	if not virtualenv:
+		raise FileNotFoundError("`virtualenv` not found. Install it and try again.")
+
 	pvenv = os.path.join(path, nvenv)
 
 	# Clear Cache before Bench Dies.

docs/bench_usage.md

@@ -130,7 +130,7 @@ The setup commands used for setting up the Frappe environment in context of the
 
  - **sudoers**: Add commands to sudoers list for allowing bench commands execution without root password
 
- - **env**: Setup virtualenv for bench. This sets up a `env` folder under the root of the bench directory.
+ - **env**: Setup Python virtual environment for bench. This sets up a `env` folder under the root of the bench directory.
  - **redis**: Generates configuration for Redis
  - **fonts**: Add Frappe fonts to system
  - **config**: Generate or over-write sites/common_site_config.json