accounting/bench
2
0
Fork 0
mirror of https://github.com/frappe/bench.git synced 2025-01-09 08:30:39 +00:00
Code Issues Releases Activity

Merge branch 'develop' into staging

Browse Source
This commit is contained in:
Gavin D'souza 2022-08-02 15:13:16 +05:30
commit 57a4773c25
15 changed files with 242 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,122 @@
name: 'CI'
on:
pull_request:
workflow_dispatch:
push:
branches: [ develop ]
permissions:
contents: read
jobs:
base_setup:
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
python-version: [ '3.7', '3.8', '3.9', '3.10' ]
name: Base (${{ matrix.python-version }})
services:
mariadb:
image: mariadb:10.6
env:
MARIADB_ROOT_PASSWORD: travis
ports:
- 3306:3306
options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/setup-node@v3
with:
node-version: 14
- run: |
wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb;
sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb;
sudo apt install redis-server
python -m pip install -U --no-cache-dir --force-reinstall urllib3 pyOpenSSL ndg-httpsclient pyasn1 wheel setuptools pip;
python -m pip install -U -e ${GITHUB_WORKSPACE};
- run: python ${GITHUB_WORKSPACE}/bench/tests/test_init.py TestBenchInit.basic
production_setup:
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
python-version: [ '3.7', '3.10' ]
name: Production (${{ matrix.python-version }})
services:
mariadb:
image: mariadb:10.6
env:
MARIADB_ROOT_PASSWORD: travis
ports:
- 3306:3306
options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/setup-node@v3
with:
node-version: 14
- run: |
wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb;
sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb;
sudo apt install redis-server
python -m pip install -U --no-cache-dir --force-reinstall urllib3 pyOpenSSL ndg-httpsclient pyasn1 wheel setuptools pip;
python -m pip install -U -e ${GITHUB_WORKSPACE};
- run: python bench/tests/test_setup_production.py TestSetupProduction.production
tests:
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
python-version: [ '3.7', '3.10' ]
name: Tests (${{ matrix.python-version }})
services:
mariadb:
image: mariadb:10.6
env:
MARIADB_ROOT_PASSWORD: travis
ports:
- 3306:3306
options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/setup-node@v3
with:
node-version: 14
- run: |
wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb;
sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb;
sudo apt install redis-server
python -m pip install -U --no-cache-dir --force-reinstall urllib3 pyOpenSSL ndg-httpsclient pyasn1 wheel setuptools pip;
python -m pip install -U -e ${GITHUB_WORKSPACE};
- run: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init

39
.github/workflows/linters.yml vendored Normal file
View File

@ -0,0 +1,39 @@
name: Linters
on:
pull_request:
workflow_dispatch:
push:
branches: [ develop ]
permissions:
contents: read
jobs:
linter:
name: 'Pre Commit Hooks'
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- uses: pre-commit/action@v3.0.0
with:
extra_args: --files 'bench/'
deps-vulnerable-check:
name: 'Vulnerable Dependency Check'
runs-on: ubuntu-latest
steps:
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- uses: actions/checkout@v3
- name: 'Pip Audit'
run: |
pip install pip-audit
pip-audit ${GITHUB_WORKSPACE}

76
.travis.yml
View File

@ -1,76 +0,0 @@
language: python
dist: bionic
sudo: true
git:
depth: 1
addons:
mariadb: '10.3'
matrix:
include:
- name: "Python 3.7 Basic Setup"
python: 3.7
env: TEST=bench
script: python bench/tests/test_init.py TestBenchInit.basic
- name: "Python 3.8 Basic Setup"
python: 3.8
env: TEST=bench
script: python bench/tests/test_init.py TestBenchInit.basic
- name: "Python 3.9 Basic Setup"
python: 3.9
env: TEST=bench
script: python bench/tests/test_init.py TestBenchInit.basic
- name: "Python 3.10 Basic Setup"
python: "3.10"
env: TEST=bench
script: python bench/tests/test_init.py TestBenchInit.basic
- name: "Python 3.7 Production Setup"
python: 3.7
env: TEST=bench
script: python bench/tests/test_setup_production.py TestSetupProduction.production
- name: "Python 3.10 Production Setup"
python: "3.10"
env: TEST=bench
script: python bench/tests/test_setup_production.py TestSetupProduction.production
- name: "Python 3.7 Tests"
python: 3.7
env: TEST=bench
script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init
- name: "Python 3.10 Tests"
python: "3.10"
env: TEST=bench
script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init
install:
- python -m pip install -U --no-cache-dir --force-reinstall urllib3 pyOpenSSL ndg-httpsclient pyasn1 wheel setuptools pip
- if [ $TEST == "bench" ];then
wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb;
sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb;
nvm install 14;
nvm use 14;
mkdir -p ~/.bench;
cp -r $TRAVIS_BUILD_DIR/* ~/.bench;
python -m pip install -U -e ~/.bench;
mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'";
mysql -u root -e "FLUSH PRIVILEGES";
fi
- if [ $TEST == "easy_install" ];then
mkdir -p /tmp/.bench;
cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench;
fi

4
bench/app.py
View File

@ -1,5 +1,5 @@
# imports - standard imports
import functools
from functools import lru_cache
import json
import logging
import os
@ -149,7 +149,7 @@ class AppMeta:
return f"git@{self.remote_server}:{self.org}/{self.repo}.git"
@functools.lru_cache(maxsize=None)
@lru_cache(maxsize=None)
class App(AppMeta):
def __init__(
self,

22
bench/bench.py
View File

@ -1,6 +1,6 @@
# imports - standard imports
import subprocess
import functools
from functools import lru_cache
import os
import shutil
import json
@ -30,7 +30,6 @@ from bench.utils.bench import (
restart_process_manager,
remove_backups_crontab,
get_venv_path,
get_virtualenv_path,
get_env_cmd,
)
from bench.utils.render import job, step
@ -55,7 +54,7 @@ class Validator:
validate_app_installed_on_sites(app, bench_path=self.name)
@functools.lru_cache(maxsize=None)
@lru_cache(maxsize=None)
class Bench(Base, Validator):
def __init__(self, path):
self.name = path
@ -71,7 +70,7 @@ class Bench(Base, Validator):
@property
def python(self) -> str:
return get_env_cmd("python", bench_path=self.name)
return get_env_cmd("python*", bench_path=self.name)
@property
def shallow_clone(self) -> bool:
@ -347,20 +346,19 @@ class BenchSetup(Base):
click.secho("Setting Up Environment", fg="yellow")
frappe = os.path.join(self.bench.name, "apps", "frappe")
virtualenv = get_virtualenv_path(verbose=verbose)
quiet_flag = "" if verbose else "--quiet"
if not os.path.exists(self.bench.python):
if virtualenv:
self.run(f"{virtualenv} {quiet_flag} env -p {python}")
else:
venv = get_venv_path(verbose=verbose, python=python)
self.run(f"{venv} env")
self.run(f"{venv} env", cwd=self.bench.name)
self.pip()
if os.path.exists(frappe):
self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade -e {frappe}")
self.run(
f"{self.bench.python} -m pip install {quiet_flag} --upgrade -e {frappe}",
cwd=self.bench.name,
)
@step(title="Setting Up Bench Config", success="Bench Config Set Up")
def config(self, redis=True, procfile=True):
@ -388,7 +386,9 @@ class BenchSetup(Base):
verbose = bench.cli.verbose or verbose
quiet_flag = "" if verbose else "--quiet"
return self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade pip")
return self.run(
f"{self.bench.python} -m pip install {quiet_flag} --upgrade pip", cwd=self.bench.name
)
def logging(self):
from bench.utils import setup_logging

20
bench/cli.py
View File

@ -1,7 +1,6 @@
# imports - standard imports
import atexit
from contextlib import contextmanager
import json
from logging import Logger
import os
import pwd
@ -16,11 +15,10 @@ from bench.bench import Bench
from bench.commands import bench_command
from bench.config.common_site_config import get_config
from bench.utils import (
bench_cache_file,
check_latest_version,
drop_privileges,
find_parent_bench,
generate_command_cache,
get_env_frappe_commands,
get_cmd_output,
is_bench_directory,
is_dist_editable,
@ -190,33 +188,26 @@ def change_uid():
def app_cmd(bench_path="."):
f = get_env_cmd("python", bench_path=bench_path)
f = get_env_cmd("python*", bench_path=bench_path)
os.chdir(os.path.join(bench_path, "sites"))
os.execv(f, [f] + ["-m", "frappe.utils.bench_helper"] + sys.argv[1:])
def frappe_cmd(bench_path="."):
f = get_env_cmd("python", bench_path=bench_path)
f = get_env_cmd("python*", bench_path=bench_path)
os.chdir(os.path.join(bench_path, "sites"))
os.execv(f, [f] + ["-m", "frappe.utils.bench_helper", "frappe"] + sys.argv[1:])
def get_cached_frappe_commands():
if os.path.exists(bench_cache_file):
command_dump = open(bench_cache_file).read() or "[]"
return set(json.loads(command_dump))
return set()
def get_frappe_commands():
if not is_bench_directory():
return set()
return set(generate_command_cache())
return set(get_env_frappe_commands())
def get_frappe_help(bench_path="."):
python = get_env_cmd("python", bench_path=bench_path)
python = get_env_cmd("python*", bench_path=bench_path)
sites_path = os.path.join(bench_path, "sites")
try:
out = get_cmd_output(
@ -245,6 +236,7 @@ def setup_clear_cache():
def _chdir(*args, **kwargs):
Bench.cache_clear()
get_env_cmd.cache_clear()
return f(*args, **kwargs)
os.chdir = _chdir

4
bench/commands/__init__.py
View File

@ -74,11 +74,9 @@ bench_command.add_command(switch_to_develop)
from bench.commands.utils import (
backup_all_sites,
bench_src,
clear_command_cache,
disable_production,
download_translations,
find_benches,
generate_command_cache,
migrate_env,
renew_lets_encrypt,
restart,
@ -110,8 +108,6 @@ bench_command.add_command(disable_production)
bench_command.add_command(bench_src)
bench_command.add_command(find_benches)
bench_command.add_command(migrate_env)
bench_command.add_command(generate_command_cache)
bench_command.add_command(clear_command_cache)
from bench.commands.setup import setup

2
bench/commands/make.py
View File

@ -227,5 +227,5 @@ def pip(ctx, args):
from bench.utils.bench import get_env_cmd
env_py = get_env_cmd("python")
env_py = get_env_cmd("python*")
os.execv(env_py, (env_py, "-m", "pip") + args)

2
bench/commands/setup.py
View File

@ -96,7 +96,7 @@ def setup_backups():
Bench(".").setup.backups()
@click.command("env", help="Setup virtualenv for bench")
@click.command("env", help="Setup Python environment for bench")
@click.option(
"--python", type=str, default="python3", help="Path to Python Executable."
)

14
bench/commands/utils.py
View File

@ -176,17 +176,3 @@ def migrate_env(python, backup=True):
from bench.utils.bench import migrate_env
migrate_env(python=python, backup=backup)
@click.command("generate-command-cache", help="Caches Frappe Framework commands")
def generate_command_cache(bench_path="."):
from bench.utils import generate_command_cache
return generate_command_cache(bench_path=bench_path)
@click.command("clear-command-cache", help="Clears Frappe Framework cached commands")
def clear_command_cache(bench_path="."):
from bench.utils import clear_command_cache
return clear_command_cache(bench_path=bench_path)

30
bench/tests/test_init.py
View File

@ -80,7 +80,6 @@ class TestBenchInit(TestBenchBase):
site_config_path = os.path.join(site_path, "site_config.json")
self.init_bench(bench_name)
exec_cmd("bench setup requirements --node", cwd=bench_path)
self.new_site(site_name, bench_name)
self.assertTrue(os.path.exists(site_path))
@ -99,7 +98,7 @@ class TestBenchInit(TestBenchBase):
def test_get_app(self):
self.init_bench("test-bench")
bench_path = os.path.join(self.benches_path, "test-bench")
exec_cmd(f"bench get-app {TEST_FRAPPE_APP}", cwd=bench_path)
exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --skip-assets", cwd=bench_path)
self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP)))
app_installed_in_env = TEST_FRAPPE_APP in subprocess.check_output(
["bench", "pip", "freeze"], cwd=bench_path
@ -111,7 +110,7 @@ class TestBenchInit(TestBenchBase):
FRAPPE_APP = "healthcare"
self.init_bench("test-bench")
bench_path = os.path.join(self.benches_path, "test-bench")
exec_cmd(f"bench get-app {FRAPPE_APP} --resolve-deps", cwd=bench_path)
exec_cmd(f"bench get-app {FRAPPE_APP} --resolve-deps --skip-assets", cwd=bench_path)
self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", FRAPPE_APP)))
states_path = os.path.join(bench_path, "sites", "apps.json")
@ -128,9 +127,9 @@ class TestBenchInit(TestBenchBase):
bench_path = os.path.join(self.benches_path, "test-bench")
self.init_bench(bench_name)
exec_cmd("bench setup requirements --node", cwd=bench_path)
exec_cmd("bench build", cwd=bench_path)
exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --branch master", cwd=bench_path)
exec_cmd(
f"bench get-app {TEST_FRAPPE_APP} --branch master --skip-assets", cwd=bench_path
)
self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP)))
@ -143,23 +142,24 @@ class TestBenchInit(TestBenchBase):
# create and install app on site
self.new_site(site_name, bench_name)
installed_app = not exec_cmd(
f"bench --site {site_name} install-app {TEST_FRAPPE_APP}", cwd=bench_path
f"bench --site {site_name} install-app {TEST_FRAPPE_APP}",
cwd=bench_path,
_raise=False,
)
if installed_app:
app_installed_on_site = subprocess.check_output(
["bench", "--site", site_name, "list-apps"], cwd=bench_path
).decode("utf8")
if installed_app:
self.assertTrue(TEST_FRAPPE_APP in app_installed_on_site)
def test_remove_app(self):
self.init_bench("test-bench")
bench_path = os.path.join(self.benches_path, "test-bench")
exec_cmd("bench setup requirements --node", cwd=bench_path)
exec_cmd(
f"bench get-app {TEST_FRAPPE_APP} --branch master --overwrite", cwd=bench_path
f"bench get-app {TEST_FRAPPE_APP} --branch master --overwrite --skip-assets",
cwd=bench_path,
)
exec_cmd(f"bench remove-app {TEST_FRAPPE_APP}", cwd=bench_path)
@ -183,14 +183,18 @@ class TestBenchInit(TestBenchBase):
prevoius_branch = f"version-{int(FRAPPE_BRANCH.split('-')[1]) - 1}"
successful_switch = not exec_cmd(
f"bench switch-to-branch {prevoius_branch} frappe --upgrade", cwd=bench_path
f"bench switch-to-branch {prevoius_branch} frappe --upgrade",
cwd=bench_path,
_raise=False,
)
if successful_switch:
app_branch_after_switch = str(git.Repo(path=app_path).active_branch)
self.assertEqual(prevoius_branch, app_branch_after_switch)
successful_switch = not exec_cmd(
f"bench switch-to-branch {FRAPPE_BRANCH} frappe --upgrade", cwd=bench_path
f"bench switch-to-branch {FRAPPE_BRANCH} frappe --upgrade",
cwd=bench_path,
_raise=False,
)
if successful_switch:
app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch)

12
bench/tests/test_utils.py
View File

@ -75,6 +75,18 @@ class TestUtils(unittest.TestCase):
f.write("__version__ = '11.0'")
subprocess.run(["git", "add", "."], cwd=frappe_path, capture_output=True, check=True)
subprocess.run(
["git", "config", "user.email", "bench-test_app_states@gha.com"],
cwd=frappe_path,
capture_output=True,
check=True,
)
subprocess.run(
["git", "config", "user.name", "App States Test"],
cwd=frappe_path,
capture_output=True,
check=True,
)
subprocess.run(
["git", "commit", "-m", "temp"], cwd=frappe_path, capture_output=True, check=True
)

36
bench/utils/__init__.py
View File

@ -22,7 +22,6 @@ from bench.exceptions import (
)
logger = logging.getLogger(PROJECT_NAME)
bench_cache_file = ".bench.cmd"
paths_in_app = ("hooks.py", "modules.txt", "patches.txt")
paths_in_bench = ("apps", "sites", "config", "logs", "config/pids")
sudoers_file = "/etc/sudoers.d/frappe"
@ -231,10 +230,10 @@ def run_frappe_cmd(*args, **kwargs):
from bench.utils.bench import get_env_cmd
bench_path = kwargs.get("bench_path", ".")
f = get_env_cmd("python", bench_path=bench_path)
f = get_env_cmd("python*", bench_path=bench_path)
sites_dir = os.path.join(bench_path, "sites")
is_async = False if from_command_line else True
is_async = not from_command_line
if is_async:
stderr = stdout = subprocess.PIPE
else:
@ -247,11 +246,7 @@ def run_frappe_cmd(*args, **kwargs):
stderr=stderr,
)
if is_async:
return_code = print_output(p)
else:
return_code = p.wait()
return_code = print_output(p) if is_async else p.wait()
if return_code > 0:
sys.exit(return_code)
@ -384,25 +379,21 @@ def find_parent_bench(path: str) -> str:
return find_parent_bench(parent_dir)
def generate_command_cache(bench_path=".") -> List:
def get_env_frappe_commands(bench_path=".") -> List:
"""Caches all available commands (even custom apps) via Frappe
Default caching behaviour: generated the first time any command (for a specific bench directory)
"""
from bench.utils.bench import get_env_cmd
python = get_env_cmd("python", bench_path=bench_path)
python = get_env_cmd("python*", bench_path=bench_path)
sites_path = os.path.join(bench_path, "sites")
if os.path.exists(bench_cache_file):
os.remove(bench_cache_file)
try:
output = get_cmd_output(
return json.loads(
get_cmd_output(
f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path
)
with open(bench_cache_file, "w") as f:
json.dump(eval(output), f)
return json.loads(output)
)
except subprocess.CalledProcessError as e:
if hasattr(e, "stderr"):
@ -411,17 +402,6 @@ def generate_command_cache(bench_path=".") -> List:
return []
def clear_command_cache(bench_path="."):
"""Clears commands cached
Default invalidation behaviour: destroyed on each run of `bench update`
"""
if os.path.exists(bench_cache_file):
os.remove(bench_cache_file)
else:
print("Bench command cache doesn't exist in this folder!")
def find_org(org_repo):
import requests

29
bench/utils/bench.py
View File

@ -6,6 +6,8 @@ import os
import re
import subprocess
import sys
from functools import lru_cache
from glob import glob
from json.decoder import JSONDecodeError
# imports - third party imports
@ -19,19 +21,21 @@ from bench.utils import exec_cmd, get_bench_name, get_cmd_output, log, which
logger = logging.getLogger(bench.PROJECT_NAME)
def get_env_cmd(cmd, bench_path="."):
@lru_cache(maxsize=None)
def get_env_cmd(cmd: str, bench_path: str = ".") -> str:
# this supports envs' generated by patched virtualenv or venv (which may cause an extra 'local' folder to be created)
existing_python_bins = glob(
os.path.join(bench_path, "env", "**", "bin", cmd), recursive=True
)
if existing_python_bins:
return os.path.abspath(existing_python_bins[0])
cmd = cmd.strip("*")
return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd))
def get_virtualenv_path(verbose=False):
virtualenv_path = which("virtualenv")
if not virtualenv_path and verbose:
log("virtualenv cannot be found", level=2)
return virtualenv_path
def get_venv_path(verbose=False, python="python3"):
with open(os.devnull, "wb") as devnull:
is_venv_installed = not subprocess.call(
@ -40,7 +44,7 @@ def get_venv_path(verbose=False, python="python3"):
if is_venv_installed:
return f"{python} -m venv"
else:
log("virtualenv cannot be found", level=2)
log("venv cannot be found", level=2)
def update_node_packages(bench_path=".", apps=None):
@ -172,6 +176,9 @@ def migrate_env(python, backup=False):
path = os.getcwd()
python = which(python)
virtualenv = which("virtualenv")
if not virtualenv:
raise FileNotFoundError("`virtualenv` not found. Install it and try again.")
pvenv = os.path.join(path, nvenv)
# Clear Cache before Bench Dies.

2
docs/bench_usage.md
View File

@ -130,7 +130,7 @@ The setup commands used for setting up the Frappe environment in context of the
- **sudoers**: Add commands to sudoers list for allowing bench commands execution without root password
- **env**: Setup virtualenv for bench. This sets up a `env` folder under the root of the bench directory.
- **env**: Setup Python virtual environment for bench. This sets up a `env` folder under the root of the bench directory.
- **redis**: Generates configuration for Redis
- **fonts**: Add Frappe fonts to system
- **config**: Generate or over-write sites/common_site_config.json