From 072375fe07c096192f1d7dd899c93fbc55c2daa1 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 21 May 2021 11:29:18 +0530 Subject: [PATCH 1/3] fix: add WORKER_CLASS under erpnext-python in docker-compose.yml --- docker-compose.yml | 1 + env-local | 1 + env-production | 1 + 3 files changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 1a63f631..5ca8569b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,6 +59,7 @@ services: - REDIS_SOCKETIO=redis-socketio:6379 - SOCKETIO_PORT=9000 - AUTO_MIGRATE=1 + - WORKER_CLASS=${WORKER_CLASS} volumes: - sites-vol:/home/frappe/frappe-bench/sites:rw - assets-vol:/home/frappe/frappe-bench/sites/assets:rw diff --git a/env-local b/env-local index 639bbb54..821c23ea 100644 --- a/env-local +++ b/env-local @@ -15,3 +15,4 @@ HTTPS_REDIRECT_ENTRYPOINT_LABEL=erpnext.local.no-entrypoint HTTPS_REDIRECT_MIDDLEWARE_LABEL=erpnext.local.no-middleware HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=erpnext.local-no-redirect-middleware SKIP_NGINX_TEMPLATE_GENERATION=0 +WORKER_CLASS=gthread diff --git a/env-production b/env-production index 47ff1458..b6de88bc 100644 --- a/env-production +++ b/env-production @@ -15,3 +15,4 @@ HTTPS_REDIRECT_ENTRYPOINT_LABEL=traefik.http.routers.http-catchall.entrypoints=w HTTPS_REDIRECT_MIDDLEWARE_LABEL=traefik.http.routers.http-catchall.middlewares=redirect-to-https HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https SKIP_NGINX_TEMPLATE_GENERATION=0 +WORKER_CLASS=gthread From c6da8a1d98bcc097d871d19ebf3cd46e26ea6b92 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 21 May 2021 14:30:55 +0530 Subject: [PATCH 2/3] feat: improve and upgrade build use nodejs v14 allow build args to pass repo and branch --- build/bench/Dockerfile | 15 ++-- build/erpnext-nginx/Dockerfile | 13 +-- build/erpnext-worker/Dockerfile | 10 ++- build/frappe-nginx/Dockerfile | 10 ++- build/frappe-socketio/Dockerfile | 18 ++-- build/frappe-socketio/package.json | 6 +- build/frappe-worker/Dockerfile | 7 +- development/.vscode/launch.json | 5 -- travis.py | 127 +++++++++++++++++++++++------ 9 files changed, 145 insertions(+), 66 deletions(-) diff --git a/build/bench/Dockerfile b/build/bench/Dockerfile index d185fadb..418693a4 100644 --- a/build/bench/Dockerfile +++ b/build/bench/Dockerfile @@ -1,5 +1,9 @@ # Frappe Bench Dockerfile FROM debian:buster-slim + +ARG GIT_REPO=https://github.com/frappe/bench.git +ARG GIT_BRANCH=develop + LABEL author=frappé RUN apt-get update -y && apt-get install \ @@ -71,7 +75,7 @@ WORKDIR /home/frappe # Clone and install bench in the local user home directory # For development, bench source is located in ~/.bench -RUN git clone https://github.com/frappe/bench.git .bench \ +RUN git clone ${GIT_REPO} -b ${GIT_BRANCH} .bench \ && pip3 install --user -e .bench # Export python executables for Dockerfile @@ -87,13 +91,12 @@ RUN bash -c "bench --version" # !!! UPDATE NODEJS PERIODICALLY WITH LATEST VERSIONS !!! # https://nodejs.org/en/about/releases/ # https://nodejs.org/download/release/latest-v10.x/ -# https://nodejs.org/download/release/latest-v12.x/ -# https://nodejs.org/download/release/latest-v13.x/ -ENV NODE_VERSION=12.20.0 -ENV NODE_VERSION_FRAPPEV11=10.23.0 +# https://nodejs.org/download/release/latest-v14.x/ +ENV NODE_VERSION=14.17.0 +ENV NODE_VERSION_FRAPPEV11=10.24.1 # Install nvm with node -RUN wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh +RUN wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh RUN chmod +x install.sh RUN ./install.sh ENV NVM_DIR=/home/frappe/.nvm diff --git a/build/erpnext-nginx/Dockerfile b/build/erpnext-nginx/Dockerfile index 61bfa49d..3f6f758b 100644 --- a/build/erpnext-nginx/Dockerfile +++ b/build/erpnext-nginx/Dockerfile @@ -1,9 +1,12 @@ -ARG NODE_IMAGE_TAG=12-buster-slim -ARG GIT_BRANCH=develop +ARG NODE_IMAGE_TAG=14-buster-slim ARG DOCKER_REGISTRY_PREFIX=frappe +ARG IMAGE_TAG=develop + FROM node:${NODE_IMAGE_TAG} -ARG GIT_BRANCH +ARG GIT_REPO=https://github.com/frappe/erpnext +ARG GIT_BRANCH=develop + COPY build/erpnext-nginx/install_app.sh /install_app RUN chmod +x /install_app && \ apt-get update -y && \ @@ -11,9 +14,9 @@ RUN chmod +x /install_app && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN /install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH} +RUN /install_app erpnext ${GIT_REPO} ${GIT_BRANCH} -FROM ${DOCKER_REGISTRY_PREFIX}/frappe-nginx:${GIT_BRANCH} +FROM ${DOCKER_REGISTRY_PREFIX}/frappe-nginx:${IMAGE_TAG} COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/ COPY --from=0 /rsync /rsync diff --git a/build/erpnext-worker/Dockerfile b/build/erpnext-worker/Dockerfile index ef8718ca..8b1e16f0 100644 --- a/build/erpnext-worker/Dockerfile +++ b/build/erpnext-worker/Dockerfile @@ -1,6 +1,8 @@ -ARG GIT_BRANCH=develop +ARG IMAGE_TAG=develop ARG DOCKER_REGISTRY_PREFIX=frappe -FROM ${DOCKER_REGISTRY_PREFIX}/frappe-worker:${GIT_BRANCH} +FROM ${DOCKER_REGISTRY_PREFIX}/frappe-worker:${IMAGE_TAG} -ARG GIT_BRANCH -RUN install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH} +ARG GIT_REPO=https://github.com/frappe/erpnext +ARG GIT_BRANCH=develop + +RUN install_app erpnext ${GIT_REPO} ${GIT_BRANCH} diff --git a/build/frappe-nginx/Dockerfile b/build/frappe-nginx/Dockerfile index 9abeb22e..5c5a607d 100644 --- a/build/frappe-nginx/Dockerfile +++ b/build/frappe-nginx/Dockerfile @@ -4,14 +4,17 @@ # by Website Manager role in Frappe Framework FROM python:3.7-slim-buster +ARG GIT_REPO=https://github.com/frappe/frappe +ARG GIT_BRANCH=develop + ENV NVM_DIR=/root/.nvm -ENV NODE_VERSION=12.20.0 +ENV NODE_VERSION=14.17.0 ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" RUN apt-get update -y \ && apt-get install wget python2 git build-essential -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ - && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ + && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh \ && chmod +x install.sh \ && ./install.sh \ && . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} \ @@ -21,10 +24,9 @@ WORKDIR /home/frappe/frappe-bench RUN mkdir -p /home/frappe/frappe-bench/sites \ && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt -ARG GIT_BRANCH=develop RUN mkdir -p apps sites/assets/css \ && cd apps \ - && git clone --depth 1 https://github.com/frappe/frappe --branch $GIT_BRANCH + && git clone --depth 1 ${GIT_REPO} --branch $GIT_BRANCH RUN cd /home/frappe/frappe-bench/apps/frappe \ && yarn \ diff --git a/build/frappe-socketio/Dockerfile b/build/frappe-socketio/Dockerfile index 4bb1d36b..e7b72e09 100644 --- a/build/frappe-socketio/Dockerfile +++ b/build/frappe-socketio/Dockerfile @@ -1,3 +1,10 @@ +FROM alpine/git + +ARG GIT_REPO=https://github.com/frappe/frappe.git +ARG GIT_BRANCH=develop + +RUN git clone ${GIT_REPO} /opt/frappe -b ${GIT_BRANCH} --depth 1 + FROM node:buster-slim # Add frappe user @@ -9,15 +16,8 @@ RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frap # Download socketio and purge curl package COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe -ARG GIT_BRANCH=develop -RUN apt-get update && apt-get install -y curl \ - && cd /home/frappe/frappe-bench/apps/frappe \ - && curl "https://raw.githubusercontent.com/frappe/frappe/$GIT_BRANCH/socketio.js" \ - --output /home/frappe/frappe-bench/apps/frappe/socketio.js \ - && curl "https://raw.githubusercontent.com/frappe/frappe/$GIT_BRANCH/node_utils.js" \ - --output /home/frappe/frappe-bench/apps/frappe/node_utils.js \ - && apt-get purge -y curl && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* +COPY --from=0 /opt/frappe/socketio.js /home/frappe/frappe-bench/apps/frappe/socketio.js +COPY --from=0 /opt/frappe/node_utils.js /home/frappe/frappe-bench/apps/frappe/node_utils.js RUN cd /home/frappe/frappe-bench/apps/frappe \ && npm install --only=production \ diff --git a/build/frappe-socketio/package.json b/build/frappe-socketio/package.json index 4f8572d5..0c969142 100644 --- a/build/frappe-socketio/package.json +++ b/build/frappe-socketio/package.json @@ -1,6 +1,6 @@ { "name": "frappe-socketio", - "version": "1.0.0", + "version": "1.0.1", "description": "Frappe SocketIO Server", "main": "socketio.js", "scripts": { @@ -10,8 +10,8 @@ "license": "MIT", "dependencies": { "express": "^4.17.1", - "redis": "^2.8.0", - "socket.io": "^2.3.0", + "redis": "^3.1.1", + "socket.io": "^2.4.0", "superagent": "^5.1.0" } } diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 3e8313e6..ada2bda3 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -3,11 +3,12 @@ FROM python:3.7-slim-buster # Add non root user without password RUN useradd -ms /bin/bash frappe +ARG GIT_REPO=https://github.com/frappe/frappe ARG GIT_BRANCH=develop ARG ARCH=amd64 ENV PYTHONUNBUFFERED 1 ENV NVM_DIR=/home/frappe/.nvm -ENV NODE_VERSION=12.20.0 +ENV NODE_VERSION=14.17.0 ENV PATH="/home/frappe/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" # Install dependencies @@ -44,7 +45,7 @@ RUN apt-get update -y && apt-get install \ && if [ `uname -m` = 'x86_64' ]; then export ARCH=amd64; fi \ && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ && dpkg -i wkhtmltox_0.12.6-1.buster_${ARCH}.deb && rm wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ - && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ + && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh \ && chown -R frappe:frappe /home/frappe # Setup docker-entrypoint @@ -68,7 +69,7 @@ RUN python -m venv env \ && pip3 install --upgrade pip \ && pip3 install gevent \ && cd apps \ - && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch ${GIT_BRANCH} \ + && git clone --depth 1 -o upstream ${GIT_REPO} --branch ${GIT_BRANCH} \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe # Copy scripts and templates diff --git a/development/.vscode/launch.json b/development/.vscode/launch.json index efdc0903..8cd61e74 100644 --- a/development/.vscode/launch.json +++ b/development/.vscode/launch.json @@ -12,7 +12,6 @@ "args": [ "frappe", "serve", "--port", "8000", "--noreload", "--nothreading" ], - "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", "cwd": "${workspaceFolder}/frappe-bench/sites", "env": { "DEV_SERVER": "1" @@ -26,7 +25,6 @@ "args": [ "frappe", "worker", "--queue", "default" ], - "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", "cwd": "${workspaceFolder}/frappe-bench/sites", "env": { "DEV_SERVER": "1" @@ -40,7 +38,6 @@ "args": [ "frappe", "worker", "--queue", "short" ], - "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", "cwd": "${workspaceFolder}/frappe-bench/sites", "env": { "DEV_SERVER": "1" @@ -54,7 +51,6 @@ "args": [ "frappe", "worker", "--queue", "long" ], - "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", "cwd": "${workspaceFolder}/frappe-bench/sites", "env": { "DEV_SERVER": "1" @@ -64,7 +60,6 @@ "name": "Honcho SocketIO Watch Schedule Worker", "type": "python", "request": "launch", - "pythonPath": "/usr/bin/python3", "program": "/home/frappe/.local/bin/honcho", "cwd": "${workspaceFolder}/frappe-bench", "console": "internalConsole", diff --git a/travis.py b/travis.py index 6f801d36..bff2d3b4 100755 --- a/travis.py +++ b/travis.py @@ -4,37 +4,104 @@ import argparse import subprocess import os -def parse_args(): - parser = argparse.ArgumentParser(description='frappe_docker common CI elements', add_help=True) - parser.add_argument('service', action='store', type=str, help='Name of the service to build: "erpnext" or "frappe"') - parser.add_argument('-o', '--tag-only', required=False, action='store_true', dest='tag_only', help='Only tag an image and push it.') - parser.add_argument('-b', '--is-beta', required=False, default=False, action='store_true', dest='is_beta', help='Specify if tag is beta') +def parse_args(): + parser = argparse.ArgumentParser( + description="frappe_docker common CI elements", add_help=True + ) + + parser.add_argument( + "service", + action="store", + type=str, + help='Name of the service to build: "erpnext" or "frappe"', + ) + parser.add_argument( + "-o", + "--tag-only", + required=False, + action="store_true", + dest="tag_only", + help="Only tag an image and push it.", + ) + parser.add_argument( + "-b", + "--is-beta", + required=False, + default=False, + action="store_true", + dest="is_beta", + help="Specify if tag is beta", + ) image_type = parser.add_mutually_exclusive_group(required=True) - image_type.add_argument('-a', '--nginx', action='store_const', dest='image_type', const='nginx', help='Build the nginx + static assets image') - image_type.add_argument('-s', '--socketio', action='store_const', dest='image_type', const='socketio', help='Build the frappe-socketio image') - image_type.add_argument('-w', '--worker', action='store_const', dest='image_type', const='worker', help='Build the python environment image') + image_type.add_argument( + "-a", + "--nginx", + action="store_const", + dest="image_type", + const="nginx", + help="Build the nginx + static assets image", + ) + image_type.add_argument( + "-s", + "--socketio", + action="store_const", + dest="image_type", + const="socketio", + help="Build the frappe-socketio image", + ) + image_type.add_argument( + "-w", + "--worker", + action="store_const", + dest="image_type", + const="worker", + help="Build the python environment image", + ) tag_type = parser.add_mutually_exclusive_group(required=True) - tag_type.add_argument('-g', '--git-version', action='store', type=str, dest='version', help='The version number of service (i.e. "11", "12", etc.)') - tag_type.add_argument('-t', '--tag', action='store', type=str, dest='tag', help='The image tag (i.e. erpnext-worker:$TAG )') + tag_type.add_argument( + "-g", + "--git-version", + action="store", + type=str, + dest="version", + help='The version number of service (i.e. "11", "12", etc.)', + ) + tag_type.add_argument( + "-t", + "--tag", + action="store", + type=str, + dest="tag", + help="The image tag (i.e. erpnext-worker:$TAG )", + ) args = parser.parse_args() return args def git_version(service, version, branch, is_beta=False): - print(f'Pulling {service} v{version}') - subprocess.run(f'git clone https://github.com/frappe/{service} --branch {branch}', shell=True) + print(f"Pulling {service} v{version}") + subprocess.run( + f"git clone https://github.com/frappe/{service} --branch {branch}", shell=True + ) cd = os.getcwd() - os.chdir(os.getcwd() + f'/{service}') - subprocess.run('git fetch --tags', shell=True) + os.chdir(os.getcwd() + f"/{service}") + subprocess.run("git fetch --tags", shell=True) # XX-beta becomes XX for tags search - version = version.split('-')[0] + version = version.split("-")[0] - version_tag = subprocess.check_output(f'git tag --list --sort=-version:refname "v{version}*" | sed -n 1p | sed -e \'s#.*@\(\)#\\1#\'', shell=True).strip().decode('ascii') + version_tag = ( + subprocess.check_output( + f"git tag --list --sort=-version:refname \"v{version}*\" | sed -n 1p | sed -e 's#.*@\(\)#\\1#'", + shell=True, + ) + .strip() + .decode("ascii") + ) if not is_beta: version_tag = version_tag.split("-")[0] @@ -44,34 +111,40 @@ def git_version(service, version, branch, is_beta=False): def build(service, tag, image, branch): - build_args = f'--build-arg GIT_BRANCH={branch}' - if service == 'nginx' and branch == 'version-11': - build_args += f' --build-arg NODE_IMAGE_TAG=10-prod' + build_args = f"--build-arg GIT_BRANCH={branch}" + if service == "erpnext": + build_args += f" --build-arg IMAGE_TAG={branch}" + if image == "nginx" and branch == "version-11": + build_args += f" --build-arg NODE_IMAGE_TAG=10-prod" - print(f'Building {service} {image} image') - subprocess.run(f'docker build {build_args} -t {service}-{image} -f build/{service}-{image}/Dockerfile .', shell=True) - tag_and_push(f'{service}-{image}', tag) + print(f"Building {service} {image} image") + subprocess.run( + f"docker build {build_args} -t {service}-{image} -f build/{service}-{image}/Dockerfile .", + shell=True, + ) + tag_and_push(f"{service}-{image}", tag) def tag_and_push(image_name, tag): print(f'Tagging {image_name} as "{tag}" and pushing') - subprocess.run(f'docker tag {image_name} frappe/{image_name}:{tag}', shell=True) - subprocess.run(f'docker push frappe/{image_name}:{tag}', shell=True) + subprocess.run(f"docker tag {image_name} frappe/{image_name}:{tag}", shell=True) + subprocess.run(f"docker push frappe/{image_name}:{tag}", shell=True) def main(): args = parse_args() tag = args.tag - branch = 'develop' + branch = "develop" if args.version: - branch = 'version-' + args.version + branch = "version-" + args.version tag = git_version(args.service, args.version, branch, args.is_beta) if args.tag_only: - tag_and_push(f'{args.service}-{args.image_type}', tag) + tag_and_push(f"{args.service}-{args.image_type}", tag) else: build(args.service, tag, args.image_type, branch) + if __name__ == "__main__": main() From b3ac8383fec3dfc2aaa3a74c71b445cacc7db115 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 21 May 2021 18:11:00 +0530 Subject: [PATCH 3/3] fix(frappe-nginx): sync assets path for all versions --- build/frappe-nginx/docker-entrypoint.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build/frappe-nginx/docker-entrypoint.sh b/build/frappe-nginx/docker-entrypoint.sh index b478e738..68314676 100755 --- a/build/frappe-nginx/docker-entrypoint.sh +++ b/build/frappe-nginx/docker-entrypoint.sh @@ -6,9 +6,8 @@ set -e -rsync -a --delete /var/www/html/assets/js /assets -rsync -a --delete /var/www/html/assets/css /assets -rsync -a --delete /var/www/html/assets/frappe /assets +rsync -a --delete /var/www/html/assets/* /assets + . /rsync touch /var/www/html/sites/.build -r $(ls -td /assets/* | head -n 1)