From 6b94d991e218623b41007b5d5186e6e2e6059012 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 5 Aug 2020 21:40:37 +0530 Subject: [PATCH 01/11] fix: add nodejs via nvm --- build/common/worker/docker-entrypoint.sh | 4 ++++ build/frappe-nginx/build.sh | 2 +- build/frappe-worker/Dockerfile | 18 ++++++++++++++++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/build/common/worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh index 5b68eec4..7fb731f1 100755 --- a/build/common/worker/docker-entrypoint.sh +++ b/build/common/worker/docker-entrypoint.sh @@ -71,6 +71,10 @@ fi # Allow user process to create files in logs directory chown -R frappe:frappe /home/frappe/frappe-bench/logs +# symlink node_modules +ln -s /home/frappe/frappe-bench/sites/assets/frappe/node_modules \ + /home/frappe/frappe-bench/apps/frappe/node_modules + if [ "$1" = 'start' ]; then configureEnv checkConnection diff --git a/build/frappe-nginx/build.sh b/build/frappe-nginx/build.sh index 370e2104..130c34b7 100755 --- a/build/frappe-nginx/build.sh +++ b/build/frappe-nginx/build.sh @@ -8,7 +8,7 @@ function nodeCleanUp() { cd /home/frappe/frappe-bench/apps/frappe yarn yarn run production -yarn add nunjucks -D +yarn add nunjucks node-sass -D if [[ "$GIT_BRANCH" =~ ^(version-12|version-11)$ ]]; then nodeCleanUp diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index bcf78612..89f1a5bf 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -4,6 +4,9 @@ RUN useradd -ms /bin/bash frappe ARG GIT_BRANCH=develop ENV PYTHONUNBUFFERED 1 +ENV NVM_DIR=/home/frappe/.nvm +ENV NODE_VERSION=14.7.0 +ENV PATH="/home/frappe/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" # Install dependencies WORKDIR /home/frappe/frappe-bench @@ -27,7 +30,17 @@ RUN install_packages \ libpq-dev \ && wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_amd64.deb \ && dpkg -i wkhtmltox_0.12.5-1.buster_amd64.deb && rm wkhtmltox_0.12.5-1.buster_amd64.deb \ - && apt-get purge -y wget && apt-get autoremove -y + && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ + && apt-get purge -y wget && apt-get autoremove -y \ + && chown -R frappe:frappe /home/frappe + +USER frappe +# Install nvm with node +RUN bash install.sh \ + && . "$NVM_DIR/nvm.sh" \ + && nvm install ${NODE_VERSION} \ + && nvm use v${NODE_VERSION} \ + && nvm alias default v${NODE_VERSION} # Create frappe-bench directories RUN mkdir -p apps logs commands /home/frappe/backups @@ -39,6 +52,7 @@ RUN python -m venv env \ && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch ${GIT_BRANCH} \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe +USER root # Copy scripts and templates COPY build/common/commands/* /home/frappe/frappe-bench/commands/ COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template @@ -52,7 +66,7 @@ RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites # Set ownership of sites directory -RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites /home/frappe/backups +RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/backups" ] From 6c3ccd12fa071290a1a508b6d672d6714f8ff47b Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 6 Aug 2020 14:24:22 +0530 Subject: [PATCH 02/11] fix: add node-sass to nodejs dependencies --- build/frappe-nginx/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/frappe-nginx/build.sh b/build/frappe-nginx/build.sh index 130c34b7..f7e63092 100755 --- a/build/frappe-nginx/build.sh +++ b/build/frappe-nginx/build.sh @@ -8,7 +8,8 @@ function nodeCleanUp() { cd /home/frappe/frappe-bench/apps/frappe yarn yarn run production -yarn add nunjucks node-sass -D +# remove this when frappe framework moves this to dependencies from devDependencies +yarn add nunjucks node-sass if [[ "$GIT_BRANCH" =~ ^(version-12|version-11)$ ]]; then nodeCleanUp From a69853e950693078ccf1806f5d417883a9cf78bf Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 6 Aug 2020 14:26:05 +0530 Subject: [PATCH 03/11] fix: add nunjucks to nodejs devDependencies --- build/frappe-nginx/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/frappe-nginx/build.sh b/build/frappe-nginx/build.sh index f7e63092..5dbee8e9 100755 --- a/build/frappe-nginx/build.sh +++ b/build/frappe-nginx/build.sh @@ -8,8 +8,10 @@ function nodeCleanUp() { cd /home/frappe/frappe-bench/apps/frappe yarn yarn run production +# Remove this after python container starts using node command +yarn add nunjucks -D # remove this when frappe framework moves this to dependencies from devDependencies -yarn add nunjucks node-sass +yarn add node-sass if [[ "$GIT_BRANCH" =~ ^(version-12|version-11)$ ]]; then nodeCleanUp From 0d9aa27f5fd34cab6179cf79bba21e61e0f6fec1 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 6 Aug 2020 15:18:32 +0530 Subject: [PATCH 04/11] fix: symlink to node_modules --- build/common/worker/docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh index 7fb731f1..2ad58b9c 100755 --- a/build/common/worker/docker-entrypoint.sh +++ b/build/common/worker/docker-entrypoint.sh @@ -72,7 +72,7 @@ fi chown -R frappe:frappe /home/frappe/frappe-bench/logs # symlink node_modules -ln -s /home/frappe/frappe-bench/sites/assets/frappe/node_modules \ +ln -sfn /home/frappe/frappe-bench/sites/assets/frappe/node_modules \ /home/frappe/frappe-bench/apps/frappe/node_modules if [ "$1" = 'start' ]; then From a7cd40a2b74b0c78dd5b5afdd6dba4fbe4630e81 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 6 Aug 2020 19:30:27 +0530 Subject: [PATCH 05/11] fix: add node-sass to nodejs dependencies after prod install --- build/frappe-nginx/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/frappe-nginx/build.sh b/build/frappe-nginx/build.sh index 5dbee8e9..0a1d52f5 100755 --- a/build/frappe-nginx/build.sh +++ b/build/frappe-nginx/build.sh @@ -10,8 +10,6 @@ yarn yarn run production # Remove this after python container starts using node command yarn add nunjucks -D -# remove this when frappe framework moves this to dependencies from devDependencies -yarn add node-sass if [[ "$GIT_BRANCH" =~ ^(version-12|version-11)$ ]]; then nodeCleanUp @@ -23,4 +21,6 @@ else /home/frappe/frappe-bench/sites/assets/css/standard_style.css \ "$(cat /home/frappe/standard_templates_string)" nodeCleanUp + # remove this when frappe framework moves this to dependencies from devDependencies + yarn add node-sass fi From 7c293d7f9e7406c8ca84726b4ee5e1d9e7698691 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 6 Aug 2020 22:38:52 +0530 Subject: [PATCH 06/11] fix: use same python and nginx image --- build/frappe-nginx/Dockerfile | 13 +++++++++++-- build/frappe-worker/Dockerfile | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/build/frappe-nginx/Dockerfile b/build/frappe-nginx/Dockerfile index fc17a7b6..e9147aad 100644 --- a/build/frappe-nginx/Dockerfile +++ b/build/frappe-nginx/Dockerfile @@ -1,5 +1,14 @@ -ARG NODE_IMAGE_TAG=12-prod -FROM bitnami/node:${NODE_IMAGE_TAG} +FROM bitnami/python:3.7-prod + +ENV NVM_DIR=/root/.nvm +ENV NODE_VERSION=12.18.3 +ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" +RUN install_packages wget \ + && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ + && chmod +x install.sh \ + && ./install.sh \ + && . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} \ + && nvm use v${NODE_VERSION} && npm install -g yarn WORKDIR /home/frappe/frappe-bench RUN mkdir -p /home/frappe/frappe-bench/sites \ diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 89f1a5bf..68226660 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -5,7 +5,7 @@ RUN useradd -ms /bin/bash frappe ARG GIT_BRANCH=develop ENV PYTHONUNBUFFERED 1 ENV NVM_DIR=/home/frappe/.nvm -ENV NODE_VERSION=14.7.0 +ENV NODE_VERSION=12.18.3 ENV PATH="/home/frappe/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" # Install dependencies From 9dff8f2561d2e61c0304ce52620480185217cbdb Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Fri, 7 Aug 2020 12:21:15 +0530 Subject: [PATCH 07/11] chore: change public file slug in push-backup Signed-off-by: Chinmay D. Pai --- build/common/commands/push_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/commands/push_backup.py b/build/common/commands/push_backup.py index ce3fa3d8..b968862f 100644 --- a/build/common/commands/push_backup.py +++ b/build/common/commands/push_backup.py @@ -17,7 +17,7 @@ def get_file_ext(): return { "database": "-database.sql.gz", "private_files": "-private-files.tar", - "public_files": "-files.tar", + "public_files": "-public-files.tar", "site_config": "-site_config_backup.json" } From dcf0c0f8b14b29cc18b1cc84ee5e3c616a8cfa3c Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Fri, 7 Aug 2020 12:47:20 +0530 Subject: [PATCH 08/11] chore: change public files slug for restore-backup Signed-off-by: Chinmay D. Pai --- build/common/commands/restore_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 6368d03a..0dd8f3f4 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -77,7 +77,7 @@ def restore_database(files_base, site_config_path, site): def restore_files(files_base): - public_files = files_base + '-files.tar' + public_files = files_base + '-public-files.tar' # extract tar public_tar = tarfile.open(public_files) print('Extracting {}'.format(public_files)) From a3bdc3c8f90f32c6ef6af7d5b532ee40288a930f Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 7 Aug 2020 15:28:15 +0530 Subject: [PATCH 09/11] fix: remove nunjucks from nginx image preparation --- build/frappe-nginx/Dockerfile | 7 ++++--- build/frappe-nginx/build.sh | 8 -------- build/frappe-nginx/generate_standard_style_css.js | 14 -------------- 3 files changed, 4 insertions(+), 25 deletions(-) delete mode 100644 build/frappe-nginx/generate_standard_style_css.js diff --git a/build/frappe-nginx/Dockerfile b/build/frappe-nginx/Dockerfile index e9147aad..d360f87f 100644 --- a/build/frappe-nginx/Dockerfile +++ b/build/frappe-nginx/Dockerfile @@ -1,3 +1,7 @@ +# This image uses nvm and same base image as the worker image. +# This is done to ensures that node-sass binary remains common. +# node-sass is required to enable website theme feature used +# by Website Manager role in Frappe Framework FROM bitnami/python:3.7-prod ENV NVM_DIR=/root/.nvm @@ -21,9 +25,6 @@ RUN mkdir -p apps sites/assets/css \ && cd apps \ && git clone --depth 1 https://github.com/frappe/frappe --branch $GIT_BRANCH -COPY build/frappe-nginx/generate_standard_style_css.js \ - /home/frappe/frappe-bench/apps/frappe/generate_standard_style_css.js - COPY build/frappe-nginx/build.sh /build.sh RUN /build.sh RUN node --version \ diff --git a/build/frappe-nginx/build.sh b/build/frappe-nginx/build.sh index 0a1d52f5..f9b01dbb 100755 --- a/build/frappe-nginx/build.sh +++ b/build/frappe-nginx/build.sh @@ -8,18 +8,10 @@ function nodeCleanUp() { cd /home/frappe/frappe-bench/apps/frappe yarn yarn run production -# Remove this after python container starts using node command -yarn add nunjucks -D if [[ "$GIT_BRANCH" =~ ^(version-12|version-11)$ ]]; then nodeCleanUp else - node generate_standard_style_css.js \ - frappe/website/doctype/website_theme/website_theme_template.scss > \ - /home/frappe/standard_templates_string - node generate_bootstrap_theme.js \ - /home/frappe/frappe-bench/sites/assets/css/standard_style.css \ - "$(cat /home/frappe/standard_templates_string)" nodeCleanUp # remove this when frappe framework moves this to dependencies from devDependencies yarn add node-sass diff --git a/build/frappe-nginx/generate_standard_style_css.js b/build/frappe-nginx/generate_standard_style_css.js deleted file mode 100644 index 275ab47d..00000000 --- a/build/frappe-nginx/generate_standard_style_css.js +++ /dev/null @@ -1,14 +0,0 @@ -const nunjucks = require("nunjucks"); - -const templatePath = process.argv[2]; -const pathArray = templatePath.split("/"); -const templateFile = pathArray.pop(); -const templateDir = pathArray.join("/"); - -nunjucks.configure(templateDir); -const rendered = nunjucks.render(templateFile, { - button_rounded_corners: true, - font_properties: "300,600", -}); - -console.log(rendered.replace(/\n/gm, "\\n")); From 1c74ac2e0bb7e41e13ad9ca8ae8be78c04f8536a Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 7 Aug 2020 16:02:15 +0530 Subject: [PATCH 10/11] Revert "chore: change public file slug in push-backup" This reverts commit 9dff8f2561d2e61c0304ce52620480185217cbdb. --- build/common/commands/push_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/commands/push_backup.py b/build/common/commands/push_backup.py index b968862f..ce3fa3d8 100644 --- a/build/common/commands/push_backup.py +++ b/build/common/commands/push_backup.py @@ -17,7 +17,7 @@ def get_file_ext(): return { "database": "-database.sql.gz", "private_files": "-private-files.tar", - "public_files": "-public-files.tar", + "public_files": "-files.tar", "site_config": "-site_config_backup.json" } From bd157c6de9613694bcb803fd450510a0dcf47a36 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 7 Aug 2020 16:02:45 +0530 Subject: [PATCH 11/11] Revert "chore: change public files slug for restore-backup" This reverts commit dcf0c0f8b14b29cc18b1cc84ee5e3c616a8cfa3c. --- build/common/commands/restore_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 0dd8f3f4..6368d03a 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -77,7 +77,7 @@ def restore_database(files_base, site_config_path, site): def restore_files(files_base): - public_files = files_base + '-public-files.tar' + public_files = files_base + '-files.tar' # extract tar public_tar = tarfile.open(public_files) print('Extracting {}'.format(public_files))