feat!: remove dependency on Paver scripts (#1042)

BREAKING CHANGE: `openedx-assets` is replaed with `npm run` subcommands.
For details, see the changelog entry.
For further details and rationale, see the upstream DEPR ticket:
https://github.com/openedx/edx-platform/issues/31895

changelog.d/20240318_154804_kyle_assets_build.md

@@ -0,0 +1,21 @@
+- 💥[Feature] The `openedx-assets` command is replaced with `npm run` subcommands.
+  This will slightly reduce the build time for edx-platform assets and comprehensive themes.
+  It will also open up the door for more significant build time reductions in the future.
+  Here is a migration guide, where each command is to be run in the `lms` or `cms` container.
+
+  **Before**                               | **After**                                                                           
+  -----------------------------------------|-------------------------------------------------------------------------------------
+  `openedx-assets build --env=prod ARGS`   | `npm run build -- ARGS`                                                             
+  `openedx-assets build --env=dev ARGS`    | `npm run build-dev -- ARGS`                                                         
+  `openedx-assets common --env=prod ARGS`  | `npm run compile-sass -- --skip-themes ARGS`                                    
+  `openedx-assets common --env=dev ARGS`   | `npm run compile-sass-dev -- --skip-themes ARGS`                                    
+  `openedx-assets webpack --env=prod ARGS` | `npm run webpack -- ARGS`                                                           
+  `openedx-assets webpack --env=dev ARGS`  | `npm run webpack-dev -- ARGS`                                                       
+  `openedx-assets npm`                     | `npm run postinstall` (`npm clean-install` runs this automatically)                 
+  `openedx-assets xmodule`                 | (no longer necessary)                                                               
+  `openedx-assets collect ARGS`            | `./manage.py lms collectstatic --noinput ARGS && ./manage.py cms collectstatic ARGS` 
+  `openedx-assets watch-themes ARGS`       | `npm run watch-themes -- ARGS`                                                      
+
+  For more details, see the [deprecation notice for paver](https://github.com/openedx/edx-platform/issues/34467)
+  and the [static assets reference](https://github.com/openedx/edx-platform/tree/open-release/redwood.master/docs/references/static-assets.rst)
+  in edx-platform.

docs/dev.rst

@@ -99,9 +99,9 @@ To open a python shell in the LMS or CMS, run::
 
 You can then import edx-platform and django modules and execute python code.
 
-To rebuild assets, you can use the ``openedx-assets`` command that ships with Tutor::
+To rebuild assets, you can run the ``build-dev`` NPM script that comes with edx-plaform::
 
-    tutor dev run lms openedx-assets build --env=dev
+    tutor dev run lms npm run build-dev
 
 
 .. _specialized for developer usage:

tutor/templates/apps/openedx/config/cms.env.yml

@@ -27,8 +27,6 @@ CELERY_BROKER_USER: "{{ REDIS_USERNAME }}"
 CELERY_BROKER_PASSWORD: "{{ REDIS_PASSWORD }}"
 ALTERNATE_WORKER_QUEUES: "lms"
 ENABLE_COMPREHENSIVE_THEMING: true
-COMPREHENSIVE_THEME_DIRS: ["/openedx/themes"]
-STATIC_ROOT_BASE: "/openedx/staticfiles"
 EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend"
 EMAIL_HOST: "{{ SMTP_HOST }}"
 EMAIL_PORT: {{ SMTP_PORT }}

tutor/templates/apps/openedx/config/lms.env.yml

@@ -33,8 +33,6 @@ CELERY_BROKER_USER: "{{ REDIS_USERNAME }}"
 CELERY_BROKER_PASSWORD: "{{ REDIS_PASSWORD }}"
 ALTERNATE_WORKER_QUEUES: "cms"
 ENABLE_COMPREHENSIVE_THEMING: true
-COMPREHENSIVE_THEME_DIRS: ["/openedx/themes"]
-STATIC_ROOT_BASE: "/openedx/staticfiles"
 EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend"
 EMAIL_HOST: "{{ SMTP_HOST }}"
 EMAIL_PORT: {{ SMTP_PORT }}

tutor/templates/apps/openedx/settings/partials/common_cms.py

@@ -21,7 +21,7 @@ FRONTEND_LOGIN_URL = LMS_ROOT_URL + '/login'
 FRONTEND_REGISTER_URL = LMS_ROOT_URL + '/register'
 
 # Create folders if necessary
-for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]:
+for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT, ORA2_FILEUPLOAD_ROOT]:
     if not os.path.exists(folder):
         os.makedirs(folder, exist_ok=True)
 

tutor/templates/apps/openedx/settings/partials/common_lms.py

@@ -38,7 +38,7 @@ CACHES["staticfiles"] = {
 }
 
 # Create folders if necessary
-for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]:
+for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT, ORA2_FILEUPLOAD_ROOT]:
     if not os.path.exists(folder):
         os.makedirs(folder, exist_ok=True)
 

tutor/templates/build/openedx/Dockerfile

@@ -162,6 +162,10 @@ RUN ln -s /openedx/node_modules /openedx/edx-platform/node_modules
 
 ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH}
 ENV VIRTUAL_ENV /openedx/venv/
+ENV COMPREHENSIVE_THEME_DIRS /openedx/themes
+ENV STATIC_ROOT_LMS /openedx/staticfiles
+ENV STATIC_ROOT_CMS /openedx/staticfiles/studio
+
 WORKDIR /openedx/edx-platform
 
 {# Install auto-mounted directories as Python packages. #}
@@ -204,27 +208,22 @@ ENV PATH /openedx/bin:${PATH}
 
 {{ patch("openedx-dockerfile-pre-assets") }}
 
-# Collect production assets. By default, only assets from the default theme
+# Build & collect production assets. By default, only assets from the default theme
 # will be processed. This makes the docker image lighter and faster to build.
-# Only the custom themes added to /openedx/themes will be compiled.
+RUN npm run postinstall  # Postinstall artifacts are stuck in nodejs-requirements layer. Create them here too.
-# Here, we don't run "paver update_assets" which is slow, compiles all themes
+RUN npm run compile-sass -- --skip-themes
-# and requires a complex settings file. Instead, we decompose the commands
+RUN npm run webpack
-# and run each one individually to collect the production static assets to
+
-# /openedx/staticfiles.
+# Now that the default theme is built, build any custom themes
-ENV NO_PYTHON_UNINSTALL 1
+COPY --chown=app:app ./themes/ /openedx/themes
-ENV NO_PREREQ_INSTALL 1
+RUN npm run compile-sass -- --skip-default
-# We need to rely on a separate openedx-assets command to accelerate asset processing.
+
-# For instance, we don't want to run all steps of asset collection every time the theme
+# and finally, collect assets for the production image,
-# is modified.
+# de-duping assets with symlinks.
-RUN openedx-assets xmodule \
+RUN ./manage.py lms collectstatic --noinput --settings=tutor.assets && \
-    && openedx-assets npm \
+    ./manage.py cms collectstatic --noinput --settings=tutor.assets && \
-    && openedx-assets webpack --env=prod \
+    # De-duplicate static assets with symlinks \
-    && openedx-assets common
+    rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/
-COPY --chown=app:app ./themes/ /openedx/themes/
-RUN openedx-assets themes \
-    && openedx-assets collect --settings=tutor.assets \
-    # De-duplicate static assets with symlinks
-    && rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/
 
 # Create a data directory, which might be used (or not)
 RUN mkdir /openedx/data
@@ -277,7 +276,7 @@ ENV PYTHONBREAKPOINT=ipdb.set_trace
 # static assets, then production assets will be served instead.
 RUN rm -r /openedx/staticfiles && \
     mkdir /openedx/staticfiles && \
-    openedx-assets webpack --env=dev
+    npm run build-dev
 
 {{ patch("openedx-dev-dockerfile-post-python-requirements") }}
 

tutor/templates/build/openedx/bin/openedx-assets

@@ -1,218 +0,0 @@
-#! /usr/bin/env python
-from __future__ import print_function
-import argparse
-import os
-import subprocess
-import sys
-import traceback
-
-from path import Path
-
-from pavelib import assets
-
-
-DEFAULT_STATIC_ROOT = "/openedx/staticfiles"
-DEFAULT_THEMES_DIR = "/openedx/themes"
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Various assets processing/building/collection utility for Open edX"
-    )
-    subparsers = parser.add_subparsers()
-
-    npm = subparsers.add_parser("npm", help="Copy static assets from node_modules")
-    npm.set_defaults(func=run_npm)
-
-    build = subparsers.add_parser("build", help="Build all assets")
-    build.add_argument("-e", "--env", choices=["prod", "dev"], default="prod")
-    build.add_argument("--theme-dirs", nargs="+", default=[DEFAULT_THEMES_DIR])
-    build.add_argument("--themes", nargs="+", default=["all"])
-    build.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT)
-    build.add_argument("--systems", nargs="+", default=["lms", "cms"])
-    build.set_defaults(func=run_build)
-
-    xmodule = subparsers.add_parser("xmodule", help="Process assets from xmodule")
-    xmodule.set_defaults(func=run_xmodule)
-
-    webpack = subparsers.add_parser("webpack", help="Run webpack")
-    webpack.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT)
-    webpack.add_argument("-e", "--env", choices=["prod", "dev"], default="prod")
-    webpack.set_defaults(func=run_webpack)
-
-    common = subparsers.add_parser(
-        "common", help="Compile static assets for common theme"
-    )
-    common.add_argument("--systems", nargs="+", default=["lms", "cms"])
-    common.set_defaults(func=run_common)
-
-    themes = subparsers.add_parser(
-        "themes", help="Compile static assets for custom themes"
-    )
-    themes.add_argument("--theme-dirs", nargs="+", default=[DEFAULT_THEMES_DIR])
-    themes.add_argument("--themes", nargs="+", default=["all"])
-    themes.add_argument("--systems", nargs="+", default=["lms", "cms"])
-    themes.set_defaults(func=run_themes)
-
-    collect = subparsers.add_parser(
-        "collect", help="Collect static assets to be served by webserver"
-    )
-    collect.add_argument(
-        "-s",
-        "--settings",
-        default="tutor.assets",
-        help="Django settings module",
-    )
-    collect.add_argument(
-        "--systems",
-        nargs="+",
-        choices=["lms", "cms"],
-        default=["lms", "cms"],
-        help="Limit collection to lms or cms",
-    )
-    collect.set_defaults(func=run_collect)
-
-    watch_themes = subparsers.add_parser(
-        "watch-themes", help="Watch theme assets for changes and recompile on-the-fly"
-    )
-    watch_themes.add_argument(
-        "-e",
-        "--env",
-        choices=["prod", "dev"],
-        default="prod",
-        help="Webpack target to run",
-    )
-    watch_themes.add_argument("--theme-dirs", default=[DEFAULT_THEMES_DIR])
-    watch_themes.set_defaults(func=run_watch_themes)
-
-    args = parser.parse_args()
-    args.func(args)
-
-
-def run_build(args):
-    run_xmodule(args)
-    run_npm(args)
-    run_webpack(args)
-    run_common(args)
-    run_themes(args)
-
-
-def run_xmodule(_args):
-    # Collecting xmodule assets is incompatible with setting the django path, because
-    # of an unfortunate call to settings.configure()
-    django_settings_module = os.environ.get("DJANGO_SETTINGS_MODULE")
-    if django_settings_module:
-        os.environ.pop("DJANGO_SETTINGS_MODULE")
-
-    sys.argv[1:] = ["common/static/xmodule"]
-    import xmodule.static_content
-
-    xmodule.static_content.main()
-
-    if django_settings_module:
-        os.environ["DJANGO_SETTINGS_MODULE"] = django_settings_module
-
-
-def run_npm(_args):
-    assets.process_npm_assets()
-
-
-def run_webpack(args):
-    os.environ["STATIC_ROOT_LMS"] = args.static_root
-    os.environ["STATIC_ROOT_CMS"] = os.path.join(args.static_root, "studio")
-    os.environ["NODE_ENV"] = {"prod": "production", "dev": "development"}[args.env]
-    subprocess.check_call(
-        [
-            "webpack",
-            "--progress",
-            "--config=webpack.{env}.config.js".format(env=args.env),
-        ]
-    )
-
-
-def run_common(args):
-    for system in args.systems:
-        print("Compiling {} sass assets from common theme...".format(system))
-        assets._compile_sass(system, None, False, False, [])
-
-
-def run_themes(args):
-    for theme_dir in args.theme_dirs:
-        local_themes = (
-            list_subdirectories(theme_dir) if "all" in args.themes else args.themes
-        )
-        for theme in local_themes:
-            theme_path = os.path.join(theme_dir, theme)
-            if os.path.exists(theme_path):
-                for system in args.systems:
-                    print(
-                        "Compiling {} sass assets from theme {}...".format(
-                            system, theme_path
-                        )
-                    )
-                    assets._compile_sass(system, Path(theme_path), False, False, [])
-
-
-def run_collect(args):
-    assets.collect_assets(args.systems, args.settings)
-
-
-def run_watch_themes(args):
-    """
-    Watch static assets for changes and re-compile those changes when
-    necessary. This piece of code is heavily inspired from the
-    edx-platform/pavelib/assets.py:watch_assets function, which could not be
-    used directly because it does not properly read the platform settings
-    environment variable.
-
-    Note that this function will only work for watching assets in development
-    mode. In production, watching changes does not make much sense anyway.
-    """
-    observer = assets.Observer()
-    for theme_dir in args.theme_dirs:
-        print("Watching changes in {}...".format(theme_dir))
-        ThemeWatcher(theme_dir).register(observer)
-    observer.start()
-    try:
-        while True:
-            observer.join(2)
-    except KeyboardInterrupt:
-        observer.stop()
-
-
-def list_subdirectories(path):
-    return [
-        subpath
-        for subpath in os.listdir(path)
-        if os.path.isdir(os.path.join(path, subpath))
-    ]
-
-
-class ThemeWatcher(assets.SassWatcher):
-    def __init__(self, theme_dir):
-        super(ThemeWatcher, self).__init__()
-        self.theme_dir = theme_dir
-
-    # pylint: disable=arguments-differ
-    def register(self, observer):
-        return super(ThemeWatcher, self).register(observer, [self.theme_dir])
-
-    @assets.debounce()
-    def on_any_event(self, event):
-        components = os.path.relpath(event.src_path, self.theme_dir).split("/")
-        try:
-            theme = components[0]
-            system = components[1]
-        except IndexError:
-            return
-        try:
-            print("Detected change:", event.src_path)
-            print("\tRecompiling {} theme for {}".format(theme, system))
-            assets._compile_sass(system, Path(self.theme_dir) / theme, False, False, [])
-            print("\tDone recompiling {} theme for {}".format(theme, system))
-        except Exception:  # pylint: disable=broad-except
-            traceback.print_exc()
-
-
-if __name__ == "__main__":
-    main()

tutor/templates/build/openedx/settings/cms/assets.py

@@ -1,6 +1,5 @@
 {% include "build/openedx/settings/partials/assets.py" %}
 
-STATIC_ROOT = path(STATIC_ROOT_BASE) / 'studio'
+WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = path(STATIC_ROOT) / "webpack-stats.json"
-WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"
 
 derive_settings(__name__)

tutor/templates/build/openedx/settings/lms/assets.py

@@ -1,6 +1,5 @@
 {% include "build/openedx/settings/partials/assets.py" %}
 
-STATIC_ROOT = path(STATIC_ROOT_BASE)
+WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = path(STATIC_ROOT) / "webpack-stats.json"
-WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"
 
 derive_settings(__name__)

tutor/templates/build/openedx/settings/partials/assets.py

@@ -5,9 +5,6 @@ from ..common import *
 from openedx.core.lib.derived import derive_settings
 
 ENABLE_COMPREHENSIVE_THEMING = True
-COMPREHENSIVE_THEME_DIRS.append('/openedx/themes')
-
-STATIC_ROOT_BASE = '/openedx/staticfiles'
 
 SECRET_KEY = 'secret'
 XQUEUE_INTERFACE = {

tutor/templates/build/openedx/settings/partials/i18n.py

@@ -1,8 +1,6 @@
 from ..common import *
 from openedx.core.lib.derived import derive_settings
 
-STATIC_ROOT_BASE = '/openedx/staticfiles'
-
 SECRET_KEY = 'secret'
 XQUEUE_INTERFACE = {
     'django_auth': None,

tutor/templates/dev/docker-compose.yml

@@ -43,7 +43,7 @@ services:
   # Additional service for watching theme changes
   watchthemes:
     <<: *openedx-service
-    command: openedx-assets watch-themes --env dev
+    command: npm run watch-sass
     restart: unless-stopped
 
   {% if RUN_ELASTICSEARCH and is_docker_rootless() %}

tutor/templates/jobs/init/mounted-directories.sh

@@ -37,7 +37,7 @@ pip install -e .
 npm clean-install
 
 # Regenerate static assets.
-openedx-assets build --env=dev
+npm run build-dev
 
 set -x
 echo "Done setting up bind-mounted edx-platform."