christianlight
/
tutor
mirror of https://github.com/ChristianLight/tutor.git
7
0
Fork 0
Code Issues Releases Activity

Tutor v3 complete rewrite 

Replace all make commands by a single "tutor" binary. Environment and
data are all moved to ~/.tutor/local/share/tutor. We take the
opportunity to add a web UI and revamp the documentation.

This is a complete rewrite.

Close #121.
Close #147.
This commit is contained in:
Régis Behmo 2019-01-22 21:25:04 +01:00
parent 2c955ea9bb
commit 4331bc5712
131 changed files with 2585 additions and 1457 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/ISSUE_TEMPLATE.md vendored
View File

@ -6,8 +6,8 @@ Include the exact command that you are running and that is causing an error. In
## Unexpected behavior
Include the full, exact output from the command that is causing your issue. Also include relevant error logs; for instance, to debug the LMS take a look at the files in `data/lms/logs`.
Include the full, exact output from the command that is causing your issue. Also include relevant error logs; for instance, to debug the LMS provide the output of `tutor local logs lms --tail=100`
## Additional info (IMPORTANT)
Include the output of the `make info` command.
Provide the output of `tutor --version`.

6
.gitignore vendored
View File

@ -1,5 +1,7 @@
.*.swp
!.gitignore
/config.json
/data*/
TODO
/build/tutor
/dist/
/tutor_openedx.egg-info/

79
.travis.yml
View File

@ -1,11 +1,72 @@
language: minimal
services:
- docker
language: python
matrix:
include:
- os: linux
# We need an older version of python in order to have compatibility with
# older versions of libc
dist: trusty
python: 3.6
services:
- docker
env:
BUILD_BINARY: "true"
- os: linux
dist: xenial
python: 3.6
services:
- docker
env:
BUILD_DOCKER: "true"
- os: linux
dist: xenial
python: 3.6
services:
- docker
env:
BUILD_PYPI: "true"
- os: osx
language: generic
env:
BUILD_BINARY: "true"
script:
- make travis
- python3 --version
- pip3 --version
- pip3 install -U setuptools
- pip3 install -r requirements/dev.txt
- make bundle
- ./dist/tutor config noninteractive
- ./dist/tutor images env
- ./dist/tutor local env
before_deploy:
- cp ./dist/tutor ./dist/tutor-$TRAVIS_OS_NAME
deploy:
provider: script
script: cd build/ && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && make push
on:
all_branches: true
condition: $TRAVIS_BRANCH =~ ^master|release\/.*$
# Push tutor binary to github releases
- provider: releases
api_key:
secure: ffrhGvgxHf3WVnKsKiJZTUoEf3+ZBtIydPHQoTDmHiYMyPnaO6RJWxq5PnoDRr47b1vnvbCE0lhNsGr3sgnruQSN3xHYfT4wz1F6Lz7Y5Jt7Uq1W0/UfSRlB3BwYcFpkUb5fSXLSku+QrV8OTk/yItlY9tppnvXA9h4CZjQqgJYHYc1DRKWQVYnVs/dCttUpiPV3eyIFeQoPKFsHwZXKcm9cVzom5r+lkjwpw3AIuAutPd71jyj/Va/By6B/43yAIqw3sA/thBeL76vqd8C4cMeWE00of1EWnH+sx6pKz32Fqe/o6dVop5PZPCiI+TVIDxR8oLevHtPTCfIwRDg60y23DdH3bMGSw1bAyjeWTnhRGcdYQJi1NKq3hJ0ldy0lOFlUiMkKPdQBtU7i0xIpoXTIhnro0YUUtjbh/QEtyRn8nbMVeenId42Bymah5P8srhE8S2z0x0mirIqNlM/tgLKEOJXdSL4sPKCMwRLcTUIc0fCiwLeHJnHIrMNEfnWsqO1odzv/PS5nLsdiGHBmC63d+xLNllzincIV1djyi5SEzMZxcqmjX1n/G3nKYVXhaIQE0wWQSHqNwLlKluY0kPXCNtU1TPr5SJHGnomQKizVaEsDrdAjylBL+rPwVCAv9z8FCtyOg7uMx2WdD0pvky2Iy4rbl2RBLUHmUZoAjPY=
file: dist/tutor-$TRAVIS_OS_NAME
skip_cleanup: true
on:
tags: true
condition: $BUILD_BINARY = true
# Push docker images to docker hub
- provider: script
script: ./dist/tutor images build all && \
./dist/tutor local databases && \
docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && \
./dist/tutor images push all
skip_cleanup: true
on:
tags: true
condition: $BUILD_DOCKER = true
# Push to pypi
- provider: script
script: pip install twine && python setup.py sdist && twine upload dist/*.tar.gz
skip_cleanup: true
on:
tags: true
condition: $BUILD_PYPI = true

9
CHANGELOG.md
View File

@ -1,5 +1,14 @@
# Changelog
## 3.0.0 (2019-06-09)
- [Improvement] Complete rewrite of Tutor: switch from a make-based project to a single binary which runs all commands.
- [Feature] An web user interface can be created with `tutor webui start`
- [Bugfix] Add missing elasticsearch to Kubernetes deployment (#147)
- [Improvement] Upload `tutor-openedx` to pypi
## Older changes
- 2019-01-27 [Bugfix] Fix video transcript/srt upload and download of user-uploaded files. Thanks @dannielariola!
- 2019-01-20 [Improvement] Make it easy to load custom settings for the local production install
- 2019-01-16 [Improvement] Switch license from MIT to AGPL

69
Makefile
View File

@ -1,66 +1,17 @@
.PHONY: android build
.DEFAULT_GOAL := help
PWD = $$(pwd)
USERID ?= $$(id -u)
compile-requirements: ## Compile requirements files
pip-compile -o requirements/base.txt requirements/base.in
pip-compile -o requirements/dev.txt requirements/dev.in
build: ## Build all docker images
cd build/ && make build
bundle: ## Bundle the tutor package in a single "dist/tutor" executable
pyinstaller --onefile --name=tutor --add-data=./tutor/templates:./tutor/templates ./bin/main
config.json: ## Generate config.json configuration file interactively
@$(MAKE) -s upgrade-to-tutor
@$(MAKE) -s -C build/ build-configurator 1> /dev/null
@docker run --rm -it \
--volume="$(PWD):/openedx/config/" \
-e USERID=$(USERID) -e SILENT=$(SILENT) \
regis/openedx-configurator:hawthorn \
configurator interactive
substitute: config.json
@docker run --rm -it \
--volume="$(PWD)/config.json:/openedx/config/config.json" \
--volume="$(TEMPLATES):/openedx/templates" \
--volume="$(OUTPUT):/openedx/output" \
-e USERID=$(USERID) -e SILENT=$(SILENT) $(CONFIGURE_OPTS) \
regis/openedx-configurator:hawthorn \
configurator substitute /openedx/templates/ /openedx/output/
local: ## Configure and run a ready-to-go Open edX platform
$(MAKE) -C deploy/local all
stop: ## Stop all single server services
$(MAKE) -C deploy/local stop
android: ## Configure and build a development Android app
cd android/ && make all
travis:
cd build && make build
cd deploy/local \
&& make configure SILENT=1 CONFIGURE_OPTS="-e SETTING_ACTIVATE_NOTES=1 -e SETTING_ACTIVATE_XQUEUE=1" \
&& make databases
upgrade-to-tutor: ## Upgrade from earlier versions of tutor
@(stat config/config.json > /dev/null 2>&1 && (\
echo "You are running an older version of Tutor. Now migrating to the latest version" \
&& echo "Moving config/config.json to ./config.json" && mv config/config.json config.json \
&& echo "Moving config/ to deploy/env/" && mv config/ deploy/env/ \
&& ((ls openedx/themes/* > /dev/null 2>&1 && echo "Moving openedx/themes/* to build/openedx/themes/" && mv openedx/themes/* build/openedx/themes/) || true) \
&& (mv .env deploy/local/ > /dev/null 2>&1 || true)\
&& echo "Done migrating to tutor. This command will not be run again."\
)) || true
info: ## Print some information about the current install, for debugging
uname -a
@echo "-------------------------"
git rev-parse HEAD
@echo "-------------------------"
docker version
@echo "-------------------------"
docker-compose --version
@echo "-------------------------"
echo $$EDX_PLATFORM_PATH
echo $$EDX_PLATFORM_SETTINGS
travis: bundle ## Run tests on travis-ci
./dist/tutor config noninteractive
./dist/tutor images env
./dist/tutor images build all
./dist/tutor local databases
ESCAPE = 
help: ## Print this help

18
README.rst
View File

@ -13,6 +13,10 @@ Tutor 🎓 Open edX 1-click install for everyone
:alt: GitHub closed issues
:target: https://github.com/regisb/tutor/issues?q=is%3Aclosed
.. image:: https://img.shields.io/github/license/regisb/tutor.svg
:alt: AGPL License
:target: https://www.gnu.org/licenses/agpl-3.0.en.html
**Tutor** is a one-click install of `Open edX <https://openedx.org>`_, both for production and local development, inside docker containers. Tutor is easy to run, fast, full of cool features, and it is already used by dozens of Open edX platforms in the world.
.. image:: https://asciinema.org/a/octNfEnvIA6jNohCBmODBKizE.png
@ -22,18 +26,16 @@ Tutor 🎓 Open edX 1-click install for everyone
Quickstart
----------
::
git clone https://github.com/regisb/tutor
cd tutor/deploy/local
make all
1. `Download <https://github.com/regisb/tutor/releases>`_ the latest stable release of Tutor, uncompress the file and place the ``tutor`` executable in your path.
2. Run ``tutor local quickstart``
3. You're done!
Documentation
-------------
Extensive documentation is available online: http://docs.tutor.overhang.io/
How to contribute
-----------------
Contributing
------------
We go to great lengths to make it as easy as possible for people to run Open edX inside Docker containers. If you have an improvement idea, feel free to `open an issue on Github <https://github.com/regisb/tutor/issues/new>`_ so that we can discuss it. `Pull requests <https://github.com/regisb/tutor/pulls>`_ will be happily examined, too! However, we should be careful to keep the project lean and simple: both to use and to modify. Optional features should not make the user experience more complex. Instead, documentation on how to add the feature is preferred.
We go to great lengths to make it as easy as possible for people to run Open edX inside Docker containers. If you have an improvement idea, feel free to `open an issue on Github <https://github.com/regisb/tutor/issues/new>`_ so that we can discuss it. `Pull requests <https://github.com/regisb/tutor/pulls>`_ will be happily examined, too!

2
android/.gitignore vendored
View File

@ -1,2 +0,0 @@
/config/
/data/

32
android/Makefile
View File

@ -1,32 +0,0 @@
PWD ?= $$(pwd)
.DEFAULT_GOAL := help
all: environment update android ## Configure and build a development Android app
env: ## Generate the environment
@$(MAKE) -s -C .. substitute TEMPLATES=$(PWD)/templates OUTPUT=$(PWD)/config
update: ## Download most recent Android image
docker pull regis/openedx-android:latest
android: ## Build the Android app, for development
docker run --rm -it \
--volume=$(PWD)/config/:/openedx/config/ \
--volume=$(PWD)/data/:/openedx/data \
regis/openedx-android:latest
@echo "Your development APK file is ready in $(PWD)/data/"
android-release: ## Build the final Android app (beta)
# Note that this requires that you edit ./config/android/gradle.properties
docker run --rm -it \
--volume=$(PWD)/config/:/openedx/config/ \
--volume=$(PWD)/data/:/openedx/data \
regis/openedx-android:latest \
./gradlew assembleProdRelease
@echo "Your production APK file is ready in $(PWD)/data/"
ESCAPE = 
help: ## Print this help
@grep -E '^([a-zA-Z_-]+:.*?## .*|######* .+)$$' Makefile \
| sed 's/######* \(.*\)/\n $(ESCAPE)[1;31m\1$(ESCAPE)[0m/g' \
| awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'

4
bin/main Executable file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env python3
from tutor.cli import main
main()

1
build/.gitignore vendored
View File

@ -1 +0,0 @@
openedx/requirements/private.txt

59
build/Makefile
View File

@ -1,59 +0,0 @@
#################### Docker image building
.DEFAULT_GOAL := help
build: build-openedx build-configurator build-forum build-notes build-xqueue build-android ## Build all docker images
openedx_build_args =
ifdef EDX_PLATFORM_REPOSITORY
openedx_build_args += --build-arg="EDX_PLATFORM_REPOSITORY=$(EDX_PLATFORM_REPOSITORY)"
endif
ifdef EDX_PLATFORM_VERSION
openedx_build_args += --build-arg="EDX_PLATFORM_VERSION=$(EDX_PLATFORM_VERSION)"
endif
ifdef THEMES
openedx_build_args += --build-arg="THEMES=$(THEMES)"
endif
build-openedx: ## Build the Open edX docker image
docker build -t regis/openedx:latest -t regis/openedx:hawthorn $(openedx_build_args) openedx/
build-configurator: ## Build the configurator docker image
docker build -t regis/openedx-configurator:latest -t regis/openedx-configurator:hawthorn configurator/
build-forum: ## Build the forum docker image
docker build -t regis/openedx-forum:latest -t regis/openedx-forum:hawthorn forum/
build-notes: ## Build the Notes docker image
docker build -t regis/openedx-notes:latest -t regis/openedx-notes:hawthorn notes/
build-xqueue: ## Build the Xqueue docker image
docker build -t regis/openedx-xqueue:latest -t regis/openedx-xqueue:hawthorn xqueue/
build-android: ## Build the docker image for Android
docker build -t regis/openedx-android:latest android/
################### Pushing images to docker hub
push: push-openedx push-configurator push-forum push-notes push-xqueue push-android ## Push all images to dockerhub
push-openedx: ## Push Open edX images to dockerhub
docker push regis/openedx:hawthorn
docker push regis/openedx:latest
push-configurator: ## Push configurator image to dockerhub
docker push regis/openedx-configurator:hawthorn
docker push regis/openedx-configurator:latest
push-forum: ## Push forum image to dockerhub
docker push regis/openedx-forum:hawthorn
docker push regis/openedx-forum:latest
push-notes: ## Push notes image to dockerhub
docker push regis/openedx-notes:hawthorn
docker push regis/openedx-notes:latest
push-xqueue: ## Push Xqueue image to dockerhub
docker push regis/openedx-xqueue:hawthorn
docker push regis/openedx-xqueue:latest
push-android: ## Push the Android image to dockerhub
docker push regis/openedx-android:latest
dockerhub: build push ## Build and push all images to dockerhub
##################### Information
ESCAPE = 
help: ## Print this help
@grep -E '^([a-zA-Z_-]+:.*?## .*|######* .+)$$' Makefile \
| sed 's/######* \(.*\)/\n $(ESCAPE)[1;31m\1$(ESCAPE)[0m/g' \
| awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'

12
build/configurator/Dockerfile
View File

@ -1,12 +0,0 @@
FROM python:3.6-alpine
MAINTAINER Régis Behmo <regis@behmo.com>
RUN apk add --no-cache curl
RUN pip install jinja2
RUN mkdir /openedx /openedx/config /openedx/templates
COPY ./bin/configurator /usr/local/bin
WORKDIR /openedx/
CMD configurator -c /openedx/config/config.json interactive && \
configurator -c /openedx/config/config.json substitute /openedx/templates/ /openedx/output/

242
build/configurator/bin/configurator
View File

@ -1,242 +0,0 @@
#! /usr/bin/env python3
# coding: utf8
import argparse
import codecs
import json
import os
import random
import string
import sys
from collections import OrderedDict
import jinja2
class Configurator:
def __init__(self, **default_overrides):
"""
Default values are read, in decreasing order of priority, from:
- SETTING_<name> environment variable
- Existing config file (in `default_overrides`)
- Value passed to add()
"""
self.__values = OrderedDict()
self.__default_values = default_overrides
if os.environ.get('SILENT'):
self.__input = None
else:
self.__input = input
print("====================================\n"
" Interactive configuration \n"
"====================================")
def as_dict(self):
return self.__values
def get_default_value(self, name, default):
setting_name = 'SETTING_' + name.upper()
if os.environ.get(setting_name):
return os.environ[setting_name]
if name in self.__default_values:
return self.__default_values[name]
return default
def add(self, name, default, question=""):
default = self.get_default_value(name, default)
if not self.__input or not question:
return self.set(name, default)
return self.set(name, self.ask(question, default))
def add_bool(self, name, default, question=""):
default = self.get_default_value(name, default)
if default in [1, '1']:
default = True
if default in [0, '0', '']:
default = False
if not self.__input or not question:
return self.set(name, default)
question += " (y/n)"
while True:
answer = self.ask(question, 'y' if default else 'n')
if answer is None or answer == '':
return self.set(name, default)
if answer.lower() in ['y', 'yes']:
return self.set(name, True)
if answer.lower() in ['n', 'no']:
return self.set(name, False)
def add_choice(self, name, default, choices, question=""):
default = self.get_default_value(name, default)
if not self.__input or not question:
return self.set(name, default)
while True:
answer = self.ask(question, default)
if answer in choices:
return self.set(name, answer)
print("Invalid value. Choices are: {}".format(", ".join(choices)))
def ask(self, question, default):
return self.__input('\1\2\x1b[35m> {} [{}] \x1b[39;49;00m'.format(question, default)) or default
def get(self, name):
return self.__values.get(name)
def set(self, name, value):
self.__values[name] = value
return self
def main():
parser = argparse.ArgumentParser("Config file generator for Open edX")
parser.add_argument('-c', '--config', default=os.path.join("/", "openedx", "config", "config.json"),
help="Load default values from this file. Config values will be saved there.")
subparsers = parser.add_subparsers()
parser_interactive = subparsers.add_parser('interactive')
parser_interactive.set_defaults(func=interactive)
parser_substitute = subparsers.add_parser('substitute')
parser_substitute.add_argument('src', help="Template source directory")
parser_substitute.add_argument('dst', help="Destination configuration directory")
parser_substitute.set_defaults(func=substitute)
args = parser.parse_args()
args.func(args)
def load_config(path):
if os.path.exists(path):
with open(path) as f:
return json.load(f)
return {}
def interactive(args):
interactive_configuration(args.config)
def interactive_configuration(config_path):
configurator = Configurator(**load_config(config_path))
configurator.add(
'LMS_HOST', 'www.myopenedx.com', "Your website domain name for students (LMS)"
).add(
'CMS_HOST', 'studio.' + configurator.get('LMS_HOST'), "Your website domain name for teachers (CMS)"
).add(
'PLATFORM_NAME', "My Open edX", "Your platform name/title"
).add(
'CONTACT_EMAIL', 'contact@' + configurator.get('LMS_HOST'), "Your public contact email address",
).add_choice(
'LANGUAGE_CODE', 'en',
['en', 'am', 'ar', 'az', 'bg-bg', 'bn-bd', 'bn-in', 'bs', 'ca',
'ca@valencia', 'cs', 'cy', 'da', 'de-de', 'el', 'en-uk', 'en@lolcat',
'en@pirate', 'es-419', 'es-ar', 'es-ec', 'es-es', 'es-mx', 'es-pe',
'et-ee', 'eu-es', 'fa', 'fa-ir', 'fi-fi', 'fil', 'fr', 'gl', 'gu',
'he', 'hi', 'hr', 'hu', 'hy-am', 'id', 'it-it', 'ja-jp', 'kk-kz',
'km-kh', 'kn', 'ko-kr', 'lt-lt', 'ml', 'mn', 'mr', 'ms', 'nb', 'ne',
'nl-nl', 'or', 'pl', 'pt-br', 'pt-pt', 'ro', 'ru', 'si', 'sk', 'sl',
'sq', 'sr', 'sv', 'sw', 'ta', 'te', 'th', 'tr-tr', 'uk', 'ur', 'vi',
'uz', 'zh-cn', 'zh-hk', 'zh-tw'],
"The default language code for the platform"
).add(
'SECRET_KEY', random_string(24)
).add(
'MYSQL_DATABASE', 'openedx'
).add(
'MYSQL_USERNAME', 'openedx'
).add(
'MYSQL_PASSWORD', random_string(8)
).add(
'MONGODB_DATABASE', 'openedx'
).add(
'NOTES_MYSQL_DATABASE', 'notes',
).add(
'NOTES_MYSQL_USERNAME', 'notes',
).add(
'NOTES_MYSQL_PASSWORD', random_string(8)
).add(
'NOTES_SECRET_KEY', random_string(24)
).add(
'NOTES_OAUTH2_SECRET', random_string(24)
).add(
'XQUEUE_AUTH_USERNAME', 'lms'
).add(
'XQUEUE_AUTH_PASSWORD', random_string(8)
).add(
'XQUEUE_MYSQL_DATABASE', 'xqueue',
).add(
'XQUEUE_MYSQL_USERNAME', 'xqueue',
).add(
'XQUEUE_MYSQL_PASSWORD', random_string(8)
).add(
'XQUEUE_SECRET_KEY', random_string(24)
).add_bool(
'ACTIVATE_HTTPS', False, "Activate SSL/TLS certificates for HTTPS access? Important note: this will NOT work in a development environment.",
).add_bool(
'ACTIVATE_NOTES', False, "Activate Student Notes service (https://open.edx.org/features/student-notes)?",
).add_bool(
'ACTIVATE_XQUEUE', False, "Activate Xqueue for external grader services? (https://github.com/edx/xqueue)",
).add(
'ID', random_string(8)
)
# Save values
with open(config_path, 'w') as f:
json.dump(configurator.as_dict(), f, sort_keys=True, indent=4)
set_owner(config_path)
def substitute(args):
config = load_config(args.config)
for root, _, filenames in os.walk(args.src):
for filename in filenames:
if filename.startswith('.'):
# Skip hidden files, such as files generated by the IDE
continue
src_file = os.path.join(root, filename)
dst_file = os.path.join(args.dst, os.path.relpath(src_file, args.src))
substitute_file(config, src_file, dst_file)
def substitute_file(config, src, dst):
with codecs.open(src, encoding='utf-8') as fi:
template = jinja2.Template(fi.read(), undefined=jinja2.StrictUndefined)
try:
substituted = template.render(**config)
except jinja2.exceptions.UndefinedError as e:
sys.stderr.write("ERROR Missing config value '{}' for template {}\n".format(e.args[0], src))
sys.exit(1)
dst_dir = os.path.dirname(dst)
ensure_path_exists(dst_dir)
with codecs.open(dst, encoding='utf-8', mode='w') as fo:
fo.write(substituted)
set_owner(dst)
# Set same permissions as original file
os.chmod(dst, os.stat(src).st_mode)
def ensure_path_exists(directory):
"""
Create the required subfolders one by one, if necessary, and assign them
the right uid/gid.
"""
to_create = []
while not os.path.exists(directory):
to_create.append(directory)
directory = os.path.dirname(directory)
for d in to_create[::-1]:
os.mkdir(d)
set_owner(d)
def set_owner(path):
"""
If the USER_ID environment variable is defined, use it to set the
owner/group of the given file path.
"""
user_id = int(os.environ.get('USERID', 0))
if user_id:
os.chown(path, user_id, user_id)
def random_string(length):
return "".join([random.choice(string.ascii_letters + string.digits) for _ in range(length)])
if __name__ == '__main__':
main()

1
build/openedx/requirements/.gitignore vendored
View File

@ -1 +0,0 @@
private.txt

3
build/openedx/requirements/README
View File

@ -1,3 +0,0 @@
Add your additional requirements, such as xblocks, to private.txt. This file
is not under version control, which means that your changes will not be
committed to the upstream repository.

1
build/openedx/themes/.gitignore vendored
View File

@ -1 +0,0 @@
*

13
data/.gitignore vendored
View File

@ -1,13 +0,0 @@
android/
cms/
cms_worker/
letsencrypt/
lms/
lms_worker/
elasticsearch/
mysql/
mongodb/
openedx/
portainer/
rabbitmq/
xqueue/

1
deploy/.gitignore vendored
View File

@ -1 +0,0 @@
/env/

5
deploy/Makefile
View File

@ -1,5 +0,0 @@
.PHONY: env
PWD = $$(pwd)
env: ## Generate the environment from templates and configuration values
@$(MAKE) -s -C .. substitute TEMPLATES=$(PWD)/templates OUTPUT=$(PWD)/env

2
deploy/k8s/.gitignore vendored
View File

@ -1,2 +0,0 @@
/env/
/data

128
deploy/k8s/Makefile
View File

@ -1,128 +0,0 @@
.PHONY: env volumes services deployments
.DEFAULT_GOAL := help
podname = kubectl get pods -n openedx --selector=app=$(1) -o name | head -1 | cut -d '/' -f 2
podexec = kubectl exec -n openedx -it $$($(call podname,$(1))) -- $(2)
all: configure deploy databases ## Provision a full platform from scratch
configure: ## Interactive configuration
@$(MAKE) -s -C ../.. --always-make config.json
@$(MAKE) -s env
env: ## Generate the environment from templates and configuration values
@$(MAKE) -s -C .. env
@$(MAKE) -s -C ../.. substitute TEMPLATES=$(PWD)/templates OUTPUT=$(PWD)/env
namespace: ## Create the platform namespace
kubectl create -f namespace.yml
configmaps: ## Create configmap objects
kubectl create configmap nginx-config --from-file=../env/nginx --namespace=openedx
kubectl create configmap mysql-config --from-env-file=../env/mysql/auth.env --namespace=openedx
kubectl create configmap openedx-settings-lms --from-file=../env/openedx/settings/lms --namespace=openedx
kubectl create configmap openedx-settings-cms --from-file=../env/openedx/settings/cms --namespace=openedx
kubectl create configmap openedx-config --from-file=../env/openedx/config --namespace=openedx
kubectl create configmap openedx-scripts --from-file=../env/openedx/scripts --namespace=openedx
volumes: ## Create volumes
kubectl create -f volumes/ --recursive=true --namespace=openedx
services: ## Create services
kubectl create -f services/ --recursive=true --namespace=openedx
deployments: ## Create deployments
kubectl create -f deployments/ --recursive=true --namespace=openedx
ingress: ## Create ingress
kubectl create -f env/ingress.yml --namespace=openedx
deploy: namespace volumes configmaps services deployments ingress ## Deploy a platform from scratch
upgrade: down ## Upgrade an already running platform
@$(MAKE) -s namespace || true
@$(MAKE) -s configmaps || true
@$(MAKE) -s volumes || true
@$(MAKE) -s services || true
@$(MAKE) -s deployments || true
@$(MAKE) -s ingress || true
down: ## Delete all non-persistent objects
kubectl delete -f services/ --recursive=true --namespace=openedx || true
kubectl delete -f deployments/ --recursive=true --namespace=openedx || true
kubectl delete configmap --all --namespace openedx || true
delete: ## Delete EVERYTHING! (with no confirmation at all)
# TODO ask question to make sure user reaaaaaaally want to do this
kubectl delete namespace openedx
##################### Databases
databases: provision-databases migrate #provision-oauth2 ## Bootstrap databases
provision-databases: ## Create necessary databases and users
$(call podexec,lms,bash /openedx/scripts/provision.sh)
provision-oauth2: ## Create users for SSO between services
$(call podexec,lms,bash /openedx/scripts/oauth2.sh)
migrate: migrate-openedx migrate-forum ## Perform all database migrations
migrate-openedx: migrate-lms migrate-cms reindex-courses ## Perform database migrations on LMS/CMS
migrate-lms:
$(call podexec,lms,bash -c 'dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py lms migrate')
migrate-cms:
$(call podexec,cms,bash -c 'dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py cms migrate')
migrate-forum: ## Perform database migrations on discussion forums
$(call podexec,forum,bash -c "bundle exec rake search:initialize && \
bundle exec rake search:rebuild_index")
migrate-notes: ## Perform database migrations for the Notes service
$(call podexec,notes,./manage.py migrate)
migrate-xqueue: ## Perform database migrations for the XQueue service
$(call podexec,xqueue,./manage.py migrate)
reindex-courses: ## Refresh course index so they can be found in the LMS search engine
$(call podexec,cms,./manage.py cms reindex_course --all --setup)
##################### Various Open edX commands
lms-shell: ## Launch a shell inside an lms pod
$(call podexec,lms,bash)
lms-exec: ## Execute a command inside an lms pod: make lms-exec COMMAND="..."
$(call podexec,lms,$(COMMAND))
pod-exec: ## Execute a command inside an arbitrary pod: make pod-exec APP=appname COMMAND="..."
$(call podexec,$(APP),$(COMMAND))
# TODO replace these tasks with Job objects
# Note that here, settings must be defined manually because "exec" does not use
# the docker entrypoint, and thus does not define the DJANGO_SETTINGS_MODULE
# environment variable.
demo-course: ## Import the demo course from edX
$(call podexec,cms,/bin/bash -c "\
git clone https://github.com/edx/edx-demo-course --branch open-release/hawthorn.2 --depth 1 ../edx-demo-course \
&& python ./manage.py cms --settings=tutor.production import ../data ../edx-demo-course")
staff-user: ## Create a user with admin rights: make staff-user USERNAME=... EMAIL=...
$(call podexec,lms,/bin/bash -c "\
./manage.py lms manage_user --superuser --staff ${USERNAME} ${EMAIL} \
&& ./manage.py lms --settings=tutor.production changepassword ${USERNAME}")
##################### Various minikube command
minikube-start: ## Start minikube
minikube start
minikube-stop: ## Stop minikube
minikube stop
minikube-dashboard: ## Open the minikube dashboard
minikube dashboard
minikube-ingress: ## Enable the ingress addon
minikube addons enable ingress
##################### Various k8s commands
k8s-proxy: ## Create a proxy to the Kubernetes API server
kubectl proxy
k8s-dashboard: ## Create the dashboard
kubectl create -f https://raw.githubusercontent.com/kubernetes/dashboard/master/aio/deploy/recommended/kubernetes-dashboard.yaml
k8s-admin: ## Create an admin user for accessing the Kubernetes dashboard
kubectl -f admin.yml
k8s-admin-token: ## Print the admin token required to log in the dashboard
kubectl -n kube-system describe secret $$(kubectl -n kube-system get secret | grep admin-user | awk '{print $$1}') | grep token: | awk '{print $$2}'
##################### Information
ESCAPE = 
help: ## Print this help
@grep -E '^([a-zA-Z0-9_-]+:.*?## .*|######* .+)$$' Makefile \
| sed 's/######* \(.*\)/\n $(ESCAPE)[1;31m\1$(ESCAPE)[0m/g' \
| awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'

50
deploy/k8s/deployments/cms.yml
View File

@ -1,50 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: cms
spec:
replicas: 1
selector:
matchLabels:
app: cms
template:
metadata:
labels:
app: cms
spec:
containers:
- name: cms
image: regis/openedx:hawthorn
env:
- name: SERVICE_VARIANT
value: cms
ports:
- containerPort: 8000
volumeMounts:
- mountPath: /openedx/edx-platform/lms/envs/tutor/
name: settings-lms
- mountPath: /openedx/edx-platform/cms/envs/tutor/
name: settings-cms
- mountPath: /openedx/config
name: config
- mountPath: /openedx/scripts
name: scripts
- mountPath: /openedx/data
name: data
#imagePullPolicy: Always
volumes:
- name: settings-lms
configMap:
name: openedx-settings-lms
- name: settings-cms
configMap:
name: openedx-settings-cms
- name: config
configMap:
name: openedx-config
- name: scripts
configMap:
name: openedx-scripts
- name: data
persistentVolumeClaim:
claimName: cms-data

20
deploy/k8s/deployments/forum.yml
View File

@ -1,20 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: forum
spec:
replicas: 1
selector:
matchLabels:
app: forum
template:
metadata:
labels:
app: forum
spec:
containers:
- name: forum
image: regis/openedx-forum:hawthorn
ports:
- containerPort: 4567
imagePullPolicy: Always

47
deploy/k8s/deployments/lms.yml
View File

@ -1,47 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: lms
spec:
replicas: 1
selector:
matchLabels:
app: lms
template:
metadata:
labels:
app: lms
spec:
containers:
- name: lms
image: regis/openedx:hawthorn
ports:
- containerPort: 8000
volumeMounts:
- mountPath: /openedx/edx-platform/lms/envs/tutor/
name: settings-lms
- mountPath: /openedx/edx-platform/cms/envs/tutor/
name: settings-cms
- mountPath: /openedx/config
name: config
- mountPath: /openedx/scripts
name: scripts
- mountPath: /openedx/data
name: data
imagePullPolicy: Always
volumes:
- name: settings-lms
configMap:
name: openedx-settings-lms
- name: settings-cms
configMap:
name: openedx-settings-cms
- name: config
configMap:
name: openedx-config
- name: scripts
configMap:
name: openedx-scripts
- name: data
persistentVolumeClaim:
claimName: lms-data

19
deploy/k8s/deployments/memcached.yml
View File

@ -1,19 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: memcached
spec:
replicas: 1
selector:
matchLabels:
app: memcached
template:
metadata:
labels:
app: memcached
spec:
containers:
- name: memcached
image: memcached:1.4.38
ports:
- containerPort: 11211

26
deploy/k8s/deployments/mongodb.yml
View File

@ -1,26 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: mongodb
spec:
replicas: 1
selector:
matchLabels:
app: mongodb
template:
metadata:
labels:
app: mongodb
spec:
containers:
- name: mongodb
image: mongo:3.2.16
command: ["mongod", "--smallfiles", "--nojournal", "--storageEngine", "wiredTiger"]
ports:
- containerPort: 27017
volumeMounts:
- mountPath: /data/db
name: data
volumes:
- name: data
emptyDir: {}

32
deploy/k8s/deployments/mysql.yml
View File

@ -1,32 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: mysql
spec:
replicas: 1
selector:
matchLabels:
app: mysql
template:
metadata:
labels:
app: mysql
spec:
containers:
- name: mysql
image: mysql:5.6.36
env:
- name: MYSQL_ROOT_PASSWORD
valueFrom:
configMapKeyRef:
name: mysql-config
key: MYSQL_ROOT_PASSWORD
ports:
- containerPort: 3306
volumeMounts:
- mountPath: /var/lib/mysql
name: data
volumes:
- name: data
persistentVolumeClaim:
claimName: mysql

54
deploy/k8s/deployments/nginx.yml
View File

@ -1,54 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
initContainers:
- name: clean-openedx-staticfiles
image: regis/openedx:hawthorn
command: ['rm', '-rf', '/var/www/openedx/staticfiles']
volumeMounts:
- mountPath: /var/www/openedx/
name: openedx-staticfiles
imagePullPolicy: Always
- name: init-openedx-staticfiles
image: regis/openedx:hawthorn
command: ['cp', '-r', '/openedx/staticfiles', '/var/www/openedx/']
volumeMounts:
- mountPath: /var/www/openedx/
name: openedx-staticfiles
imagePullPolicy: Always
containers:
- name: nginx
image: nginx:1.13
volumeMounts:
- mountPath: /etc/nginx/conf.d/
name: config
- mountPath: /var/www/openedx/
name: openedx-staticfiles
- mountPath: /openedx/data/lms
name: data
ports:
- containerPort: 80
name: http-port
- containerPort: 443
name: https-port
volumes:
- name: config
configMap:
name: nginx-config
- name: openedx-staticfiles
persistentVolumeClaim:
claimName: openedx-staticfiles
- name: data
persistentVolumeClaim:
claimName: lms-data

26
deploy/k8s/deployments/rabbitmq.yml
View File

@ -1,26 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: rabbitmq
spec:
replicas: 1
selector:
matchLabels:
app: rabbitmq
template:
metadata:
labels:
app: rabbitmq
spec:
containers:
- name: rabbitmq
image: rabbitmq:3.6.10
ports:
- containerPort: 5672
volumeMounts:
- mountPath: /var/lib/rabbitmq
name: data
volumes:
- name: data
persistentVolumeClaim:
claimName: rabbitmq

11
deploy/k8s/services/cms.yml
View File

@ -1,11 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: cms
spec:
type: NodePort
ports:
- port: 8000
protocol: TCP
selector:
app: cms

11
deploy/k8s/services/forum.yml
View File

@ -1,11 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: forum
spec:
type: NodePort
ports:
- port: 4567
protocol: TCP
selector:
app: forum

11
deploy/k8s/services/lms.yml
View File

@ -1,11 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: lms
spec:
type: NodePort
ports:
- port: 8000
protocol: TCP
selector:
app: lms

11
deploy/k8s/services/memcached.yml
View File

@ -1,11 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: memcached
spec:
type: NodePort
ports:
- port: 11211
protocol: TCP
selector:
app: memcached

11
deploy/k8s/services/mongodb.yml
View File

@ -1,11 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: mongodb
spec:
type: NodePort
ports:
- port: 27017
protocol: TCP
selector:
app: mongodb

11
deploy/k8s/services/mysql.yml
View File

@ -1,11 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: mysql
spec:
type: NodePort
ports:
- port: 3306
protocol: TCP
selector:
app: mysql

17
deploy/k8s/services/nginx.yml
View File

@ -1,17 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: nginx
spec:
type: NodePort
ports:
- port: 80
protocol: TCP
name: http
targetPort: http-port
- port: 443
protocol: TCP
name: https
targetPort: https-port
selector:
app: nginx

11
deploy/k8s/services/rabbitmq.yml
View File

@ -1,11 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: rabbitmq
spec:
type: NodePort
ports:
- port: 5672
protocol: TCP
selector:
app: rabbitmq

10
deploy/k8s/volumes/cms-data.yml
View File

@ -1,10 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: cms-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi

10
deploy/k8s/volumes/lms-data-pvc.yml
View File

@ -1,10 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: lms-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi

10
deploy/k8s/volumes/mysql-pvc.yml
View File

@ -1,10 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: mysql
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi

10
deploy/k8s/volumes/openedx-staticfiles.yml
View File

@ -1,10 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: openedx-staticfiles
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi

10
deploy/k8s/volumes/rabbitmq.yml
View File

@ -1,10 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rabbitmq
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi

3
deploy/local/.gitignore vendored
View File

@ -1,3 +0,0 @@
/Makefile.env
/docker-compose.yml
/.env

194
deploy/local/Makefile
View File

@ -1,194 +0,0 @@
.PHONY: env
.DEFAULT_GOAL := help
PWD = $$(pwd)
USERID ?= $$(id -u)
EDX_PLATFORM_SETTINGS ?= tutor.production
-include Makefile.env
DOCKER_COMPOSE_RUN = docker-compose run --rm
DOCKER_COMPOSE_RUN_OPENEDX = $(DOCKER_COMPOSE_RUN) -e SETTINGS=$(EDX_PLATFORM_SETTINGS) \
--volume="$(PWD)/../../build/openedx/themes:/openedx/themes"
ifneq ($(EDX_PLATFORM_PATH),)
DOCKER_COMPOSE_RUN_OPENEDX += -e USERID=$(USERID) --volume="$(EDX_PLATFORM_PATH):/openedx/edx-platform"
endif
DOCKER_COMPOSE_RUN_LMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8000:8000 lms
DOCKER_COMPOSE_RUN_CMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8001:8001 cms
##################### Running Open edX
all: env ## Configure and run a full-featured platform
@$(MAKE) stats
@$(MAKE) https-certificate
@$(MAKE) update
@$(MAKE) databases
@$(MAKE) daemonize
@echo "All set \o/ You can access the LMS at http://localhost and the CMS at http://studio.localhost"
run: ## Run the complete platform
docker-compose up
up: run
daemonize: ## Run the complete platform, with daemonization
docker-compose up -d
@echo "Daemon is up and running"
daemon: daemonize
stop: ## Stop all services
docker-compose rm --stop --force
##################### Docker image management
update: ## Download most recent images
docker-compose pull
##################### Configuration
configure: ## Interactive configuration
@$(MAKE) -s -C ../.. --always-make config.json
@$(MAKE) -s env
env: ## Generate the environment from templates and configuration values
@$(MAKE) -s -C .. env
@$(MAKE) -s -C ../.. substitute TEMPLATES=$(PWD)/templates OUTPUT=$(PWD)
##################### Database
databases: init-mysql provision-databases migrate provision-oauth2 ## Bootstrap databases
init-mysql: ## Make sure that mysql is properly initialized
@(stat ../../data/mysql/mysql 1> /dev/null 2>&1 && echo "MySQL has already been initialized") || (\
echo "Waiting for mysql initialization..." \
&& docker-compose up -d mysql \
&& until docker-compose logs mysql | grep "MySQL init process done. Ready for start up."; do \
sleep 1;\
done \
&& docker-compose stop mysql\
)
provision-databases: ## Create necessary databases and users
$(DOCKER_COMPOSE_RUN) -v $(PWD)/../env/openedx/scripts:/openedx/scripts lms /openedx/scripts/provision.sh
provision-oauth2: ## Create users for SSO between services
$(DOCKER_COMPOSE_RUN) -v $(PWD)/../env/openedx/scripts:/openedx/scripts lms /openedx/scripts/oauth2.sh
migrate: migrate-openedx migrate-forum ## Perform all database migrations
@if [ "$(ACTIVATE_XQUEUE)" = "1" ] ; then \
$(MAKE) migrate-xqueue ;\
fi
@if [ "$(ACTIVATE_NOTES)" = "1" ] ; then \
$(MAKE) migrate-notes ; \
fi
migrate-openedx: ## Perform database migrations on LMS/CMS
$(DOCKER_COMPOSE_RUN) lms bash -c "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py lms migrate"
$(DOCKER_COMPOSE_RUN) cms bash -c "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py cms migrate"
$(MAKE) reindex-courses
migrate-forum: ## Perform database migrations on discussion forums
$(DOCKER_COMPOSE_RUN) forum bash -c "bundle exec rake search:initialize && \
bundle exec rake search:rebuild_index"
migrate-notes: ## Perform database migrations for the Notes service
$(DOCKER_COMPOSE_RUN) notes ./manage.py migrate
migrate-xqueue: ## Perform database migrations for the XQueue service
$(DOCKER_COMPOSE_RUN) xqueue ./manage.py migrate
reindex-courses: ## Refresh course index so they can be found in the LMS search engine
$(DOCKER_COMPOSE_RUN) cms ./manage.py cms reindex_course --all --setup
##################### Static assets
# To collect assets we don't rely on the "paver update_assets" command because
# webpack collection incorrectly sets the NODE_ENV variable when using custom
# settings. Thus, each step must be performed separately. This should be fixed
# in the next edx-platform release thanks to https://github.com/edx/edx-platform/pull/18430/
assets: ## Copy production-ready static assets. Note that you should not have to run this command yourself: all assets should be sent to nginx on boot.
docker-compose run --rm openedx-assets
assets-development: ## Generate static assets for local development
$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev"
assets-development-lms:
$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system lms"
assets-development-cms:
$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system cms"
watch-themes: ## Watch for changes in your themes and build development assets
$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms openedx-assets watch-themes --env dev
#################### Logging
logs: ## Print all logs from a service since it started. E.g: "make logs service=lms", "make logs service=nginx"
docker-compose logs $(service)
tail: ## Similar to "tail" on the logs of a service. E.g: "make tail service=lms", "make tail service=nginx"
docker-compose logs --tail=10 $(service)
tail-follow: ## Similar to "tail -f" on the logs of a service. E.g: "make tail-follow service=lms", "make tail-follow service=nginx"
docker-compose logs --tail=10 -f $(service)
##################### Development
lms: ## Open a bash shell in the LMS
$(DOCKER_COMPOSE_RUN_LMS) bash
cms: ## Open a bash shell in the CMS
$(DOCKER_COMPOSE_RUN_CMS) bash
lms-python: ## Open a python shell in the LMS
$(DOCKER_COMPOSE_RUN_OPENEDX) lms ./manage.py lms shell
lms-shell: lms-python
lms-runserver: ## Run a local webserver, useful for debugging
$(DOCKER_COMPOSE_RUN_LMS) ./manage.py lms runserver 0.0.0.0:8000
cms-python: ## Open a python shell in the CMS
$(DOCKER_COMPOSE_RUN_OPENEDX) cms ./manage.py cms shell
cms-shell: cms-python
cms-runserver: ## Run a local webserver, useful for debugging
$(DOCKER_COMPOSE_RUN_CMS) ./manage.py cms runserver 0.0.0.0:8001
restart-openedx: ## Restart lms, cms, and workers
docker-compose restart openedx-assets lms lms_worker cms cms_worker
##################### SSL/TLS (HTTPS certificates)
https_command = docker run --rm -it \
--volume="$(PWD)/../env/letsencrypt/:/openedx/letsencrypt/env/" \
--volume="$(PWD)/../../data/letsencrypt/:/etc/letsencrypt/" \
-p "80:80"
certbot_image = certbot/certbot:latest
https-certificate: ## Generate https certificates
@if [ "$(ACTIVATE_HTTPS)" = "1" ] ; then \
$(https_command) --entrypoint "/openedx/letsencrypt/env/certonly.sh" $(certbot_image); \
fi
https-certificate-renew: ## Renew https certificates
$(https_command) $(certbot_image) renew
##################### Additional commands
stats: ## Collect anonymous information about the platform
@if [ "$(DISABLE_STATS)" != "1" ] ; then \
docker run --rm -it --volume="$(PWD)/../env/openedx/scripts:/openedx/scripts" \
regis/openedx-configurator:hawthorn /openedx/scripts/stats 2> /dev/null || true ; \
fi
demo-course: ## Import the demo course from edX
$(DOCKER_COMPOSE_RUN_OPENEDX) cms /bin/bash -c " \
git clone https://github.com/edx/edx-demo-course --branch open-release/hawthorn.2 --depth 1 ../edx-demo-course \
&& python ./manage.py cms import ../data ../edx-demo-course"
staff-user: ## Create a user with admin rights: make staff-user USERNAME=... EMAIL=...
$(DOCKER_COMPOSE_RUN_OPENEDX) lms /bin/bash -c "./manage.py lms manage_user --superuser --staff ${USERNAME} ${EMAIL} && ./manage.py lms changepassword ${USERNAME}"
portainer: ## Run Portainer (https://portainer.io), a UI for container supervision
docker run --rm -it \
--volume=/var/run/docker.sock:/var/run/docker.sock \
--volume=$(PWD)/../../data/portainer:/data \
-p 9000:9000 \
portainer/portainer:latest
@echo "View the Portainer UI at http://localhost:9000"
##################### Information
info:
$(MAKE) -s -C ../.. info
# Obtained by running "echo '\033' in a shell
ESCAPE = 
help: ## Print this help
@grep -E '^([a-zA-Z_-]+:.*?## .*|######* .+)$$' Makefile \
| sed 's/######* \(.*\)/\n $(ESCAPE)[1;31m\1$(ESCAPE)[0m/g' \
| awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'

3
deploy/local/templates/Makefile.env
View File

@ -1,3 +0,0 @@
ACTIVATE_HTTPS ?= {{ 1 if ACTIVATE_HTTPS else 0 }}
ACTIVATE_XQUEUE ?= {{ 1 if ACTIVATE_XQUEUE else 0 }}
ACTIVATE_NOTES ?= {{ 1 if ACTIVATE_NOTES else 0 }}

6
deploy/templates/letsencrypt/certonly.sh
View File

@ -1,6 +0,0 @@
#!/bin/sh
certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ LMS_HOST }} -d {{ CMS_HOST }} -d preview.{{ LMS_HOST }}
{% if ACTIVATE_NOTES %}
certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d notes.{{ LMS_HOST }}
{% endif %}

11
deploy/templates/openedx/scripts/oauth2.sh
View File

@ -1,11 +0,0 @@
#!/bin/bash -e
{% if ACTIVATE_NOTES %}
./manage.py lms manage_user notes notes@{{ LMS_HOST }} --staff --superuser
./manage.py lms create_oauth2_client \
"http://notes.openedx:8000" \
"http://notes.openedx:8000/complete/edx-oidc/" \
confidential \
--client_name edx-notes --client_id notes --client_secret {{ NOTES_OAUTH2_SECRET }} \
--trusted --logout_uri "http://notes.openedx:8000/logout/" --username notes
{% endif %}

20
deploy/templates/openedx/scripts/provision.sh
View File

@ -1,20 +0,0 @@
#!/bin/bash -e
dockerize -wait tcp://mysql:3306 -timeout 20s
while ! mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e="\r"; do
echo "Waiting for mysql database to be ready..."
sleep 1
done
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS {{ MYSQL_DATABASE }};'
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'GRANT ALL ON {{ MYSQL_DATABASE }}.* TO "{{ MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ MYSQL_PASSWORD }}";'
{% if ACTIVATE_NOTES %}
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};'
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";'
{% endif %}
{% if ACTIVATE_XQUEUE %}
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};'
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";'
{% endif %}

5
deploy/templates/openedx/scripts/stats
View File

@ -1,5 +0,0 @@
#!/bin/sh
# This is a small script that helps me collect stats about who is using this
# project and understand how they are using it. Please don't remove me! As you
# can see, no personal data is leaked.
curl --silent -X POST --max-time 5 "https://openedx.overhang.io/stats?id={{ ID }}&lms={{ LMS_HOST|urlencode }}" 2> /dev/null || true

2
docs/Makefile
View File

@ -13,7 +13,7 @@ help:
browse:
sensible-browser _build/html/index.html
watch:
watch: html
while true; do inotifywait -e modify *.rst; $(MAKE) html; done
.PHONY: help Makefile

72
docs/customise.rst
View File

@ -1,26 +1,29 @@
.. _customise:
Customising the ``openedx`` docker image
========================================
Open edX platform customisation
===============================
The LMS and the CMS all run from the ``openedx`` docker image. The base image is downloaded from `Docker Hub <https://hub.docker.com/r/regis/openedx/>`_ when we run ``make update`` (or ``make all``). But you can also customise and build the image yourself. All image-building commands must be run inside the ``build`` folder::
There are different ways you can customise your Open edX platform. For instance, optional features can be activated during configuration. But if you want to add unique features to your Open edX platform, you are going to have to modify and re-build the ``openedx`` docker image. This is the image that contains the ``edx-platform`` repository: it is in charge of running the web application for the Open edX "core". Both the LMS and the CMS run from the ``openedx`` docker image.
cd build
On a vanilla platform deployed by Tutor, the image that is run is downloaded from the `regis/openedx repository on Docker Hub <https://hub.docker.com/r/regis/openedx/>`_. This is also the image that is downloaded whenever we run ``tutor local pullimages``. But you can decide to build the image locally instead of downloading it. To do so, generate the image-building environment::
Build the base image with::
tutor images env
make build-openedx
Then, build and tag the ``openedx`` image::
The following sections describe how to modify various aspects of the docker image. After you have built your own image, you can run it as usual::
tutor images build openedx
make run
The following sections describe how to modify various aspects of the docker image. Every time, you will have to re-build your own image with this command. Re-building should take ~20 minutes on a server with good bandwidth. After that, your custom image will be used for all commands. For instance:
Custom themes
-------------
1. you can start a local platform with ``tutor local start``
2. restart the services that use the ``openedx`` image with ``tutor local restart openedx``
Comprehensive theming is enabled by default, but only the default theme is compiled. To compile your own theme, add it to the ``build/openedx/themes/`` folder::
Adding custom themes
--------------------
git clone https://github.com/me/myopenedxtheme.git build/openedx/themes/
Comprehensive theming is enabled by default, but only the default theme is compiled. To compile your own theme, add it to the ``env/build/openedx/themes/`` folder::
git clone https://github.com/me/myopenedxtheme.git env/build/openedx/themes/
The ``themes`` folder should have the following structure::
@ -35,20 +38,20 @@ The ``themes`` folder should have the following structure::
Then you must rebuild the openedx Docker image::
make build-openedx
tutor images build openedx
Finally, follow the `Open edX documentation to enable your themes <https://edx.readthedocs.io/projects/edx-installing-configuring-and-running/en/latest/configuration/changing_appearance/theming/enable_themes.html#apply-a-theme-to-a-site>`_.
Finally, follow the `Open edX documentation to apply your themes <https://edx.readthedocs.io/projects/edx-installing-configuring-and-running/en/latest/configuration/changing_appearance/theming/enable_themes.html#apply-a-theme-to-a-site>`_. You will not have to modify the ``lms.env.json``/``cms.env.json`` files; just follow the instructions to add a site theme in http://localhost/admin (starting from step 3).
Extra xblocks and requirements
------------------------------
Installing extra xblocks and requirements
-----------------------------------------
Would you like to include custom xblocks, or extra requirements to your Open edX platform? Additional requirements can be added to the ``openedx/requirements/private.txt`` file. For instance, to include the `polling xblock from Opencraft <https://github.com/open-craft/xblock-poll/>`_::
Would you like to include custom xblocks, or extra requirements to your Open edX platform? Additional requirements can be added to the ``env/build/openedx/requirements/private.txt`` file. For instance, to include the `polling xblock from Opencraft <https://github.com/open-craft/xblock-poll/>`_::
echo "git+https://github.com/open-craft/xblock-poll.git" >> openedx/requirements/private.txt
echo "git+https://github.com/open-craft/xblock-poll.git" >> env/build/openedx/requirements/private.txt
Then, the ``openedx`` docker image must be rebuilt::
make build-openedx
tutor images build openedx
To install xblocks from a private repository that requires authentication, you must first clone the repository inside the ``openedx/requirements`` folder on the host::
@ -56,29 +59,34 @@ To install xblocks from a private repository that requires authentication, you m
Then, declare your extra requirements with the ``-e`` flag in ``openedx/requirements/private.txt``::
echo "-e ./myprivaterepo" >> openedx/requirements/private.txt
echo "-e ./myprivaterepo" >> env/build/openedx/requirements/private.txt
Forked version of edx-platform
------------------------------
.. _edx_platform_fork:
Running a fork of ``edx-platform``
----------------------------------
You may want to run your own flavor of edx-platform instead of the `official version <https://github.com/edx/edx-platform/>`_. To do so, you will have to re-build the openedx image with the proper environment variables pointing to your repository and version::
export EDX_PLATFORM_REPOSITORY=https://mygitrepo/edx-platform.git EDX_PLATFORM_VERSION=my-tag-or-branch
make build-openedx
You can then restart the services which will now be running your forked version of edx-platform::
make restart-openedx
tutor images build openedx \
--build-arg EDX_PLATFORM_REPOSITORY=https://mygitrepo/edx-platform.git \
--build-arg EDX_PLATFORM_VERSION=my-tag-or-branch
Note that your release must be a fork of Hawthorn in order to work. Otherwise, you may have important compatibility issues with other services. In particular, **don't try to run Tutor with older versions of Open edX**.
Running a different ``openedx`` Docker image
--------------------------------------------
By default, Tutor runs the `regis/openedx <https://hub.docker.com/r/regis/openedx/>`_ docker image from Docker Hub. If you have an account on `hub.docker.com <https://hub.docker.com>`_ or you have a private image registry, you can build your image and push it to your registry. Then add the following content to the ``deploy/local/.env`` file::
By default, Tutor runs the `regis/openedx <https://hub.docker.com/r/regis/openedx/>`_ docker image from Docker Hub. If you have an account on `hub.docker.com <https://hub.docker.com>`_ or you have a private image registry, you can build your image and push it to your registry::
OPENEDX_DOCKER_IMAGE=myusername/myimage:mytag
tutor images build openedx --namespace=myusername --version=mytag
tutor images push openedx --namespace=myusername --version=mytag
Your own image will be used next time you run ``make run``.
Then add the following value to your ``config.yml``::
Note that the ``make build`` and ``make push`` commands (from the ``build/`` folder) will no longer work as you expect and that you are responsible for building and pushing the image yourself.
OPENEDX_DOCKER_IMAGE: myusername/openedx:mytag
This value will then be used by Tutor when generating your environment. For instance, this is how you would use your image in a local deployment::
tutor local env
tutor local quickstart

58
docs/dev.rst
View File

@ -5,61 +5,56 @@ Using Tutor for Open edX development
In addition to running Open edX in production, you can use the docker containers for local development. This means you can hack on Open edX without setting up a Virtual Machine. Essentially, this replaces the devstack provided by edX.
To begin with, define development settings::
export EDX_PLATFORM_SETTINGS=tutor.development
These settings are necessary to run a local platform in debug mode.
Run a local webserver
---------------------
Run a local development webserver
---------------------------------
::
make lms-runserver
make cms-runserver
tutor dev runserver lms # Access the lms at http://localhost:8000
tutor dev runserver cms # Access the cms at http://localhost:8001
Open a bash shell
-----------------
::
make lms
make cms
tutor dev shell lms
tutor dev shell cms
Debug edx-platform
------------------
If you have one, you can point to a local version of `edx-platform <https://github.com/edx/edx-platform/>`_ on your host machine::
If you have one, you can point to a local version of `edx-platform <https://github.com/edx/edx-platform/>`_ on your host machine. To do so, pass a ``-P/--edx-platform-path`` option to the commands. For instance::
export EDX_PLATFORM_PATH=/path/to/your/edx-platform
tutor dev shell lms --edx-platform-path=/path/to/edx-platform
Note that you should use an absolute path here, not a relative path (e.g: ``/path/to/edx-platform`` and not ``../edx-platform``).
If you don't want to rewrite this option every time, you can instead define the environment variable::
export TUTOR_EDX_PLATFORM_PATH=/path/to/edx-platform
All development commands will then automatically mount your local repo. For instance, you can add a ``import pdb; pdb.set_trace()`` breakpoint anywhere in your code and run::
make lms-runserver
tutor dev runserver lms --edx-platform-path=/path/to/edx-platform
With a customised edx-platform repo, you must be careful to have settings that are compatible with the docker environment. You are encouraged to copy the ``tutor.development`` settings files to our own repo::
cp -r config/openedx/tutor/lms/ /path/to/edx-platform/lms/envs/tutor
cp -r config/openedx/tutor/cms/ /path/to/edx-platform/cms/envs/tutor
cp -r $(tutor config printroot)/env/apps/openedx/tutor/lms/ /path/to/edx-platform/lms/envs/tutor
cp -r $(tutor config printroot)/env/apps/openedx/tutor/cms/ /path/to/edx-platform/cms/envs/tutor
You can then run your platform with the ``tutor.development`` settings.
**Note**: containers are built on the Hawthorn release. If you are working on a different version of Open edX, you will have to rebuild the images with the right ``EDX_PLATFORM_VERSION`` argument. You may also want to change the ``EDX_PLATFORM_REPOSITORY`` argument to point to your own fork of edx-platform.
You can then run your platform with the ``tutor.development`` settings. See :ref:`the custom settings section <custom_edx_platform_settings>` for settings that are named differently.
**Note:** containers are built on the Hawthorn release. If you are working on a different version of Open edX, you will have to rebuild the ``openedx`` docker images with the version. See the ":ref:`fork edx-platform <edx_platform_fork>`.
Customised themes
-----------------
With Tutor, it's pretty easy to develop your own themes. Start by placing your files inside the ``build/openedx/themes`` directory. For instance, you could start from the ``edx.org`` theme present inside the ``edx-platform`` repository::
With Tutor, it's pretty easy to develop your own themes. Start by placing your files inside the ``env/build/openedx/themes`` directory. For instance, you could start from the ``edx.org`` theme present inside the ``edx-platform`` repository::
cp -r /path/to/edx-platform/themes/edx.org /path/to/tutor/build/openedx/themes/
cp -r /path/to/edx-platform/themes/edx.org $(tutor config printroot)/env/build/openedx/themes/
Don't forget to set the ``EDX_PLATFORM_SETTINGS`` environment variable, as explained above. Then, run a local webserver inside the ``deploy/local`` folder::
Then, run a local webserver::
make lms-runserver
tutor dev runserver lms
The LMS can then be accessed at http://localhost:8000.
@ -67,7 +62,7 @@ You should follow the `Open edX documentation to enable your themes <https://edx
Watch the themes folders for changes (in a different terminal)::
make watch-themes
tutor dev watchthemes
Make changes to some of the files inside your theme directory: the theme assets should be automatically recompiled and visible at http://localhost:8000.
@ -85,7 +80,14 @@ Running python commands
These commands will open a python shell in the lms or the cms::
make lms-python
make cms-python
tutor dev run lms python
tutor dev run cms python
You can then import edx-platform and django modules and execute python code.
.. _custom_edx_platform_settings:
Custom edx-platform settings
----------------------------
In the various ``dev`` commands, the default ``edx-platform`` settings module is ``tutor.development``. If, for some reason, you want to use different settings, you will need to pass the ``-S/--edx-platform-settings`` option to each command. Alternatively, you can define the ``TUTOR_EDX_PLATFORM_SETTINGS`` environment variable.

52
docs/faq.rst Normal file
View File

@ -0,0 +1,52 @@
.. _faq:
FAQ
===
What is Tutor?
--------------
Tutor is a distribution of `Open edX <https://open.edx.org>`_. It uses the original code from the various Open edX repositories, such as `edx-platform <https://github.com/edx/edx-platform/>`_, `cs_comments_service <https://github.com/edx/cs_comments_service>`_, etc. and packages everything in a way that makes it very easy to install, administer and upgrade Open edX. In particular, all services are run inside Docker containers.
What's the difference with the official "native" install?
---------------------------------------------------------
The `native installation <https://openedx.atlassian.net/wiki/spaces/OpenOPS/pages/146440579/Native+Open+edX+Ubuntu+16.04+64+bit+Installation>`_ maintained by edX relies on `Ansible scripts <https://github.com/edx/configuration/>`_ to deploy Open edX on one or multiple servers. These scripts suffer from a couple issues that Tutor tries to address:
1. Complexity: the scripts contain close to 35k lines of code spread over 780 files. They are really hard to understand, debug, and modify, and they are extremly slow. As a consequence, Open edX is often wrongly perceived as a project that is overly complex to manage. In contrast, Tutor generates mostly ``Dockerfile`` and ``docker-compose.yml`` files that make it easy to understand what is going on. Also, the whole installation should take about 10 minutes.
2. Isolation from the OS: Tutor barely needs to touch your server because the entire platform is packaged inside Docker containers. You are thus free to run other services on your server without fear of indirectly crashing your Open edX platform.
3. Compatibility: Open edX is only compatible with Ubuntu 16.04, but that shouldn't mean you are forced to run this specific OS. With Tutor, you can deploy Open edX on just any server you like: Ubuntu 18.04, Red Hat, Debian... All docker-compatible platforms are supported.
4. Security: because you are no longer bound to a single OS, with Tutor you are now free to install security-related upgrades as soon as they become available.
5. Portability: Tutor makes it easy to move your platform from one server to another. Just zip-compress your Tutor project root, send it to another server and you're done.
There are also many features that are not included in the native install, such as a :ref:`web user interface <webui>` for remotely installing the platform, :ref:`Kubernetes deployment <k8s>`, additional languages, etc. You'll discover these differences as you explore Tutor :)
What's the difference with the official devstack?
-------------------------------------------------
The `devstack <https://github.com/edx/devstack>`_ is meant for development only, not for production deployment. Tutor can be used both for production deployment and :ref:`locally hacking on Open edX <development>`.
Is Tutor officially supported by edX?
-------------------------------------
No. Tutor is developed independently from edX. That means that the folks at edX.org are *not* responsible for troubleshooting issues of this project. Please don't bother Ned ;-)
What features are missing from Tutor?
-------------------------------------
Those features are currently not available in Tutor:
- `discovery service <https://github.com/edx/course-discovery/>`_
- `ecommerce <https://github.com/edx/ecommerce>`_
- `analytics <https://github.com/edx/edx-analytics-pipeline>`_
Those extra services were considered low priority while developing this project, but we are planning on adding them to Tutor, eventually. If you need one or more of these services, feel free to let me know by opening a `Github issue <https://github.com/regisb/tutor/issues/>`_. In particular, support for the Analytics stack is going to require a lot of work and I am looking forward to financial sponsorship. Please get in touch if you're interested.
Why should I trust software written by some random guy on the Internet?
-----------------------------------------------------------------------
You shouldn't :) I'm `Régis Behmo <https://github.com/regisb/>`_ and I have been working on Tutor since early 2018. I have been a contributor of the Open edX project since 2015. In particular, I have worked for 2 years on `FUN-MOOC <https://www.fun-mooc.fr/>`_, one of the top 5 largest Open edX platforms in the world. Here are the talks I have presented at the Open edX conferences:
- *FUN: Life in the Avant-Garde*, Oct. 2015 (`video <https://www.youtube.com/watch?v=V1EBo1l8BgY>`__, `slides <http://regisb.github.io/openedx-conference-2015/>`__)
- *Open edX 101: A Source Code Review*, June 2016 (`video <https://www.youtube.com/watch?v=DVku7Y7XQII>`__, `slides <http://regisb.github.io/openedx-conference-2016/>`__)
- *Videofront: a Self-Hosted YouTube*, June 2017 (`video <https://www.youtube.com/watch?v=e7bJchJrmP8&t=5m53s>`__, `slides <http://regisb.github.io/openedx-conference-2017/>`__)

BIN
docs/img/webui.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 200 KiB

31
docs/index.rst
View File

@ -10,6 +10,10 @@ Tutor 🎓 Open edX 1-click install for everyone
:alt: Build status
:target: https://travis-ci.org/regisb/tutor
.. image:: https://img.shields.io/github/stars/regisb/tutor.svg?style=social
:alt: Github stars
:target: https://github.com/regisb/tutor/
.. image:: https://img.shields.io/github/issues/regisb/tutor.svg
:alt: GitHub issues
:target: https://github.com/regisb/tutor/issues
@ -18,6 +22,10 @@ Tutor 🎓 Open edX 1-click install for everyone
:alt: GitHub closed issues
:target: https://github.com/regisb/tutor/issues?q=is%3Aclosed
.. image:: https://img.shields.io/github/license/regisb/tutor.svg
:alt: AGPL License
:target: https://www.gnu.org/licenses/agpl-3.0.en.html
**Tutor** is a one-click install of `Open edX <https://openedx.org>`_, both for production and local development, inside docker containers. Tutor is easy to run, fast, full of cool features, and it is already used by dozens of Open edX platforms in the world.
.. image:: https://asciinema.org/a/octNfEnvIA6jNohCBmODBKizE.png
@ -29,36 +37,39 @@ Tutor 🎓 Open edX 1-click install for everyone
.. include:: quickstart.rst
:start-line: 1
For more advanced usage of Tutor, please refer to the following sections.
But there's a lot more to Tutor than that! For more advanced usage, please refer to the following sections.
.. toctree::
:maxdepth: 2
:caption: User guide
install
quickstart
requirements
local
k8s
options
customise
dev
k8s
webui
troubleshooting
missing
tutor
faq
Disclaimers & Warnings
----------------------
Source code
-----------
This project is the follow-up of my work on an `install from scratch of Open edX <https://github.com/regisb/openedx-install>`_. It does not rely on any hack or complex deployment script. In particular, we do not use the Open edX `Ansible deployment playbooks <https://github.com/edx/configuration/>`_. That means that the folks at edX.org are *not* responsible for troubleshooting issues of this project. Please don't bother Ned ;-)
In case of trouble, please follow the instructions in the :ref:`troubleshooting` section.
The complete source code for Tutor is available on Github: https://github.com/regisb/tutor
Contributing
------------
We go to great lengths to make it as easy as possible for people to run Open edX inside Docker containers. If you have an improvement idea, feel free to `open an issue on Github <https://github.com/regisb/tutor/issues/new>`_ so that we can discuss it. `Pull requests <https://github.com/regisb/tutor/pulls>`_ will be happily examined, too! However, we should be careful to keep the project lean and simple: both to use and to modify. Optional features should not make the user experience more complex. Instead, documentation on how to add the feature is preferred.
We go to great lengths to make it as easy as possible for people to run Open edX inside Docker containers. If you have an improvement idea, feel free to `open an issue on Github <https://github.com/regisb/tutor/issues/new>`_ so that we can discuss it. `Pull requests <https://github.com/regisb/tutor/pulls>`_ will be happily examined, too!
License
-------
This work is licensed under the terms of the `GNU Affero General Public License (AGPL) <https://github.com/regisb/tutor/blob/master/LICENSE.txt>`_.
The AGPL license covers the Tutor code, including the Dockerfiles, but not the content of the Docker images which can be downloaded from https://hub.docker.com. Software other than Tutor provided with the docker images retain their original license.
The :ref:`Tutor Web UI <webui>` depends on the `Gotty <https://github.com/yudai/gotty/>`_ binary, which is provided under the terms of the `MIT license <https://github.com/yudai/gotty/blob/master/LICENSE>`_.

41
docs/install.rst Normal file
View File

@ -0,0 +1,41 @@
.. _install:
Installation
============
Requirements
------------
The only prerequisite for running this is a working docker install. Both docker and docker-compose are required. Follow the instructions from the official documentation:
- `Docker <https://docs.docker.com/engine/installation/>`_
- `Docker compose <https://docs.docker.com/compose/install/>`_
Note that the production web server container will bind to port 80, so if you already have a web server running (Apache or Nginx, for instance), you should stop it.
You should be able to run Open edX on any platform that supports Docker, including Mac OS and Windows. Tutor was tested under various versions of Ubuntu and Mac OS.
At a minimum, the server running the containers should have 4 Gb of RAM. With less memory, the Open edX , the deployment procedure might crash during migrations (see the :ref:`troubleshooting` section) and the platform will be unbearably slow.
Also, the host running the containers should be a 64 bit platform. (images are not built for i386 systems)
Direct binary downloads
-----------------------
The latest binaries can be downloaded from https://github.com/regisb/tutor/releases.
Installing from pip
-------------------
If, for some reason, you'd rather install from pypi instead of downloading a binary, run::
pip install tutor-openedx
Installing from source
----------------------
::
git clone https://github.com/regisb/tutor
cd tutor
python setup.py develop

19
docs/k8s.rst
View File

@ -23,7 +23,7 @@ In the following, we assume you have a working Kubernetes platform. For a start,
Start Minikube::
make minikube-start
minikube start
When minikube starts, it spawns a virtual machine (VM) which you can configure in your VM manager: on most platforms, this is Virtualbox. You should configure your VM to have at least 4Gb RAM; otherwise, database migrations will crash halfway, and that's a nasty issue...
@ -44,16 +44,12 @@ With Kubernetes, your Open edX platform will not be available at localhost or st
where ``MINIKUBEIP`` should be replaced by the result of the command ``minikube ip``.
In the following, all commands should be run inside the ``deploy/k8s`` folder::
cd deploy/k8s
Quickstart
----------
Launch the platform on k8s in 1 click::
make all
tutor k8s quickstart
All Kubernetes resources are associated to the "openedx" namespace. If you don't see anything in the Kubernetes dashboard, you are probably looking at the wrong namespace... 😉
@ -65,26 +61,27 @@ Upgrading
After pulling updates from the Tutor repository, you can apply changes with::
make upgrade
tutor k8s stop
tutor k8s start
Accessing the Kubernetes dashboard
----------------------------------
Depending on your Kubernetes provider, you may need to create a dashboard yourself. To do so, run::
make k8s-dashboard
kubectl create -f https://raw.githubusercontent.com/kubernetes/dashboard/master/aio/deploy/recommended/kubernetes-dashboard.yaml
Then, you will have to create an admin user::
make k8s-admin
tutor k8s adminuser
Print the admin token required for authentication, and copy its value::
make k8s-admin-token
tutor k8s admintoken
Create a proxy to the Kubernetes API server::
k8s-proxy
kubectl proxy
Use the token to log in the dashboard at the following url: http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/

85
docs/local.rst
View File

@ -3,44 +3,57 @@
Local deployment
================
This method is for deploying Open edX locally on a single server. Docker images are orchestrated with `docker-compose <https://docs.docker.com/compose/overview/>`_.
This method is for deploying Open edX locally on a single server, where docker images are orchestrated with `docker-compose <https://docs.docker.com/compose/overview/>`_.
The following commands should be run inside the ``deploy/local`` folder::
In the following, environment and data files will be generated in a user-specific project folder which will be referred to as the "**project root**". On Linux, the default project root is ``~/.local/share/tutor``. An alternative project root can be defined by passing the ``--root=...`` option to most commands, or define the ``TUTOR_ROOT=...`` environment variable.
cd deploy/local
All-in-one command
------------------
You can use these commands individually instead of running the full installation with ``make all``.
A fully-functional platform can be configured and run in one command::
tutor local quickstart
But you may want to run commands one at a time: it's faster when you need to run only part of the local deployment process, and it helps you understand how your platform works. In the following we decompose the ``quickstart`` command.
Configuration
-------------
::
make configure
tutor config interactive
This is the only non-automatic step in the install process. You will be asked various questions about your Open edX platform and appropriate configuration files will be generated. If you would like to automate this step then you should run ``make configure`` interactively once. After that, you will have a ``config.json`` file at the root of the repository.
This is the only non-automatic step in the install process. You will be asked various questions about your Open edX platform and appropriate configuration files will be generated. If you would like to automate this step then you should run ``tutor config interactive`` once. After that, there will be a ``config.yml`` file at the root of the project folder: this file contains all the configuration values for your platform, such as randomly generated passwords, domain names, etc.
If you want to run a fully automated install, upload the ``config.json`` file to wherever you want to run Open edX, and then run ``make configure SILENT=1`` instead of ``make configure``. All values from ``config.json`` will be automatically loaded.
If you want to run a fully automated install, upload the ``config.yml`` file to wherever you want to run Open edX. You can then entirely skip the configuration step.
Downloading docker images
-------------------------
Environment files generation
----------------------------
::
make update
tutor local env
You will need to download the docker images from `Docker Hub <https://hub.docker.com/r/regis/openedx/>`_. Depending on your bandwidth, this might take a long time. Minor image updates will be incremental, and thus much faster.
This command generates environment files, such as the ``*.env.json``, ``*.auth.json`` files, the ``docker-compose.yml`` file, etc. They are generated from templates and the configuration values stored in ``config.yml``. The generated files are placed in the ``env/local`` subfolder of the project root. You may modify and delete those files at will, since they can be easily re-generated with the same ``tutor local env`` command.
Update docker images
--------------------
::
tutor local pullimages
This downloads the latest version of the docker images from `Docker Hub <https://hub.docker.com/r/regis/openedx/>`_. Depending on your bandwidth, this might take a long time. Minor image updates will be incremental, and thus much faster.
Database management
-------------------
::
make databases
tutor local databases
This command should be run just once. It will create the required databases tables and apply database migrations for all applications.
If migrations are stopped with a ``Killed`` message, this certainly means the docker containers don't have enough RAM. See the :ref:`troubleshooting` section.
Running Open edX
@ -48,26 +61,26 @@ Running Open edX
::
make run
tutor local start
This will launch the various docker containers required for your Open edX platform. The LMS and the Studio will then be reachable at the domain name you specified during the configuration step. You can also access them at http://localhost and http://studio.localhost.
To stop the running containers, just hit Ctrl+C.
In production, you will probably want to daemonize the services. Instead of ``make run``, run::
In production, you will probably want to daemonize the services. To do so, run::
make daemonize
tutor local start --detach
And then, to stop all services::
make stop
tutor local stop
Creating a new user with staff and admin rights
-----------------------------------------------
You will most certainly need to create a user to administer the platform. Just run::
make staff-user USERNAME=yourusername EMAIL=user@email.com
tutor createuser --staff --superuser yourusername user@email.com
You will asked to set the user password interactively.
@ -76,30 +89,30 @@ Importing the demo course
On a fresh install, your platform will not have a single course. To import the `Open edX demo course <https://github.com/edx/edx-demo-course>`_, run::
make demo-course
tutor local importdemocourse
Updating the course search index
--------------------------------
The course search index can be updated with::
make reindex-courses
tutor local indexcourses
Run this command periodically to ensure that course search results are always up-to-date.
.. _portainer:
Docker container web UI with `Portainer <https://portainer.io/>`_
-----------------------------------------------------------------
Docker container web UI with `Portainer <https://portainer.io/>`__
------------------------------------------------------------------
Portainer is a web UI for managing docker containers. It lets you view your entire Open edX platform at a glace. Try it! It's really cool::
make portainer
tutor local portainer
.. .. image:: https://portainer.io/images/screenshots/portainer.gif
..:alt: Portainer demo
After launching your platfom, the web UI will be available at `http://localhost:9000 <http://localhost:9000>`_. You will be asked to define a password for the admin user. Then, select a "Local environment" to work on and hit "Connect" and select the "local" group to view all running containers.
After launching your platfom, the web UI will be available at `http://localhost:9000 <http://localhost:9000>`_. You will be asked to define a password for the admin user. Then, select a "Local environment" to work on; hit "Connect" and select the "local" group to view all running containers.
Among many other things, you'll be able to view the logs for each container, which is really useful.
@ -118,14 +131,22 @@ Additional commands
All available commands can be listed by running::
make help
tutor local help
How to upgrade from `openedx-docker`
------------------------------------
Upgrading from earlier versions
-------------------------------
Before this project was renamed to Tutor, it was called "openedx-docker", and many additional changes were introduced at the same time. If you checked out openedx-docker before the rename, you can upgrade by running the following commands::
Versions 1 and 2 of Tutor were organized differently: they relied on many different ``Makefile`` and ``make`` commands instead of a single ``tutor`` executable. To migrate from an earlier version, you should first stop your platform::
make stop # Stop all services
git pull # Upgrade to the latest version of Tutor
make upgrade-to-tutor # Move some configuration files
make local # Re-configure and restart the platform
make stop
Then, create the Tutor project root and move your data::
mkdir -p $(tutor config printroot)
mv config.json data/ $(tutor config printroot)
`Download <https://github.com/regisb/tutor/releases>`_ the latest stable release of Tutor, uncompress the file and place the ``tutor`` executable in your path.
Finally, start your platform again::
tutor local quickstart

12
docs/missing.rst
View File

@ -1,12 +0,0 @@
.. _missing:
Missing features
================
Those features are currently not available in Tutor:
- `discovery service <https://github.com/edx/course-discovery/>`_
- `ecommerce <https://github.com/edx/ecommerce>`_
- `analytics <https://github.com/edx/edx-analytics-pipeline>`_
Those extra services were considered low priority while developing this project. If you need one or more of these services, feel free to let me know by opening an issue.

19
docs/options.rst
View File

@ -18,13 +18,13 @@ The following DNS records must exist and point to your server::
Thus, **this feature will (probably) not work in development** because the DNS records will (probably) not point to your development machine.
To download the certificate manually, run::
To create the certificate manually, run::
make https-certificate
tutor local https create
To renew the certificate, run this command once per month::
make https-certificate-renew
tutor local https renew
Student notes
-------------
@ -46,19 +46,10 @@ Android app (beta)
The Android app for your platform can be easily built in just one command::
make android
tutor android build debug
If all goes well, the debuggable APK for your platform should then be available in ./data/android. To obtain a release APK, you will need to obtain credentials from the app store and add them to ``config/android/gradle.properties``. Then run::
make android-release
tutor android build release
Building the Android app for an Open edX platform is currently labeled as a **beta feature** because it was not fully tested yet. In particular, there is no easy mechanism for overriding the edX assets in the mobile app. This is still a work-in-progress.
Stats
-----
By default, the install script will collect some information about your install and send it to a private server. The only transmitted information are the LMS domain name and the ID of the install. To disable stats collection, define the following environment variable::
export DISABLE_STATS=1
If you decide to disable stats, please send me a message to tell me about your platform!

12
docs/quickstart.rst
View File

@ -3,18 +3,16 @@
Quickstart
==========
::
git clone https://github.com/regisb/tutor
cd tutor/deploy/local
make all
1. `Download <https://github.com/regisb/tutor/releases>`_ the latest stable release of Tutor, uncompress the file and place the ``tutor`` executable in your path.
2. Run ``tutor local quickstart``
3. You're done!
**That's it?**
Yes :) This is what happens when you run ``make all``:
Yes :) This is what happens when you run ``tutor local quickstart``:
1. You answer a few questions about the configuration of your Open edX platform and your :ref:`selected options <options>`
2. Configuration files are generated.
2. Configuration files are generated from templates.
3. Docker images are downloaded.
4. Docker containers are provisioned.
5. A full, production-ready platform is run with docker-compose.

18
docs/requirements.rst
View File

@ -1,18 +0,0 @@
.. _requirements:
Requirements
============
The only prerequisite for running this is a working docker install. You will need both docker and docker-compose. Follow the instructions from the official documentation:
- `make <https://www.gnu.org/software/make/>`_
- `Docker <https://docs.docker.com/engine/installation/>`_
- `Docker compose <https://docs.docker.com/compose/install/>`_
Note that the production web server container will bind to port 80, so if you already have a web server running (Apache or Nginx, for instance), you should stop it.
You should be able to run Open edX on any platform that supports Docker, including Mac OS and Windows. Tutor was tested under various versions of Ubuntu and Mac OS.
At a minimum, the server running the containers should have 4 Gb of RAM; otherwise, the deployment procedure will crash during migrations (see the :ref:`troubleshooting` section).
Also, the host running the containers should be a 64 bit platform. (images are not built for i386 systems)

30
docs/troubleshooting.rst
View File

@ -15,21 +15,20 @@ What should you do if you have a problem?
Logging
-------
To view the logs from all containers use the `docker-compose logs <https://docs.docker.com/compose/reference/logs/>`_ command::
To view the logs from all containers use the ``tutor local logs`` command, which was modeled on the standard `docker-compose logs <https://docs.docker.com/compose/reference/logs/>`_ command::
docker-compose logs -f
tutor local logs --follow
To view the logs from just one container, for instance the web server::
docker-compose logs -f nginx
tutor local logs --follow nginx
The last commands produce the logs since the creation of the containers, which can be a lot. Similar to a ``tail -f``, you can run::
docker-compose logs --tail=0 -f
tutor local logs--tail=0 -f
If you'd rather use a graphical user interface for viewing logs, you are encouraged to try out :ref:`Portainer <portainer>`.
"Cannot start service nginx: driver failed programming external connectivity"
-----------------------------------------------------------------------------
@ -42,15 +41,14 @@ The containerized Nginx needs to listen to ports 80 and 443 on the host. If ther
However, you might not want to do that if you need a webserver for running non-Open edX related applications. In such cases...
2. Run the nginx container on different ports: you can create a ``.env`` file in the ``tutor`` directory in which you indicate different ports. For instance::
2. Run the nginx container on different ports: to do so, indicate different ports in the ``config.yml`` file. For instance::
cat .env
NGINX_HTTP_PORT=81
NGINX_HTTPS_PORT=444
NGINX_HTTP_PORT: 81
NGINX_HTTPS_PORT: 444
In this example, the nginx container ports would be mapped to 81 and 444, instead of 80 and 443.
In this example, the nginx container ports would be mapped to 81 and 444, instead of 80 and 443. Then, re-generate the environment with ``tutor local env`` and restart nginx with ``tutor local restart nginx``.
You should note that with the latter solution, it is your responsibility to configure the webserver on the host as a proxy to the nginx container. See `this <https://github.com/regisb/tutor/issues/69#issuecomment-425916825>`_ for http, and `this <https://github.com/regisb/tutor/issues/90#issuecomment-437687294>`_ for https.
You should note that with the latter solution, it is your responsibility to configure the webserver on the host as a proxy to the nginx container. See `this github issue <https://github.com/regisb/tutor/issues/69#issuecomment-425916825>`_ for http, and `this other github issue <https://github.com/regisb/tutor/issues/90#issuecomment-437687294>`_ for https.
Help! The Docker containers are eating all my RAM/CPU/CHEESE
------------------------------------------------------------
@ -62,7 +60,7 @@ You can identify which containers are consuming most resources by running::
"Running migrations... Killed!"
-------------------------------
The LMS and CMS containers require at least 4 GB RAM, in particular to run the Open edX SQL migrations. On Docker for Mac, by default, containers are allocated at most 2 GB of RAM. On Mac OS, if the ``make all`` command dies after displaying "Running migrations", you most probably need to increase the allocated RAM. Follow `these instructions from the official Docker documentation <https://docs.docker.com/docker-for-mac/#advanced>`_.
Older versions of Open edX required at least 4 GB RAM, in particular to run the Open edX SQL migrations. On Docker for Mac, by default, containers are allocated at most 2 GB of RAM. On Mac OS, if the ``tutor local quickstart`` command dies after displaying "Running migrations", you most probably need to increase the allocated RAM. Follow `these instructions from the official Docker documentation <https://docs.docker.com/docker-for-mac/#advanced>`_.
``Build failed running pavelib.servers.lms: Subprocess return code: 1``
@ -70,15 +68,15 @@ The LMS and CMS containers require at least 4 GB RAM, in particular to run the O
::
python manage.py lms --settings=development print_setting STATIC_ROOT 2>/dev/null
python manage.py lms print_setting STATIC_ROOT 2>/dev/null
...
Build failed running pavelib.servers.lms: Subprocess return code: 1`"
This might occur when you run a ``paver`` command. ``/dev/null`` eats the actual error, so you will have to run the command manually. Run ``make lms`` (or ``make cms``) to open a bash session and then::
This might occur when you run a ``paver`` command. ``/dev/null`` eats the actual error, so you will have to run the command manually. Run ``tutor dev shell lms`` (or ``tutor dev shell cms``) to open a bash session and then::
python manage.py lms --settings=development print_setting STATIC_ROOT
python manage.py lms print_setting STATIC_ROOT
Of course, you should replace `development` with your own settings. The error produced should help you better understand what is happening.
The error produced should help you better understand what is happening.
``ValueError: Unable to configure handler 'local'``
---------------------------------------------------

33
docs/tutor.rst
View File

@ -3,9 +3,34 @@
Tutor development
=================
Pushing to Docker Hub
---------------------
Start by cloning the Tutor repository::
The images are built, tagged and uploaded to Docker Hub in one command::
git clone https://github.com/regisb/tutor.git
cd tutor/
make dockerhub
Install requirements
--------------------
::
pip install -r requirements/dev.txt
Bundle ``tutor`` executable
---------------------------
::
make bundle
Generate the documentation
--------------------------
::
pip install sphinx sphinx_rtd_theme
cd docs/
make html
You can then browse the documentation with::
make browse

24
docs/webui.rst Normal file
View File

@ -0,0 +1,24 @@
.. _webui:
Web UI
======
Tutor comes with a web user interface (UI) that allows you to administer your Open edX platform remotely. It's especially convenient for remote administration of the platform.
Launching the web UI
--------------------
::
tutor webui start
You can then access the interface at http://localhost:3737, or http://youserverurl:3737.
.. image:: img/webui.png
Authentication
--------------
**WARNING** Once you launch the web UI, it is accessible by everyone, which means that your Open edX platform is at risk. If you are planning to leave the web UI up for a long time, you should setup a user and password for authentication::
tutor webui configure

6
requirements/base.in Normal file
View File

@ -0,0 +1,6 @@
appdirs
click
click_repl
jinja2
kubernetes
pyyaml>=4.2b1

36
requirements/base.txt Normal file
View File

@ -0,0 +1,36 @@
#
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements/base.txt requirements/base.in
#
adal==1.2.1 # via kubernetes
appdirs==1.4.3
asn1crypto==0.24.0 # via cryptography
cachetools==3.1.0 # via google-auth
certifi==2018.11.29 # via kubernetes, requests
cffi==1.11.5 # via cryptography
chardet==3.0.4 # via requests
click-repl==0.1.6
click==7.0
cryptography==2.5 # via adal
google-auth==1.6.2 # via kubernetes
idna==2.8 # via requests
jinja2==2.10
kubernetes==8.0.1
markupsafe==1.1.0 # via jinja2
oauthlib==3.0.1 # via requests-oauthlib
prompt-toolkit==2.0.8 # via click-repl
pyasn1-modules==0.2.4 # via google-auth
pyasn1==0.4.5 # via pyasn1-modules, rsa
pycparser==2.19 # via cffi
pyjwt==1.7.1 # via adal
python-dateutil==2.8.0 # via adal, kubernetes
pyyaml==4.2b4
requests-oauthlib==1.2.0 # via kubernetes
requests==2.21.0 # via adal, kubernetes, requests-oauthlib
rsa==4.0 # via google-auth
six==1.12.0 # via click-repl, cryptography, google-auth, kubernetes, prompt-toolkit, python-dateutil, websocket-client
urllib3==1.24.1 # via kubernetes, requests
wcwidth==0.1.7 # via prompt-toolkit
websocket-client==0.54.0 # via kubernetes

3
requirements/dev.in Normal file
View File

@ -0,0 +1,3 @@
-r base.txt
pip-tools
pyinstaller

42
requirements/dev.txt Normal file
View File

@ -0,0 +1,42 @@
#
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements/dev.txt requirements/dev.in
#
adal==1.2.1
altgraph==0.16.1 # via macholib, pyinstaller
appdirs==1.4.3
asn1crypto==0.24.0
cachetools==3.1.0
certifi==2018.11.29
cffi==1.11.5
chardet==3.0.4
click-repl==0.1.6
click==7.0
cryptography==2.5
future==0.17.1 # via pefile
google-auth==1.6.2
idna==2.8
jinja2==2.10
kubernetes==8.0.1
macholib==1.11 # via pyinstaller
markupsafe==1.1.0
oauthlib==3.0.1
pefile==2018.8.8 # via pyinstaller
pip-tools==3.2.0
prompt-toolkit==2.0.8
pyasn1-modules==0.2.4
pyasn1==0.4.5
pycparser==2.19
pyinstaller==3.4
pyjwt==1.7.1
python-dateutil==2.8.0
pyyaml==4.2b4
requests-oauthlib==1.2.0
requests==2.21.0
rsa==4.0
six==1.12.0
urllib3==1.24.1
wcwidth==0.1.7
websocket-client==0.54.0

49
setup.py Normal file
View File

@ -0,0 +1,49 @@
import io
import os
from setuptools import setup
here = os.path.abspath(os.path.dirname(__file__))
with io.open(os.path.join(here, "README.rst"), "rt", encoding="utf8") as f:
readme = f.read()
setup(
name="tutor-openedx",
version="3.0.0",
url="http://docs.tutor.overhang.io/",
project_urls={
"Documentation": "https://docs.tutor.overhang.io/",
"Code": "https://github.com/regisb/tutor",
"Issue tracker": "https://github.com/regisb/tutor/issues",
},
license="AGPLv3",
author="Régis Behmo",
author_email="regis@behmo.com",
description="The Open edX distribution for the busy system administrator",
long_description=readme,
packages=["tutor"],
include_package_data=True,
python_requires=">=3.6",
install_requires=[
"appdirs",
"click",
"click_repl",
"jinja2",
"kubernetes",
"pyyaml"
],
entry_points={
'console_scripts': [
'tutor=tutor.cli:main',
],
},
classifiers=[
"Development Status :: 5 - Production/Stable",
"Intended Audience :: Developers",
"License :: OSI Approved :: GNU Affero General Public License v3",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
],
)

32
tutor.spec Normal file
View File

@ -0,0 +1,32 @@
# -*- mode: python -*-
block_cipher = None
a = Analysis(['bin/main'],
pathex=['/home/data/regis/projets/openedx/repos/tutor'],
binaries=[],
datas=[('./tutor/templates', './tutor/templates')],
hiddenimports=[],
hookspath=[],
runtime_hooks=[],
excludes=[],
win_no_prefer_redirects=False,
win_private_assemblies=False,
cipher=block_cipher,
noarchive=False)
pyz = PYZ(a.pure, a.zipped_data,
cipher=block_cipher)
exe = EXE(pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
[],
name='tutor',
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
runtime_tmpdir=None,
console=True )

0
build/openedx/settings/cms/__init__.py → tutor/__init__.py
View File

69
tutor/android.py Normal file
View File

@ -0,0 +1,69 @@
import click
from . import config
from . import env as tutor_env
from . import fmt
from . import opts
from . import utils
DOCKER_IMAGE = "regis/openedx-android:hawthorn"
@click.group(
help="Build an Android app for your Open edX platform [BETA FEATURE]"
)
def android():
pass
@click.command(
help="Generate the environment required for building the application"
)
@opts.root
def env(root):
tutor_env.render_target(root, config.load(root), "android")
@click.group(
help="Build the application"
)
def build():
pass
@click.command(
help="Build the application in debug mode"
)
@opts.root
def debug(root):
docker_run(
root, "./gradlew", "assembleProdDebuggable", "&&",
"cp", "OpenEdXMobile/build/outputs/apk/prod/debuggable/*.apk", "/openedx/data/"
)
click.echo(fmt.info("The debuggable APK file is available in {}".format(tutor_env.data_path(root, "android"))))
@click.command(
help="Build the application in release mode"
)
@opts.root
def release(root):
docker_run(root, "./gradlew", "assembleProdRelease")
click.echo(fmt.info("The production APK file is available in {}".format(tutor_env.data_path(root, "android"))))
@click.command(
help="Pull the docker image"
)
@opts.root
def pullimage():
utils.execute("docker", "pull", DOCKER_IMAGE)
def docker_run(root, *command):
utils.docker_run(
"--volume={}/:/openedx/config/".format(tutor_env.pathjoin(root, "android")),
"--volume={}:/openedx/data".format(tutor_env.data_path(root, "android")),
DOCKER_IMAGE,
*command
)
build.add_command(debug)
build.add_command(release)
android.add_command(build)
android.add_command(env)
android.add_command(pullimage)

51
tutor/cli.py Executable file
View File

@ -0,0 +1,51 @@
#! /usr/bin/env python3
import sys
import click
import click_repl
from .android import android
from .config import config
from .dev import dev
from .images import images
from .k8s import k8s
from .local import local
from .ui import ui
from .webui import webui
from . import exceptions
from . import fmt
def main():
try:
cli()
except exceptions.TutorError as e:
sys.stderr.write(fmt.error("Error: {}\n".format(e.args[0])))
sys.exit(1)
@click.group(context_settings={'help_option_names': ['-h', '--help', 'help']})
@click.version_option()
def cli():
pass
@click.command(
help="Print this help",
name="help",
)
def print_help():
with click.Context(cli) as context:
click.echo(cli.get_help(context))
click_repl.register_repl(cli, name="ui")
cli.add_command(images)
cli.add_command(config)
cli.add_command(local)
cli.add_command(dev)
cli.add_command(android)
cli.add_command(k8s)
cli.add_command(ui)
cli.add_command(webui)
cli.add_command(print_help)
if __name__ == "__main__":
main()

178
tutor/config.py Normal file
View File

@ -0,0 +1,178 @@
import json
import os
import yaml
import click
from . import exceptions
from . import env
from . import fmt
from . import opts
@click.group(
short_help="Configure Open edX",
help="""Configure Open edX and store configuration values in $TUTOR_ROOT/config.yml"""
)
@opts.root
def config(root):
pass
@click.command(
help="Create and save configuration interactively",
)
@opts.root
@opts.key_value
def interactive(root, s):
config = {}
load_files(config, root)
for k, v in s:
config[k] = v
load_interactive(config)
save(config, root)
@click.command(
help="Create and save configuration without user interaction",
)
@opts.root
@opts.key_value
def noninteractive(root, s):
config = {}
load_files(config, root)
for k, v in s:
config[k] = v
save(config, root)
@click.command(
help="Print the tutor project root",
)
@opts.root
def printroot(root):
click.echo(root)
def load(root):
"""
Load configuration, and generate it interactively if the file does not
exist.
"""
config = {}
load_files(config, root)
if not os.path.exists(config_path(root)):
load_interactive(config)
save(config, root)
load_defaults(config)
return config
def load_files(config, root):
convert_json2yml(root)
# Load base values
base = yaml.load(env.read("config.yml"))
for k, v in base.items():
config[k] = v
# Load user file
path = config_path(root)
if os.path.exists(path):
with open(path) as fi:
loaded = yaml.load(fi.read())
for key, value in loaded.items():
config[key] = value
def load_interactive(config):
ask("Your website domain name for students (LMS)", "LMS_HOST", config)
ask("Your website domain name for teachers (CMS)", "CMS_HOST", config)
ask("Your platform name/title", "PLATFORM_NAME", config)
ask("Your public contact email address", "CONTACT_EMAIL", config)
ask_choice(
"The default language code for the platform",
"LANGUAGE_CODE", config,
['en', 'am', 'ar', 'az', 'bg-bg', 'bn-bd', 'bn-in', 'bs', 'ca',
'ca@valencia', 'cs', 'cy', 'da', 'de-de', 'el', 'en-uk', 'en@lolcat',
'en@pirate', 'es-419', 'es-ar', 'es-ec', 'es-es', 'es-mx', 'es-pe',
'et-ee', 'eu-es', 'fa', 'fa-ir', 'fi-fi', 'fil', 'fr', 'gl', 'gu',
'he', 'hi', 'hr', 'hu', 'hy-am', 'id', 'it-it', 'ja-jp', 'kk-kz',
'km-kh', 'kn', 'ko-kr', 'lt-lt', 'ml', 'mn', 'mr', 'ms', 'nb', 'ne',
'nl-nl', 'or', 'pl', 'pt-br', 'pt-pt', 'ro', 'ru', 'si', 'sk', 'sl',
'sq', 'sr', 'sv', 'sw', 'ta', 'te', 'th', 'tr-tr', 'uk', 'ur', 'vi',
'uz', 'zh-cn', 'zh-hk', 'zh-tw'],
)
ask_bool(
("Activate SSL/TLS certificates for HTTPS access? Important note:"
"this will NOT work in a development environment."),
"ACTIVATE_HTTPS", config
)
ask_bool(
"Activate Student Notes service (https://open.edx.org/features/student-notes)?",
"ACTIVATE_NOTES", config
)
ask_bool(
"Activate Xqueue for external grader services (https://github.com/edx/xqueue)?",
"ACTIVATE_XQUEUE", config
)
def load_defaults(config):
defaults = yaml.load(env.read("config-defaults.yml"))
for k, v in defaults.items():
if k not in config:
config[k] = v
def ask(question, key, config):
default = env.render_str(config[key], config)
config[key] = click.prompt(
fmt.question(question),
prompt_suffix=" ", default=default, show_default=True,
)
def ask_bool(question, key, config):
default = "y" if config[key] else "n"
suffix = " [Yn]" if config[key] else " [yN]"
answer = click.prompt(
fmt.question(question) + suffix,
type=click.Choice(["y", "n"]),
prompt_suffix=" ", default=default, show_default=False, show_choices=False,
)
config[key] = answer == "y"
def ask_choice(question, key, config, choices):
default = config[key]
answer = click.prompt(
fmt.question(question),
type=click.Choice(choices),
prompt_suffix=" ", default=default, show_choices=False,
)
config[key] = answer
def convert_json2yml(root):
json_path = os.path.join(root, "config.json")
if not os.path.exists(json_path):
return
if os.path.exists(config_path(root)):
raise exceptions.TutorError(
"Both config.json and config.yml exist in {}: only one of these files must exist to continue".format(root)
)
with open(json_path) as fi:
config = json.load(fi)
save(config, root)
os.remove(json_path)
click.echo(fmt.info("File config.json detected in {} and converted to config.yml".format(root)))
def save(config, root):
env.render_dict(config)
path = config_path(root)
directory = os.path.dirname(path)
if not os.path.exists(directory):
os.makedirs(directory)
with open(path, "w") as of:
yaml.dump(config, of, default_flow_style=False)
click.echo(fmt.info("Configuration saved to {}".format(path)))
def config_path(root):
return os.path.join(root, "config.yml")
config.add_command(interactive)
config.add_command(noninteractive)
config.add_command(printroot)

101
tutor/dev.py Normal file
View File

@ -0,0 +1,101 @@
import click
from . import env as tutor_env
from . import opts
from . import utils
@click.group(
help="Run Open edX platform with development settings",
)
def dev():
pass
@click.command(
help="Run a command in one of the containers",
context_settings={"ignore_unknown_options": True},
)
@opts.root
@opts.edx_platform_path
@opts.edx_platform_settings
@click.argument("service")
@click.argument("command", default=None, required=False)
@click.argument("args", nargs=-1, required=False)
def run(root, edx_platform_path, edx_platform_settings, service, command, args):
run_command = [service]
if command:
run_command.append(command)
if args:
run_command += args
port = service_port(service)
docker_compose_run_with_port(
root, edx_platform_path, edx_platform_settings, port, *run_command
)
@click.command(
help="Run a development server",
)
@opts.root
@opts.edx_platform_path
@opts.edx_platform_settings
@click.argument("service", type=click.Choice(["lms", "cms"]))
def runserver(root, edx_platform_path, edx_platform_settings, service):
port = service_port(service)
docker_compose_run_with_port(
root, edx_platform_path, edx_platform_settings, port,
service, "./manage.py", "runserver", "0.0.0.0:{}".format(port),
)
@click.command(
help="Launch a shell",
)
@opts.root
@opts.edx_platform_path
@opts.edx_platform_settings
@click.argument("service", type=click.Choice(["lms", "cms"]))
def shell(root, edx_platform_path, edx_platform_settings, service):
port = service_port(service)
docker_compose_run_with_port(
root, edx_platform_path, edx_platform_settings, port,
service, "bash"
)
@click.command(
help="Watch for changes in your themes and recompile assets when needed"
)
@opts.root
@opts.edx_platform_path
@opts.edx_platform_settings
def watchthemes(root, edx_platform_path, edx_platform_settings):
docker_compose_run(
root, edx_platform_path, edx_platform_settings,
"--no-deps", "lms", "openedx-assets", "watch-themes", "--env", "dev"
)
def docker_compose_run_with_port(root, edx_platform_path, edx_platform_settings, port, *command):
docker_compose_run(
root, edx_platform_path, edx_platform_settings,
"-p", "{port}:{port}".format(port=port), *command
)
def docker_compose_run(root, edx_platform_path, edx_platform_settings, *command):
run_command = [
"run", "--rm",
"-e", "SETTINGS={}".format(edx_platform_settings),
]
if edx_platform_path:
run_command.append("--volume={}:/openedx/edx-platform".format(edx_platform_path))
run_command += command
return utils.docker_compose(
"-f", tutor_env.pathjoin(root, "local", "docker-compose.yml"),
"--project-name", "tutor_dev",
*run_command
)
def service_port(service):
return 8000 if service == "lms" else 8001
dev.add_command(run)
dev.add_command(runserver)
dev.add_command(shell)
dev.add_command(watchthemes)

107
tutor/env.py Normal file
View File

@ -0,0 +1,107 @@
import codecs
import os
import shutil
import jinja2
from . import exceptions
from . import utils
TEMPLATES_ROOT = os.path.join(os.path.dirname(__file__), "templates")
def render_target(root, config, target):
"""
Render the templates located in `target` and store them with the same
hierarchy at `root`.
"""
for src, dst in walk_templates(root, target):
if is_part_of_env(src):
with codecs.open(src, encoding='utf-8') as fi:
substituted = render_str(fi.read(), config)
with open(dst, "w") as of:
of.write(substituted)
def render_dict(config):
"""
Render the values from the dict. This is useful for rendering the default
values from config.yml.
Args:
config (dict)
"""
rendered = {}
for key, value in config.items():
if isinstance(value, str):
rendered[key] = render_str(value, config)
else:
rendered[key] = value
for k, v in rendered.items():
config[k] = v
pass
def render_str(text, config):
"""
Args:
text (str)
config (dict)
Return:
substituted (str)
"""
template = jinja2.Template(text, undefined=jinja2.StrictUndefined)
try:
return template.render(
RAND8=utils.random_string(8),
RAND24=utils.random_string(24),
**config
)
except jinja2.exceptions.UndefinedError as e:
raise exceptions.TutorError("Missing configuration value: {}".format(e.args[0]))
def copy_target(root, target):
"""
Copy the templates located in `path` and store them with the same hierarchy
at `root`.
"""
for src, dst in walk_templates(root, target):
if is_part_of_env(src):
shutil.copy(src, dst)
def is_part_of_env(path):
return not os.path.basename(path).startswith(".")
def read(*path):
"""
Read template content located at `path`.
"""
src = os.path.join(TEMPLATES_ROOT, *path)
with codecs.open(src, encoding='utf-8') as fi:
return fi.read()
def walk_templates(root, target):
"""
Iterate on the template files from `templates/target`.
Yield:
src: template path
dst: destination path inside root
"""
target_root = os.path.join(TEMPLATES_ROOT, target)
for dirpath, _, filenames in os.walk(os.path.join(TEMPLATES_ROOT, target)):
dst_dir = pathjoin(
root, target,
os.path.relpath(dirpath, target_root)
)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
for filename in filenames:
src = os.path.join(dirpath, filename)
dst = os.path.join(dst_dir, filename)
yield src, dst
def data_path(root, *path):
return os.path.join(os.path.abspath(root), "data", *path)
def pathjoin(root, target, *path):
return os.path.join(root, "env", target, *path)

2
tutor/exceptions.py Normal file
View File

@ -0,0 +1,2 @@
class TutorError(Exception):
pass

26
tutor/fmt.py Normal file
View File

@ -0,0 +1,26 @@
import click
def title(text):
indent = 8
separator = "=" * (len(text) + 2 * indent)
message = "{separator}\n{indent}{text}\n{separator}".format(
separator=separator,
indent=" " * indent,
text=text,
)
return click.style(message, fg="green")
def info(text):
return click.style(text, fg="blue")
def error(text):
return click.style(text, fg="red")
def command(text):
return click.style(text, fg="magenta")
def question(text):
return click.style(text, fg="yellow")
def alert(text):
return click.style("⚠️ " + text, fg="yellow", bold=True)

93
tutor/images.py Normal file
View File

@ -0,0 +1,93 @@
import click
from . import env as tutor_env
from . import fmt
from . import opts
from . import utils
@click.group(short_help="Manage docker images")
def images():
pass
option_namespace = click.option("-n", "--namespace", default="regis", show_default=True)
option_version = click.option("-V", "--version", default="hawthorn", show_default=True)
all_images = ["openedx", "forum", "notes", "xqueue", "android"]
argument_image = click.argument(
"image", type=click.Choice(["all"] + all_images),
)
@click.command(
short_help="Generate environment",
help="""Generate the environment files required to build and customise the docker images."""
)
@opts.root
def env(root):
tutor_env.copy_target(root, "build")
click.echo(fmt.info("Environment generated in {}".format(root)))
@click.command(
short_help="Download docker images",
help=("""Download the docker images from hub.docker.com.
The images will come from {namespace}/{image}:{version}.""")
)
@option_namespace
@option_version
@argument_image
def download(namespace, version, image):
for image in image_list(image):
utils.docker('image', 'pull', get_tag(namespace, image, version))
@click.command(
short_help="Build docker images",
help=("""Build the docker images necessary for an Open edX platform.
The images will be tagged as {namespace}/{image}:{version}."""))
@opts.root
@option_namespace
@option_version
@argument_image
@click.option(
"-a", "--build-arg", multiple=True,
help="Set build-time docker ARGS in the form 'myarg=value'. This option may be specified multiple times."
)
def build(root, namespace, version, image, build_arg):
for image in image_list(image):
tag = get_tag(namespace, image, version)
click.echo(fmt.info("Building image {}".format(tag)))
command = [
"build", "-t", tag,
tutor_env.pathjoin(root, "build", image)
]
for arg in build_arg:
command += [
"--build-arg", arg
]
utils.docker(*command)
@click.command(
short_help="Push images to hub.docker.com",
)
@option_namespace
@option_version
@argument_image
def push(namespace, version, image):
for image in image_list(image):
tag = get_tag(namespace, image, version)
click.echo(fmt.info("Pushing image {}".format(tag)))
utils.execute("docker", "push", tag)
def get_tag(namespace, image, version):
name = "openedx" if image == "openedx" else "openedx-{}".format(image)
return "{namespace}{sep}{image}:{version}".format(
namespace=namespace,
sep="/" if namespace else "",
image=name,
version=version,
)
def image_list(image):
return all_images if image == "all" else [image]
images.add_command(env)
images.add_command(download)
images.add_command(build)
images.add_command(push)

186
tutor/k8s.py Normal file
View File

@ -0,0 +1,186 @@
import click
import kubernetes
from . import config as tutor_config
from . import env as tutor_env
from . import exceptions
from . import fmt
from . import opts
from . import ops
from . import utils
@click.group(help="Run Open edX on Kubernetes [BETA FEATURE]")
def k8s():
pass
@click.command(
help="Configure and run Open edX from scratch"
)
@opts.root
def quickstart(root):
click.echo(fmt.title("Interactive platform configuration"))
tutor_config.interactive.callback(root, [])
click.echo(fmt.title("Environment generation"))
env.callback(root)
click.echo(fmt.title("Stopping any existing platform"))
stop.callback(root)
click.echo(fmt.title("Starting the platform"))
start.callback(root)
@click.command(
short_help="Generate environment",
help="Generate the environment files required to run Open edX",
)
@opts.root
def env(root):
config = tutor_config.load(root)
tutor_env.render_target(root, config, "apps")
tutor_env.render_target(root, config, "k8s")
click.echo(fmt.info("Environment generated in {}".format(root)))
@click.command(help="Run all configured Open edX services")
@opts.root
def start(root):
kubectl_no_fail("create", "-f", tutor_env.pathjoin(root, "k8s", "namespace.yml"))
kubectl("create", "configmap", "nginx-config", "--from-file", tutor_env.pathjoin(root, "apps", "nginx"))
kubectl("create", "configmap", "mysql-config", "--from-env-file", tutor_env.pathjoin(root, "apps", "mysql", "auth.env"))
kubectl("create", "configmap", "openedx-settings-lms", "--from-file", tutor_env.pathjoin(root, "apps", "openedx", "settings", "lms"))
kubectl("create", "configmap", "openedx-settings-cms", "--from-file", tutor_env.pathjoin(root, "apps", "openedx", "settings", "cms"))
kubectl("create", "configmap", "openedx-config", "--from-file", tutor_env.pathjoin(root, "apps", "openedx", "config"))
kubectl("create", "-f", tutor_env.pathjoin(root, "k8s", "volumes.yml"))
kubectl("create", "-f", tutor_env.pathjoin(root, "k8s", "ingress.yml"))
kubectl("create", "-f", tutor_env.pathjoin(root, "k8s", "services.yml"))
kubectl("create", "-f", tutor_env.pathjoin(root, "k8s", "deployments.yml"))
@click.command(help="Stop a running platform")
def stop():
kubectl("delete", "deployments,services,ingress,configmaps", "--all")
@click.command(help="Completely delete an existing platform")
@click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation")
def delete(yes):
if not yes:
click.confirm('Are you sure you want to delete the platform? All data will be removed.', abort=True)
kubectl("delete", "namespace", K8s.NAMESPACE)
@click.command(
help="Create databases and run database migrations",
)
@opts.root
def databases(root):
ops.migrate(root, run_bash)
@click.command(help="Create an Open edX user and interactively set their password")
@opts.root
@click.option("--superuser", is_flag=True, help="Make superuser")
@click.option("--staff", is_flag=True, help="Make staff user")
@click.argument("name")
@click.argument("email")
def createuser(root, superuser, staff, name, email):
ops.create_user(root, run_bash, superuser, staff, name, email)
@click.command(help="Import the demo course")
@opts.root
def importdemocourse(root):
ops.import_demo_course(root, run_bash)
@click.command(help="Re-index courses for better searching")
@opts.root
def indexcourses(root):
# Note: this is currently broken with "pymongo.errors.ConnectionFailure: [Errno 111] Connection refused"
# I'm not quite sure the settings are correctly picked up. Which is weird because migrations work very well.
ops.index_courses(root, run_bash)
@click.command(
help="Launch a shell in LMS or CMS",
)
@opts.root
@click.argument("service", type=click.Choice(["lms", "cms"]))
def shell(root, service):
K8s().execute(service, "bash")
@click.command(help="Create a Kubernetesadmin user")
@opts.root
def adminuser(root):
utils.kubectl("create", "-f", tutor_env.pathjoin(root, "k8s", "adminuser.yml"))
@click.command(help="Print the Kubernetes admin user token")
@opts.root
def admintoken(root):
click.echo(K8s().admin_token())
def kubectl(*command):
"""
Run kubectl commands in the right namespace. Also, errors are completely
ignored, to avoid stopping on "AlreadyExists" errors.
"""
args = list(command)
args += [
"--namespace", K8s.NAMESPACE
]
kubectl_no_fail(*args)
def kubectl_no_fail(*command):
"""
Run kubectl commands and ignore exceptions, to avoid stopping on
"AlreadyExists" errors.
"""
try:
utils.kubectl(*command)
except exceptions.TutorError:
pass
class K8s:
CLIENT = None
NAMESPACE = "openedx"
def __init__(self):
pass
@property
def client(self):
if self.CLIENT is None:
kubernetes.config.load_kube_config()
self.CLIENT = kubernetes.client.CoreV1Api()
return self.CLIENT
def pod_name(self, app):
selector = "app=" + app
try:
return self.client.list_namespaced_pod("openedx", label_selector=selector).items[0].metadata.name
except IndexError:
raise exceptions.TutorError("Pod with app {} does not exist. Make sure that the pod is running.")
def admin_token(self):
# Note: this is a HORRIBLE way of looking for a secret
try:
secret = [
s for s in self.client.list_namespaced_secret("kube-system").items if s.metadata.name.startswith("admin-user-token")
][0]
except IndexError:
raise exceptions.TutorError("Secret 'admin-user-token'. Make sure that admin user was created.")
return self.client.read_namespaced_secret(secret.metadata.name, "kube-system").data["token"]
def execute(self, app, *command):
podname = self.pod_name(app)
kubectl_no_fail("exec", "--namespace", self.NAMESPACE, "-it", podname, "--", *command)
def run_bash(root, service, command):
K8s().execute(service, "bash", "-e", "-c", command)
k8s.add_command(quickstart)
k8s.add_command(env)
k8s.add_command(start)
k8s.add_command(stop)
k8s.add_command(delete)
k8s.add_command(databases)
k8s.add_command(createuser)
k8s.add_command(importdemocourse)
k8s.add_command(indexcourses)
k8s.add_command(shell)
k8s.add_command(adminuser)
k8s.add_command(admintoken)

261
tutor/local.py Normal file
View File

@ -0,0 +1,261 @@
import os
from time import sleep
import click
from . import config as tutor_config
from . import fmt
from . import opts
from . import scripts
from . import utils
from . import env as tutor_env
from . import ops
@click.group(
short_help="Run Open edX locally",
help="Run Open edX platform locally, with docker-compose.",
)
def local():
pass
@click.command(
help="Configure and run Open edX from scratch"
)
@opts.root
def quickstart(root):
click.echo(fmt.title("Interactive platform configuration"))
tutor_config.interactive.callback(root, [])
click.echo(fmt.title("Environment generation"))
env.callback(root)
click.echo(fmt.title("Stopping any existing platform"))
stop.callback(root)
click.echo(fmt.title("Docker image updates"))
pullimages.callback(root)
click.echo(fmt.title("Database creation and migrations"))
databases.callback(root)
click.echo(fmt.title("HTTPS certificates generation"))
https_create.callback(root)
click.echo(fmt.title("Starting the platform in detached mode"))
start.callback(root, True)
@click.command(
short_help="Generate environment",
help="Generate the environment files required to run Open edX",
)
@opts.root
def env(root):
config = tutor_config.load(root)
tutor_env.render_target(root, config, "apps")
tutor_env.render_target(root, config, "local")
click.echo(fmt.info("Environment generated in {}".format(root)))
@click.command(
help="Update docker images",
)
@opts.root
def pullimages(root):
docker_compose(root, "pull")
@click.command(
help="Run all configured Open edX services",
)
@opts.root
@click.option("-d", "--detach", is_flag=True, help="Start in daemon mode")
def start(root, detach):
command = ["up"]
if detach:
command.append("-d")
docker_compose(root, *command)
if detach:
click.echo(fmt.info("The Open edX platform is now running in detached mode"))
config = tutor_config.load(root)
http = "https" if config["ACTIVATE_HTTPS"] else "http"
urls = []
if not config["ACTIVATE_HTTPS"]:
urls += [
"http://localhost",
"http://studio.localhost",
]
urls.append("{http}://{lms_host}".format(http=http, lms_host=config["LMS_HOST"]))
urls.append("{http}://{cms_host}".format(http=http, cms_host=config["CMS_HOST"]))
click.echo(fmt.info("""Your Open edX platform is ready and can be accessed at the following urls:
{}""".format("\n ".join(urls))))
@click.command(help="Stop a running platform",)
@opts.root
def stop(root):
docker_compose(root, "rm", "--stop", "--force")
@click.command(
help="""Restart some components from a running platform.
You may specify 'openedx' to restart the lms, cms and workers, or 'all' to
restart all services.""",
)
@opts.root
@click.argument('service')
def restart(root, service):
command = ["restart"]
if service == "openedx":
command += ["lms", "cms", "lms_worker", "cms_worker"]
elif service != "all":
command += [service]
docker_compose(root, *command)
@click.command(
help="Run a command in one of the containers",
context_settings={"ignore_unknown_options": True},
)
@opts.root
@click.argument("service")
@click.argument("command", default=None, required=False)
@click.argument("args", nargs=-1, required=False)
def run(root, service, command, args):
run_command = [
"run",
"--rm",
service
]
if command:
run_command.append(command)
if args:
run_command += args
docker_compose(root, *run_command)
@click.command(
help="Create databases and run database migrations",
)
@opts.root
def databases(root):
mysql_data_path = tutor_env.data_path(root, "mysql", "mysql")
if not os.path.exists(mysql_data_path):
click.echo(fmt.info("Initializing MySQL database..."))
docker_compose(root, "up", "-d", "mysql")
while not os.path.exists(mysql_data_path):
click.echo(fmt.info(" waiting for creation of {}".format(mysql_data_path)))
sleep(4)
click.echo(fmt.info("MySQL database initialized"))
docker_compose(root, "stop", "mysql")
ops.migrate(root, run_bash)
@click.group(help="Manage https certificates")
def https():
pass
@click.command(help="Create https certificates", name="create")
@opts.root
def https_create(root):
"""
Note: there are a couple issues with https certificate generation.
1. Certificates are generated and renewed by using port 80, which is not necessarily open.
a. It may be occupied by the nginx container
b. It may be occupied by an external web server
2. On certificate renewal, nginx is not reloaded
"""
config = tutor_config.load(root)
if not config['ACTIVATE_HTTPS']:
click.echo(fmt.info("HTTPS is not activated: certificate generation skipped"))
return
utils.docker_run(
"--volume", "{}:/etc/letsencrypt/".format(tutor_env.data_path(root, "letsencrypt")),
"-p", "80:80",
"--entrypoint=sh",
"certbot/certbot:latest",
"-c", tutor_env.render_str(scripts.https_certificates_create, config),
)
@click.command(help="Renew https certificates", name="renew")
@opts.root
def https_renew(root):
config = tutor_config.load(root)
if not config['ACTIVATE_HTTPS']:
click.echo(fmt.info("HTTPS is not activated: certificate renewal skipped"))
return
docker_run = [
"--volume", "{}:/etc/letsencrypt/".format(tutor_env.data_path(root, "letsencrypt")),
"-p", "80:80",
"certbot/certbot:latest", "renew"
]
utils.docker_run(*docker_run)
@click.command(help="View output from containers")
@opts.root
@click.option("-f", "--follow", is_flag=True, help="Follow log output")
@click.option("--tail", type=int, help="Number of lines to show from each container")
@click.argument("service", nargs=-1, required=False)
def logs(root, follow, tail, service):
command = ["logs"]
if follow:
command += ["--follow"]
if tail is not None:
command += ["--tail", str(tail)]
command += service
docker_compose(root, *command)
@click.command(help="Create an Open edX user and interactively set their password")
@opts.root
@click.option("--superuser", is_flag=True, help="Make superuser")
@click.option("--staff", is_flag=True, help="Make staff user")
@click.argument("name")
@click.argument("email")
def createuser(root, superuser, staff, name, email):
ops.create_user(root, run_bash, superuser, staff, name, email)
@click.command(help="Import the demo course")
@opts.root
def importdemocourse(root):
ops.import_demo_course(root, run_bash)
@click.command(help="Re-index courses for better searching")
@opts.root
def indexcourses(root):
ops.index_courses(root, run_bash)
@click.command(
help="Run Portainer (https://portainer.io), a UI for container supervision",
short_help="Run Portainer, a UI for container supervision",
)
@opts.root
@click.option("-p", "--port", type=int, default=9000, show_default=True, help="Bind port")
def portainer(root, port):
docker_run = [
"--volume=/var/run/docker.sock:/var/run/docker.sock",
"--volume={}:/data".format(tutor_env.data_path(root, "portainer")),
"-p", "{port}:{port}".format(port=port),
"portainer/portainer:latest",
"--bind=:{}".format(port),
]
click.echo(fmt.info("View the Portainer UI at http://localhost:{port}".format(port=port)))
utils.docker_run(*docker_run)
def run_bash(root, service, command):
docker_compose(root, "run", "--rm", service, "bash", "-e", "-c", command)
def docker_compose(root, *command):
return utils.docker_compose(
"-f", tutor_env.pathjoin(root, "local", "docker-compose.yml"),
"--project-name", "tutor_local",
*command
)
https.add_command(https_create)
https.add_command(https_renew)
local.add_command(quickstart)
local.add_command(env)
local.add_command(pullimages)
local.add_command(start)
local.add_command(stop)
local.add_command(restart)
local.add_command(run)
local.add_command(databases)
local.add_command(https)
local.add_command(logs)
local.add_command(createuser)
local.add_command(importdemocourse)
local.add_command(indexcourses)
local.add_command(portainer)

51
tutor/ops.py Normal file
View File

@ -0,0 +1,51 @@
import click
from . import config as tutor_config
from . import env
from . import fmt
from . import scripts
# TODO merge this with scripts.py
def migrate(root, run_func):
config = tutor_config.load(root)
click.echo(fmt.info("Creating lms/cms databases..."))
run_template(config, root, "lms", scripts.create_databases, run_func)
click.echo(fmt.info("Running lms migrations..."))
run_template(config, root, "lms", scripts.migrate_lms, run_func)
click.echo(fmt.info("Running cms migrations..."))
run_template(config, root, "cms", scripts.migrate_cms, run_func)
click.echo(fmt.info("Running forum migrations..."))
run_template(config, root, "forum", scripts.migrate_forum, run_func)
if config["ACTIVATE_NOTES"]:
click.echo(fmt.info("Running notes migrations..."))
run_template(config, root, "notes", scripts.migrate_notes, run_func)
if config["ACTIVATE_XQUEUE"]:
click.echo(fmt.info("Running xqueue migrations..."))
run_template(config, root, "xqueue", scripts.migrate_xqueue, run_func)
click.echo(fmt.info("Creating oauth2 users..."))
run_template(config, root, "lms", scripts.oauth2, run_func)
click.echo(fmt.info("Databases ready."))
def create_user(root, run_func, superuser, staff, name, email):
config = {
"OPTS": "",
"USERNAME": name,
"EMAIL": email,
}
if superuser:
config["OPTS"] += " --superuser"
if staff:
config["OPTS"] += " --staff"
run_template(config, root, "lms", scripts.create_user, run_func)
def import_demo_course(root, run_func):
run_template({}, root, "lms", scripts.import_demo_course, run_func)
def index_courses(root, run_func):
run_template({}, root, "cms", scripts.index_courses, run_func)
def run_template(config, root, service, template, run_func):
command = env.render_str(template, config).strip()
if command:
run_func(root, service, command)

43
tutor/opts.py Normal file
View File

@ -0,0 +1,43 @@
import appdirs
import click
root = click.option(
"-r", "--root",
envvar="TUTOR_ROOT",
default=appdirs.user_data_dir(appname="tutor"), show_default=True,
type=click.Path(resolve_path=True),
help="Root project directory (environment variable: TUTOR_ROOT)"
)
edx_platform_path = click.option(
"-P", "--edx-platform-path",
envvar="TUTOR_EDX_PLATFORM_PATH",
type=click.Path(exists=True, dir_okay=True, resolve_path=True),
help="Mount a local edx-platform from the host (environment variable: TUTOR_EDX_PLATFORM_PATH)"
)
edx_platform_settings = click.option(
"-S", "--edx-platform-settings",
envvar="TUTOR_EDX_PLATFORM_SETTINGS",
default="tutor.development",
help="Mount a local edx-platform from the host (environment variable: TUTOR_EDX_PLATFORM_PATH)"
)
class YamlParamType(click.ParamType):
name = "yaml"
def convert(self, value, param, ctx):
try:
k, v = value.split("=")
except ValueError:
self.fail("'{}' is not of the form 'key=value'.".format(value), param, ctx)
if v.isdigit():
v = int(v)
elif v in ["true", "false"]:
v = (v == "true")
return (k, v)
key_value = click.option(
"-s", type=YamlParamType(), multiple=True, metavar="KEY=VAL",
help="Set a configuration value (can be used multiple times)"
)

41
tutor/scripts.py Normal file
View File

@ -0,0 +1,41 @@
create_databases = """dockerize -wait tcp://mysql:3306 -timeout 20s
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS {{ MYSQL_DATABASE }};'
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'GRANT ALL ON {{ MYSQL_DATABASE }}.* TO "{{ MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ MYSQL_PASSWORD }}";'
{% if ACTIVATE_NOTES %}
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};'
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";'
{% endif %}
{% if ACTIVATE_XQUEUE %}
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};'
mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";'
{% endif %}
"""
migrate_lms = "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py lms migrate"
migrate_cms = "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py cms migrate"
migrate_forum = "bundle exec rake search:initialize && bundle exec rake search:rebuild_index"
migrate_notes = "./manage.py migrate"
migrate_xqueue = "./manage.py migrate"
oauth2 = """{% if ACTIVATE_NOTES %}
./manage.py lms manage_user notes notes@{{ LMS_HOST }} --staff --superuser
./manage.py lms create_oauth2_client \
"http://notes.openedx:8000" \
"http://notes.openedx:8000/complete/edx-oidc/" \
confidential \
--client_name edx-notes --client_id notes --client_secret {{ NOTES_OAUTH2_SECRET }} \
--trusted --logout_uri "http://notes.openedx:8000/logout/" --username notes
{% endif %}"""
https_certificates_create = """certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ LMS_HOST }} -d {{ CMS_HOST }} -d preview.{{ LMS_HOST }}
{% if ACTIVATE_NOTES %}
certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d notes.{{ LMS_HOST }}
{% endif %}"""
create_user = """./manage.py lms --settings=tutor.production manage_user {{ OPTS }} {{ USERNAME }} {{ EMAIL }}
./manage.py lms --settings=tutor.production changepassword {{ USERNAME }}"""
import_demo_course = """git clone https://github.com/edx/edx-demo-course --branch open-release/hawthorn.2 --depth 1 ../edx-demo-course
python ./manage.py cms --settings=tutor.production import ../data ../edx-demo-course"""
index_courses = "./manage.py cms --settings=tutor.production reindex_course --all --setup"

0
android/templates/edx.properties → tutor/templates/android/edx.properties
View File

0
android/templates/gradle.properties → tutor/templates/android/gradle.properties
View File

0
android/templates/tutor.yaml → tutor/templates/android/tutor.yaml
View File

0
deploy/templates/mysql/auth.env → tutor/templates/apps/mysql/auth.env
View File

0
deploy/templates/nginx/cms.conf → tutor/templates/apps/nginx/cms.conf
View File

0
deploy/templates/nginx/extra.conf → tutor/templates/apps/nginx/extra.conf
View File

0
deploy/templates/nginx/lms.conf → tutor/templates/apps/nginx/lms.conf
View File

0
deploy/templates/nginx/tutor.conf → tutor/templates/apps/nginx/tutor.conf
View File

0
deploy/templates/notes/tutor.py → tutor/templates/apps/notes/settings/tutor.py
View File

0
deploy/templates/openedx/config/lms.auth.json → tutor/templates/apps/openedx/config/cms.auth.json
View File

0
deploy/templates/openedx/config/cms.env.json → tutor/templates/apps/openedx/config/cms.env.json
View File

2
deploy/templates/openedx/config/cms.auth.json → tutor/templates/apps/openedx/config/lms.auth.json
View File

@ -17,7 +17,7 @@
}
},
"DOC_STORE_CONFIG": {
"db": "openedx",
"db": "{{ MONGODB_DATABASE }}",
"host": "mongodb"
},
"DATABASES": {

0
deploy/templates/openedx/config/lms.env.json → tutor/templates/apps/openedx/config/lms.env.json
View File

0
build/openedx/settings/lms/__init__.py → tutor/templates/apps/openedx/settings/cms/__init__.py
View File

0
deploy/templates/openedx/settings/cms/development.py → tutor/templates/apps/openedx/settings/cms/development.py
View File

Some files were not shown because too many files have changed in this diff Show More