diff --git a/.env b/.env new file mode 100644 index 000000000..04db84fb5 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +REGISTRY=ghcr.io +VERSION=latest diff --git a/.github/workflows/core_containers.yml b/.github/workflows/core_containers.yml new file mode 100644 index 000000000..7fdce19c5 --- /dev/null +++ b/.github/workflows/core_containers.yml @@ -0,0 +1,20 @@ +name: Build and push core containers + +on: + push: + branches: [master] + + workflow_dispatch: + +jobs: + containers_build_and_push: + uses: INGInious/.github/.github/workflows/containers.yml@6506916a602fc6bf47d6e8f34586732d3c8e1502 + with: + working-directory: base-containers + context-path: context.yml + compose-path: compose.yml + registry: ghcr.io + container_type: core + secrets: + GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }} + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} diff --git a/README.rst b/README.rst index 6580ea00a..91e520cbe 100644 --- a/README.rst +++ b/README.rst @@ -27,6 +27,50 @@ INGInious can be used as an external grader for EDX. The course `Paradigms of Co .. _Docker: https://www.docker.com/ .. _Paradigms of Computer Programming - Fundamentals: https://www.edx.org/course/louvainx/louvainx-louv1-1x-paradigms-computer-2751 +How to install? +--------------- + +Simply run: + +.. code-block:: + + $ docker compose up --build + +> Note that you can override the registry and containers version by setting the `REGISTRY` and +> `VERSION` environment variables. + +And access http://localhost:9000 in your browser. + +*The default login and password are* ``superadmin``. + +The ``--build`` argument is optional, use it if you want to rebuild locally the core containers. +If you want to simply pull them from the project's registry, this argument is not required. + +Docker-compose will create a ``tasks`` folder if it does not exist already. + +You can then add new courses to your fresh INGInious instance by installing them in the ``tasks`` folder. + +For example, the INGInious tutorial course is installed with the following commands: + +.. code-block:: + + $ git clone https://github.com/UCL-INGI/INGInious-demo-tasks.git + $ mv INGInious-demo-tasks/tutorial tasks/ + +*If you encounter permission errors, you should run the following command:* + +.. code-block:: + + $ sudo chown -R : tasks + +*This can happen when the tasks directory is created by docker-compose.* + +Note that the `configuration.deploy.yaml` file provided is a sample configuration, the secret key **must** be changed by administrators in production deployments. + +.. _Manual installation: https://docs.inginious.org/en/latest/admin_doc/install_doc/installation.html + +`Manual installation`_ is also possible with pip. + Documentation ------------- diff --git a/configuration.deploy.yaml b/configuration.deploy.yaml new file mode 100644 index 000000000..690760006 --- /dev/null +++ b/configuration.deploy.yaml @@ -0,0 +1,16 @@ +backend: tcp://backend:2000 +backup_directory: /inginious/backups +local-config: {} +mongo_opt: + database: INGInious + host: db +plugins: [] +session_parameters: + ignore_change_ip: false + secret_key: 96f4628a2e4d0ce26b6352713f8ac8359b838f4800f1972790622899664bd0f4 + secure: false + timeout: 86400 +superadmins: +- superadmin +tasks_directory: /inginious/tasks +use_minified_js: true diff --git a/deploy/agent-docker.containerfile b/deploy/agent-docker.containerfile new file mode 100644 index 000000000..abe1a78de --- /dev/null +++ b/deploy/agent-docker.containerfile @@ -0,0 +1,16 @@ +ARG VERSION=latest +ARG REGISTRY + +FROM ${REGISTRY}/inginious/core-base:${VERSION} + +COPY inginious/agent/__init__.py inginious/agent/ +COPY inginious/agent/docker_agent/ inginious/agent/docker_agent/ +COPY inginious-agent-docker . + +RUN dnf install -y gcc python39-devel + +# See https://github.com/pypa/setuptools_scm/#usage-from-docker +RUN --mount=source=.git,target=.git,type=bind \ + pip3 install --no-cache-dir -e . + +CMD ["sh", "-c", "python3 inginious-agent-docker ${BACKEND} --tmpdir=/tmp/agent_data/"] diff --git a/deploy/agent-mcq.containerfile b/deploy/agent-mcq.containerfile new file mode 100644 index 000000000..292e3de85 --- /dev/null +++ b/deploy/agent-mcq.containerfile @@ -0,0 +1,14 @@ +ARG VERSION=latest +ARG REGISTRY + +FROM ${REGISTRY}/inginious/core-base:${VERSION} + +COPY inginious/agent/__init__.py inginious/agent/ +COPY inginious/agent/mcq_agent/ inginious/agent/mcq_agent/ +COPY inginious-agent-mcq . + +# See https://github.com/pypa/setuptools_scm/#usage-from-docker +RUN --mount=source=.git,target=.git,type=bind \ + pip3 install --no-cache-dir -e . + +CMD ["sh", "-c", "python3 inginious-agent-mcq ${BACKEND}"] diff --git a/deploy/backend.containerfile b/deploy/backend.containerfile new file mode 100644 index 000000000..0d15fa9f5 --- /dev/null +++ b/deploy/backend.containerfile @@ -0,0 +1,13 @@ +ARG VERSION=latest +ARG REGISTRY + +FROM ${REGISTRY}/inginious/core-base:${VERSION} + +COPY inginious/backend/ inginious/backend/ +COPY inginious-backend . + +# See https://github.com/pypa/setuptools_scm/#usage-from-docker +RUN --mount=source=.git,target=.git,type=bind \ + pip3 install --no-cache-dir -e . + +CMD ["sh", "-c", "python3 inginious-backend ${AGENT} ${CLIENT}"] diff --git a/deploy/db_setup.py b/deploy/db_setup.py new file mode 100644 index 000000000..04f424ad8 --- /dev/null +++ b/deploy/db_setup.py @@ -0,0 +1,50 @@ +from pymongo import MongoClient +from gridfs import GridFS + +from inginious.frontend.user_manager import UserManager + +def try_mongodb_opts(host="localhost", database_name='INGInious'): + """ Try MongoDB configuration """ + try: + mongo_client = MongoClient(host=host) + # Effective access only occurs when we call a method on the connexion + mongo_version = str(mongo_client.server_info()['version']) + print("Found mongodb server running version %s on %s." % (mongo_version, host)) + except Exception as e: + print("Cannot connect to MongoDB on host %s: %s" % (host, str(e))) + return None + + try: + database = mongo_client[database_name] + # Effective access only occurs when we call a method on the database. + database.list_collection_names() + except Exception as e: + print("Cannot access database %s: %s" % (database_name, str(e))) + return None + + try: + # Effective access only occurs when we call a method on the gridfs object. + GridFS(database).find_one() + except Exception as e: + print("Cannot access gridfs %s: %s" % (database_name, str(e))) + return None + + return database + +if __name__ == '__main__': + username = "superadmin" + realname = "INGInious superadmin" + email = "superadmin@inginious.org" + password = "superadmin" + + print('Initial DB setup.') + + database = try_mongodb_opts('db') + + database.users.insert_one({"username": username, + "realname": realname, + "email": email, + "password": UserManager.hash_password(password), + "bindings": {}, + "language": "en"}) + print('Superadmin user added!') diff --git a/deploy/frontend.containerfile b/deploy/frontend.containerfile new file mode 100644 index 000000000..6d9edad33 --- /dev/null +++ b/deploy/frontend.containerfile @@ -0,0 +1,18 @@ +ARG VERSION=latest +ARG REGISTRY + +FROM ${REGISTRY}/inginious/core-base:${VERSION} + +COPY inginious/frontend inginious/frontend/ +COPY inginious/client inginious/client/ +COPY inginious-webapp . + +RUN dnf install -y git gcc python39-devel + +# See https://github.com/pypa/setuptools_scm/#usage-from-docker +RUN --mount=source=.git,target=.git,type=bind \ + pip3 install --no-cache-dir -e . + +COPY deploy/db_setup.py /tmp/db_setup.py + +CMD ["sh", "-c", "python3 /tmp/db_setup.py; python3 inginious-webapp"] diff --git a/deploy/inginious-base.containerfile b/deploy/inginious-base.containerfile new file mode 100644 index 000000000..1eaf7b485 --- /dev/null +++ b/deploy/inginious-base.containerfile @@ -0,0 +1,16 @@ +# Base core container. This is not a service per-se but is used as a common layer for the core services. +FROM rockylinux:8 + +ENV INGINIOUS_COMPOSE=1 + +RUN dnf update -y && dnf install -y python39 python39-pip git +RUN pip3 install importlib-metadata + +WORKDIR /inginious +COPY setup.py README.rst ./ +COPY inginious/common/ inginious/common/ +COPY inginious/__init__.py inginious/ + +# See https://github.com/pypa/setuptools_scm/#usage-from-docker +RUN --mount=source=.git,target=.git,type=bind \ + pip3 install --no-cache-dir -e . diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..c0fb7ec9b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,100 @@ +services: + + base: + image: ${REGISTRY}/inginious/core-base:${VERSION} + build: + dockerfile: deploy/inginious-base.containerfile + args: + - VERSION=${VERSION} + - REGISTRY=${REGISTRY} + command: /bin/true + + db: + image: mongo:6.0.2 + networks: + - inginious + + backend: + image: ${REGISTRY}/inginious/core-backend:${VERSION} + depends_on: + - base + build: + dockerfile: deploy/backend.containerfile + args: + - VERSION=${VERSION} + - REGISTRY=${REGISTRY} + environment: + AGENT: "tcp://0.0.0.0:2001" + CLIENT: "tcp://0.0.0.0:2000" + networks: + - inginious + + agent-docker: + image: ${REGISTRY}/inginious/core-agent_docker:${VERSION} + depends_on: + - backend + deploy: + replicas: 1 + build: + dockerfile: deploy/agent-docker.containerfile + args: + - VERSION=${VERSION} + - REGISTRY=${REGISTRY} + environment: + BACKEND: "tcp://backend:2001" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + # See https://github.com/UCL-INGI/INGInious/issues/352 + - ./tasks/:/inginious/tasks + - ./backups/:/inginious/backups + # See https://github.com/UCL-INGI/INGInious/issues/799 + - /tmp/agent_data/:/tmp/agent_data/ + networks: + - inginious + + agent-mcq: + image: ${REGISTRY}/inginious/core-agent_mcq:${VERSION} + depends_on: + - backend + deploy: + replicas: 1 + build: + dockerfile: deploy/agent-mcq.containerfile + args: + - VERSION=${VERSION} + - REGISTRY=${REGISTRY} + environment: + BACKEND: "tcp://backend:2001" + volumes: + # See https://github.com/UCL-INGI/INGInious/issues/352 + - ./tasks/:/inginious/tasks + - ./backups/:/inginious/backups + # See https://github.com/UCL-INGI/INGInious/issues/799 + - /tmp/agent_data/:/tmp/agent_data/ + networks: + - inginious + + frontend: + image: ${REGISTRY}/inginious/core-frontend:${VERSION} + build: + dockerfile: deploy/frontend.containerfile + args: + - VERSION=${VERSION} + - REGISTRY=${REGISTRY} + depends_on: + - backend + - agent-docker + - agent-mcq + environment: + - INGINIOUS_WEBAPP_HOST=0.0.0.0 + volumes: + - ./configuration.deploy.yaml:/inginious/configuration.yaml + - ./tasks/:/inginious/tasks + - ./backups/:/inginious/backups + ports: + - 9000:8080 + networks: + - inginious + +networks: + inginious: diff --git a/inginious/frontend/arch_helper.py b/inginious/frontend/arch_helper.py index 6015fdf0a..295366d54 100644 --- a/inginious/frontend/arch_helper.py +++ b/inginious/frontend/arch_helper.py @@ -10,9 +10,6 @@ from zmq.asyncio import ZMQEventLoop, Context -from inginious.agent.docker_agent import DockerAgent -from inginious.agent.mcq_agent import MCQAgent -from inginious.backend.backend import Backend from inginious.client.client import Client def start_asyncio_and_zmq(debug_asyncio=False): @@ -88,6 +85,13 @@ def create_arch(configuration, tasks_fs, context, course_factory): else: debug_ports = range(64100, 64111) + """ Those imports are required in pip-based installation but are not available in + docker-compose based ones. """ + + from inginious.agent.docker_agent import DockerAgent + from inginious.agent.mcq_agent import MCQAgent + from inginious.backend.backend import Backend + client = Client(context, "inproc://backend_client") backend = Backend(context, "inproc://backend_agent", "inproc://backend_client") agent_docker = DockerAgent(context, "inproc://backend_agent", "Docker - Local agent", concurrency, tasks_fs, debug_host, debug_ports, tmp_dir, ssh_allowed=True) diff --git a/setup.py b/setup.py index d16428ebb..6b3dea6de 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,19 @@ else: install_requires += ["sh>=1.11"] +scripts = [] if os.environ.get("INGINIOUS_COMPOSE") else [ + 'inginious-agent-docker', + 'inginious-agent-mcq', + 'inginious-backend', + 'inginious-webapp', + 'inginious-webdav', + 'inginious-install', + 'inginious-autotest', + 'utils/sync/inginious-synchronize', + 'utils/container_update/inginious-container-update', + 'utils/database_updater/inginious-database-update' +] + # Setup setup( name="INGInious", @@ -69,20 +82,7 @@ "test": test_requires, "doc": test_requires + doc_requires }, - - scripts=[ - 'inginious-agent-docker', - 'inginious-agent-mcq', - 'inginious-backend', - 'inginious-webapp', - 'inginious-webdav', - 'inginious-install', - 'inginious-autotest', - 'utils/sync/inginious-synchronize', - 'utils/container_update/inginious-container-update', - 'utils/database_updater/inginious-database-update' - ], - + scripts=scripts, include_package_data=True, test_suite='nose.collector', author="INGInious contributors",