From 7f3a9968effb36033aa280c443cb4bb9f020d2d7 Mon Sep 17 00:00:00 2001 From: Opliko Date: Sun, 12 Nov 2023 19:38:00 +0100 Subject: [PATCH] feat: docker improvements (#12031) * use yarn and debian slim build * feat: update Dockerfile to use multistage builds * Create main.yml * remove some useless things from docker context and assume yarn by default * remove all dotfiles in docker context * no need for extra build tools, complain to the module author if there is no alpine build (cherry picked from commit 90516a3c8399e74c38be7115edb39411ba0d86b9) * specify the config file location instead of creating it (cherry picked from commit 38e4295d70682f1049fe671ade96eeccd669d908) * set explicit config path (cherry picked from commit 8dcc6f249d099cb8939a95511ec13702491958bc) * fix docker-compose example to use the exposed volumes * dockerfile: upgrade alpine to 3.16 * dockerignore: add more ignorable entries * docker-compose: change the way the docker startup process works * install: pass config path to child process as well Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * setup: move config file resolution up before setup This fixes issue with different config file location, which will otherwise default on 'config.json', which means the config save won't save to the file we specified Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * docker-entrypoint: don't fix CONFIG_DIR location but fix default location Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * docker-entrypoint: handle missing config file logic Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * README: add simple notice on how to use it Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * add missing semicolons Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * docker-compose: remove multi override, use one big profile instead However, Docker Compose doesn't support profile-based dependency and this would probably means we have less guarantee about the liveness of the database. But since this is just a sample configuration it should be fine Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * workflows: remove main.yml, add platforms to buildx matrix in docker.yml Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * workflows: set docker buildx to build for amd64 and arm64 only Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * docker-entrypoint: don't force build everytime before start Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * docker-entrypoint: implement init verb This would allow you to change between "setup" (automated setup using environmental variables which is the current preferred way to run containerized NodeBB) or "install" (web install that guides user to fill in connection information, which is similar to WordPress) Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * README: mention caveat with MongoDB Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * README: add Docker section placeholder for doc migration Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * docker-entrypoint: add SETUP variable support Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * docker-compose: add force flag to ln on setup Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * docker-compose: fix permission issue; docker-compose: fast exit if still no permission on config dir Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> * fix: remove redundant FROM * docs: remove docker stuff (in favour of docs entry, nodebb/docs#78) but add link to cloud install docs * fix: correctly check if directory is writable * fix: ignore .docker directory * fix: multi-arch docker builds and chown performance * chore: bump database image versions * fix: move from alpine to slim image * fix: use omit=dev instead of only=prod * feat: move entrypoint to install directory * feat: initialize mongodb user * feat: use separate rebuild stage * fix: disable eslint for mongodb script * fix: remove node_modules bind mount bind mounts don't save data from container, resulting in a LOONG startup * feat: prepopulate database defaults for installation * feat: enable persistence in redis container * docs: add some comments to the compose file --------- Signed-off-by: steve <29133953+stevefan1999-personal@users.noreply.github.com> Co-authored-by: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Co-authored-by: Steve Fan <19037626d@connect.polyu.hk> Co-authored-by: Julian Lam --- .dockerignore | 10 +++++ .eslintignore | 1 + .github/workflows/docker.yml | 10 ++--- .gitignore | 4 +- Dockerfile | 46 ++++++++++++--------- README.md | 4 +- docker-compose.yml | 63 +++++++++++++++++++++-------- install/docker/entrypoint.sh | 46 +++++++++++++++++++++ install/docker/mongodb-user-init.js | 1 + install/docker/setup.json | 21 ++++++++++ install/web.js | 2 + src/cli/setup.js | 9 +++-- src/install.js | 2 + 13 files changed, 175 insertions(+), 44 deletions(-) create mode 100644 .dockerignore create mode 100755 install/docker/entrypoint.sh create mode 100644 install/docker/mongodb-user-init.js create mode 100644 install/docker/setup.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..46ab37b3ab --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.* +logs +test +node_modules +commitlint.config.js +nodebb.bat +renovate.json +*.yml +*.md +Dockerfile diff --git a/.eslintignore b/.eslintignore index b7a6ad79cd..b304ee19d8 100644 --- a/.eslintignore +++ b/.eslintignore @@ -18,3 +18,4 @@ logs/ .eslintrc test/files *.min.js +install/docker/ \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d27f66173c..16fc4a0de0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -52,10 +52,10 @@ jobs: - name: Build and push Docker images uses: docker/build-push-action@v5 with: - context: . - file: ./Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - platforms: linux/amd64,linux/arm64,linux/arm/v7 cache-from: type=gha cache-to: type=gha,mode=max + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64,linux/arm/v7 + push: true + tags: ${{ steps.meta.outputs.tags }} diff --git a/.gitignore b/.gitignore index 23e38016c2..887ef337b0 100644 --- a/.gitignore +++ b/.gitignore @@ -69,4 +69,6 @@ package-lock.json /package.json *.mongodb link-plugins.sh -test.sh \ No newline at end of file +test.sh + +.docker/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d7277a3389..14f61d3ada 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,29 +13,39 @@ USER node RUN npm install --omit=dev +FROM node:lts as rebuild -FROM node:lts +ARG BUILDPLATFORM +ARG TARGETPLATFORM -RUN mkdir -p /usr/src/app && \ - chown -R node:node /usr/src/app -WORKDIR /usr/src/app +RUN mkdir -p /usr/src/build && \ + chown -R node:node /usr/src/build + +COPY --from=npm /usr/src/build /usr/src/build + +RUN if [ $BUILDPLATFORM != $TARGETPLATFORM ]; then \ + npm rebuild && \ + npm cache clean --force; fi + +FROM node:lts-slim as run ARG NODE_ENV -ENV NODE_ENV $NODE_ENV - -COPY --chown=node:node --from=npm /usr/src/build /usr/src/app - -USER node - -RUN npm rebuild && \ - npm cache clean --force - -COPY --chown=node:node . /usr/src/app - -ENV NODE_ENV=production \ +ENV NODE_ENV=$NODE_ENV \ daemon=false \ silent=false -EXPOSE 4567 +RUN mkdir -p /usr/src/app && \ + chown -R node:node /usr/src/app -CMD test -n "${SETUP}" && ./nodebb setup || node ./nodebb build; node ./nodebb start +COPY --chown=node:node --from=rebuild /usr/src/build /usr/src/app + + +WORKDIR /usr/src/app + +USER node + +COPY --chown=node:node . /usr/src/app + +EXPOSE 4567 +VOLUME ["/usr/src/app/node_modules", "/usr/src/app/build", "/usr/src/app/public/uploads", "/opt/config"] +ENTRYPOINT ["./install/docker/entrypoint.sh"] \ No newline at end of file diff --git a/README.md b/README.md index a45ba24ab8..307d90cca2 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,8 @@ NodeBB requires the following software to be installed: ## Installation -[Please refer to platform-specific installation documentation](https://docs.nodebb.org/installing/os) +[Please refer to platform-specific installation documentation](https://docs.nodebb.org/installing/os). +If installing via the cloud (or using Docker), [please see cloud-based installation documentation](https://docs.nodebb.org/installing/cloud/). ## Securing NodeBB @@ -59,6 +60,7 @@ It is important to ensure that your NodeBB and database servers are secured. Bea 2. Use `iptables` to secure your server from unintended open ports. In Ubuntu, `ufw` provides a friendlier interface to working with `iptables`. * e.g. If your NodeBB is proxied, no ports should be open except 80 (and possibly 22, for SSH access) + ## Upgrading NodeBB Detailed upgrade instructions are listed in [Upgrading NodeBB](https://docs.nodebb.org/configuring/upgrade/) diff --git a/docker-compose.yml b/docker-compose.yml index 5e382f47f9..f11fde22b7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,24 +1,55 @@ -version: '3.5' +version: '3.8' services: - node: + nodebb: build: . restart: unless-stopped - depends_on: - - db - expose: - - 4567 # use a reverse proxy like Traefik - - db: - image: mongo:bionic + ports: + - "4567:4567/tcp" # comment this out if you don't want to expose NodeBB to the host, or change the first number to any port you want + # uncomment if you want to use another container as a reverse proxy + # expose: + # - 4567 + volumes: + - ./.docker/build:/usr/src/app/build + - ./.docker/public/uploads:/usr/src/app/public/uploads + - ./.docker:/opt/config + - ./install/docker/setup.json:/usr/src/app/setup.json + mongo: + image: "mongo:6-jammy" restart: unless-stopped expose: - - 27017 + - "27017" environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: root + MONGO_INITDB_ROOT_USERNAME: nodebb + MONGO_INITDB_ROOT_PASSWORD: nodebb + MONGO_INITDB_DATABASE: nodebb volumes: - - mongo:/data/db - -volumes: - mongo: + - ./.docker/database/mongo/config:/etc/mongo + - ./.docker/database/mongo/data:/data/db + - ./install/docker/mongodb-user-init.js:/docker-entrypoint-initdb.d/user-init.js + profiles: + - mongo + postgres: + image: postgres:16.0-alpine + restart: unless-stopped + expose: + - "5432" + environment: + POSTGRES_USER: nodebb + POSTGRES_PASSWORD: nodebb + POSTGRES_DB: nodebb + volumes: + - ./.docker/database/postgresql/data:/var/lib/postgresql/data + profiles: + - postgres + redis: + image: redis:7.2.1-alpine + restart: unless-stopped + command: ["redis-server", "--appendonly", "yes", "--loglevel", "warning"] + # command: ["redis-server", "--save", "60", "1", "--loglevel", "warning"] # uncomment if you want to use snapshotting instead of AOF + expose: + - "6379" + volumes: + - ./.docker/database/redis:/data + profiles: + - redis \ No newline at end of file diff --git a/install/docker/entrypoint.sh b/install/docker/entrypoint.sh new file mode 100755 index 0000000000..e5b2036321 --- /dev/null +++ b/install/docker/entrypoint.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +export CONFIG_DIR="${CONFIG_DIR:-/opt/config}" +export CONFIG=$CONFIG_DIR/config.json +export FORCE_BUILD_BEFORE_START="${FORCE_BUILD_BEFORE_START:-false}" + +# Supported verbs: install (web install), setup (interactive CLI session). Default: web install +# TODO: constraint it using a hash set (or hash table) +export NODEBB_INIT_VERB="${NODEBB_INIT_VERB:-install}" +# Setup variable for backward compatibility, default: +export SETUP="${SETUP:-}" + +mkdir -p $CONFIG_DIR + +# if the folder is mounted as a volume this can fail, the check below is to ensure there is still write access +chmod -fR 760 $CONFIG_DIR 2> /dev/null + +if [[ ! -w $CONFIG_DIR ]]; then + echo "panic: no write permission for $CONFIG_DIR" + exit 1 +fi + +[[ -f $CONFIG_DIR/package.json ]] || cp install/package.json $CONFIG_DIR/package.json +[[ -f $CONFIG_DIR/package-lock.json ]] || touch $CONFIG_DIR/package-lock.json + +ln -fs $CONFIG_DIR/package.json package.json +ln -fs $CONFIG_DIR/package-lock.json package-lock.json + +npm install --omit=dev + +if [[ -n $SETUP ]]; then + echo "Setup environmental variable detected" + echo "Starting setup session" + ./nodebb setup --config=$CONFIG +elif [ -f $CONFIG ]; then + echo "Config file exist at $CONFIG, assuming it is a valid config" + echo "Starting forum" + if [ "$FORCE_BUILD_BEFORE_START" = true ]; then + ./nodebb build --config=$CONFIG + fi + ./nodebb start --config=$CONFIG +else + echo "Config file not found at $CONFIG" + echo "Starting installation session" + ./nodebb "${NODEBB_INIT_VERB}" --config=$CONFIG +fi \ No newline at end of file diff --git a/install/docker/mongodb-user-init.js b/install/docker/mongodb-user-init.js new file mode 100644 index 0000000000..36b7079173 --- /dev/null +++ b/install/docker/mongodb-user-init.js @@ -0,0 +1 @@ +db.createUser( { user: 'nodebb', pwd: 'nodebb', roles: [ { role: 'readWrite', db: 'nodebb' }, { role: 'clusterMonitor', db: 'admin' } ] } ) \ No newline at end of file diff --git a/install/docker/setup.json b/install/docker/setup.json new file mode 100644 index 0000000000..3fad840593 --- /dev/null +++ b/install/docker/setup.json @@ -0,0 +1,21 @@ +{ + "mongo": { + "host": "mongo", + "port": 27017, + "database": "nodebb", + "username": "nodebb", + "password": "nodebb" + }, + "redis": { + "host": "redis", + "port": 6379, + "database": 0 + }, + "postgres": { + "host": "postgres", + "port": 5432, + "database": "nodebb", + "username": "nodebb", + "password": "nodebb" + } +} \ No newline at end of file diff --git a/install/web.js b/install/web.js index 1f5a846a30..92fe675c22 100644 --- a/install/web.js +++ b/install/web.js @@ -174,6 +174,8 @@ function install(req, res) { const database = nconf.get('database') || req.body.database || 'mongo'; const setupEnvVars = { ...process.env, + CONFIG: nconf.get('config'), + NODEBB_CONFIG: nconf.get('config'), NODEBB_URL: nconf.get('url') || req.body.url || (`${req.protocol}://${req.get('host')}`), NODEBB_PORT: nconf.get('port') || 4567, NODEBB_ADMIN_USERNAME: nconf.get('admin:username') || req.body['admin:username'], diff --git a/src/cli/setup.js b/src/cli/setup.js index 51245c9262..859f674a9c 100644 --- a/src/cli/setup.js +++ b/src/cli/setup.js @@ -20,12 +20,15 @@ async function setup(initConfig) { console.log('Press enter to accept the default setting (shown in brackets).'); install.values = initConfig; - const data = await install.setup(); let configFile = paths.config; - if (nconf.get('config')) { - configFile = path.resolve(paths.baseDir, nconf.get('config')); + const config = nconf.any(['config', 'CONFIG']); + if (config) { + nconf.set('config', config); + configFile = path.resolve(paths.baseDir, config); } + const data = await install.setup(); + prestart.loadConfig(configFile); if (!nconf.get('skip-build')) { diff --git a/src/install.js b/src/install.js index ea59843f5b..89b40d7b39 100644 --- a/src/install.js +++ b/src/install.js @@ -51,6 +51,8 @@ function checkSetupFlagEnv() { let setupVal = install.values; const envConfMap = { + CONFIG: 'config', + NODEBB_CONFIG: 'config', NODEBB_URL: 'url', NODEBB_PORT: 'port', NODEBB_ADMIN_USERNAME: 'admin:username',