ERPNext is not starting after a system restart

Our ErpNext instance is setup using docker-compose, but after a server power off and power on, I get a 502 Bad Gateway error on

After,

docker-compose -p erp -f docker-compose-common.yml -f docker-compose-erpnext.yml stop
docker-compose -p erp -f docker-compose-common.yml -f docker-compose-erpnext.yml start
# docker-compose -p erp -f docker-compose-common.yml -f docker-compose-erpnext.yml ps
        Name                 Command            State             Ports       
------------------------------------------------------------------------------
erp_erpnext-nginx_1    /docker-               Up           0.0.0.0:8080->80/tc
                       entrypoint.sh ngin                  p                  
                       ...                                                    
erp_erpnext-python_1   docker-entrypoint.sh   Restarting                      
                       start                                                  
erp_frappe-            docker-entrypoint.sh   Restarting                      
schedule_1             schedule                                               
erp_frappe-            docker-entrypoint.sh   Up                              
socketio_1             start                                                  
erp_frappe-worker-     docker-entrypoint.sh   Restarting                      
default_1              worker                                                 
erp_frappe-worker-     docker-entrypoint.sh   Restarting                      
long_1                 worker                                                 
erp_frappe-worker-     docker-entrypoint.sh   Up           8000/tcp           
short_1                worker                                                 
erp_mariadb_1          docker-entrypoint.sh   Up           3306/tcp           
                       mysqld                                                 
erp_redis-cache_1      docker-entrypoint.sh   Up           6379/tcp           
                       redis ...                                              
erp_redis-queue_1      docker-entrypoint.sh   Up           6379/tcp           
                       redis ...                                              
erp_redis-socketio_1   docker-entrypoint.sh   Up           6379/tcp           
                       redis ...

4 containers are on a reboot loop.


            # cat docker-compose-common.yml 
version: '3'

services:
  redis-cache:
    image: redis:latest
    restart: on-failure
    volumes:
      - redis-cache-vol:/data

  redis-queue:
    image: redis:latest
    restart: on-failure
    volumes:
      - redis-queue-vol:/data

  redis-socketio:
    image: redis:latest
    restart: on-failure
    volumes:
      - redis-socketio-vol:/data

  mariadb:
    image: mariadb:10.3
    restart: on-failure
    environment:
      - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}
    volumes:
      - ./frappe-mariadb.cnf:/etc/mysql/conf.d/frappe.cnf
      - mariadb-vol:/var/lib/mysql

volumes:
  mariadb-vol:
  redis-cache-vol:
  redis-queue-vol:
  redis-socketio-vol:

# cat docker-compose-erpnext.yml 
version: '3'

services:
  erpnext-nginx:
    image: frappe/erpnext-nginx:${ERPNEXT_VERSION}
    restart: on-failure
    environment:
      - FRAPPE_PY=erpnext-python
      - FRAPPE_PY_PORT=8000
      - FRAPPE_SOCKETIO=frappe-socketio
      - SOCKETIO_PORT=9000
      - LETSENCRYPT_HOST=${SITES}
      - VIRTUAL_HOST=${SITES}
      - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
    depends_on:
      - erpnext-python
      - frappe-socketio
      - frappe-worker-default
      - frappe-worker-long
      - frappe-worker-short
    links:
      - erpnext-python
      - frappe-socketio
      - frappe-worker-default
      - frappe-worker-long
      - frappe-worker-short
    volumes:
      - sites-vol-1:/var/www/html/sites:rw
      - assets-vol-1:/assets:rw
    ports:
      - 8080:80

  erpnext-python:
    image: frappe/erpnext-worker:${ERPNEXT_VERSION}
    restart: on-failure
    environment:
      - MARIADB_HOST=${MARIADB_HOST}
      - REDIS_CACHE=redis-cache:6379
      - REDIS_QUEUE=redis-queue:6379
      - REDIS_SOCKETIO=redis-socketio:6379
      - SOCKETIO_PORT=9000
      - AUTO_MIGRATE=1
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw
      - assets-vol-1:/home/frappe/frappe-bench/sites/assets:rw

  frappe-socketio:
    image: frappe/frappe-socketio:v12
    restart: on-failure
    depends_on:
      - redis-socketio
    links:
      - redis-socketio
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw

  frappe-worker-default:
    image: frappe/erpnext-worker:${ERPNEXT_VERSION}
    restart: on-failure
    command: worker
    depends_on:
      - redis-queue
      - redis-cache
    links:
      - redis-queue
      - redis-cache
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw

  frappe-worker-short:
    image: frappe/erpnext-worker:${ERPNEXT_VERSION}
    restart: on-failure
    command: worker
    environment:
      - WORKER_TYPE=short
    depends_on:
      - redis-queue
      - redis-cache
    links:
      - redis-queue
      - redis-cache
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw

  frappe-worker-long:
    image: frappe/erpnext-worker:${ERPNEXT_VERSION}
    restart: on-failure
    command: worker
    environment:
      - WORKER_TYPE=long
    depends_on:
      - redis-queue
      - redis-cache
    links:
      - redis-queue
      - redis-cache
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw

  frappe-schedule:
    image: frappe/erpnext-worker:${ERPNEXT_VERSION}
    restart: on-failure
    command: schedule
    depends_on:
      - redis-queue
      - redis-cache
    links:
      - redis-queue
      - redis-cache
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw

  site-creator:
    image: frappe/erpnext-worker:v12.12.0
    restart: "no"
    command: new
    depends_on:
      - erpnext-python
    environment:
      - SITE_NAME=${SITE_NAME}
      - DB_ROOT_USER=${DB_ROOT_USER}
      - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}
      - ADMIN_PASSWORD=${ADMIN_PASSWORD}
      - INSTALL_APPS=${INSTALL_APPS}
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw


volumes:
  assets-vol-1:
  sites-vol-1:

ERPNEXT_VERSION=v12.12.0
FRAPPE_VERSION=v12.12.0
docker logs erp_frappe-worker-default_1
You are required to change your password immediately (password expired)
Current password: su: Authentication token manipulation error
Changing password for frappe.

Any ideas to resolve this?

cc @revant_one

There are many changes to the setup today.

  • We started with multiple images. The base image for all images was bitnami minideb.
  • Later all the images were combined in one for simplicity. The base image now uses official python slim images which are based on debian slim images.
  • “you are require to change your password” doesn’t happen in official python images.

Please build a custom image with following commands.

# clone repo and switch dir
git clone https://github.com/frappe/frappe_docker && cd frappe_docker

# use specific version of frappe-bench (required to install v12)
sed -i 's/pip3 install frappe-bench/pip3 install frappe-bench==5.4.1/g' images/custom/Containerfile

# Set your erpnext version
export APPS_JSON='[
  {
    "url": "https://github.com/frappe/erpnext",
    "branch": "v12.12.0"
  }
]'

# generate base64 string of above json
export APPS_JSON_BASE64=$(echo ${APPS_JSON} | base64 -w 0)

# build image with py 3.7 node 10
docker build \
  --progress=plain \
  --build-arg=FRAPPE_BRANCH=v12.12.0 \
  --build-arg=PYTHON_VERSION=3.7.17 \
  --build-arg=NODE_VERSION=10.24.1 \
  --build-arg=APPS_JSON_BASE64=$APPS_JSON_BASE64 \
  --tag=ghcr.io/user/repo/erpnext:v12.12.0 \
  --file=images/custom/Containerfile .

Following is your docker-compose.yml changed as per new setup and the custom image.

# cat docker-compose-erpnext.yml 
version: '3'

services:
  erpnext-nginx:
    image: ghcr.io/user/repo/erpnext:v12.12.0
    restart: on-failure
    depends_on:
      - erpnext-python
      - frappe-socketio
      - frappe-worker-default
      - frappe-worker-long
      - frappe-worker-short
    command:
      - nginx-entrypoint.sh
    environment:
      - BACKEND=erpnext-python:8000
      - SOCKETIO=frappe-socketio:9000
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites
    ports:
      - "8080:8080"
    links:
      - erpnext-python
      - frappe-socketio
      - frappe-worker-default
      - frappe-worker-long
      - frappe-worker-short

  erpnext-python:
    image: ghcr.io/user/repo/erpnext:v12.12.0
    volumes:
      - logs:/home/frappe/frappe-bench/logs
    restart: on-failure
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw

  frappe-socketio:
    image: ghcr.io/user/repo/erpnext:v12.12.0
    restart: on-failure
    command:
      - node
      - /home/frappe/frappe-bench/apps/frappe/socketio.js
    depends_on:
      - redis-socketio
    links:
      - redis-socketio
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw

  frappe-worker-default:
    image: ghcr.io/user/repo/erpnext:v12.12.0
    restart: on-failure
    command:
      - bench
      - worker
      - --queue
      - default
    depends_on:
      - redis-queue
      - redis-cache
    links:
      - redis-queue
      - redis-cache
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw

  frappe-worker-short:
    image: ghcr.io/user/repo/erpnext:v12.12.0
    restart: on-failure
    command:
      - bench
      - worker
      - --queue
      - short
    depends_on:
      - redis-queue
      - redis-cache
    links:
      - redis-queue
      - redis-cache
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw

  frappe-worker-long:
    image: ghcr.io/user/repo/erpnext:v12.12.0
    restart: on-failure
    command:
      - bench
      - worker
      - --queue
      - long
    depends_on:
      - redis-queue
      - redis-cache
    links:
      - redis-queue
      - redis-cache
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw

  frappe-schedule:
    image: ghcr.io/user/repo/erpnext:v12.12.0
    restart: on-failure
    command:
      - bench
      - schedule
    depends_on:
      - redis-queue
      - redis-cache
    links:
      - redis-queue
      - redis-cache
    volumes:
      - sites-vol-1:/home/frappe/frappe-bench/sites:rw

volumes:
  sites-vol-1:
2 Likes

@revant_one thanks a lot for this solution. Since we were using it only for some non critical tasks, I could not try these steps till now.

Today when I tried, docker build was failing with,

Err:1 http://deb.debian.org/debian bookworm InRelease
  The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 0E98404D386FA1D9 NO_PUBKEY 6ED0E7B82643E131 NO_PUBKEY F8D2585B8783D481

It took a while for me to update the gpg keys. I downloaded http://security.debian.org/debian-security/pool/updates/main/d/debian-archive-keyring/debian-archive-keyring_2019.1+deb10u2_all.deb and did a COPY in dockerfile so I can import the keys.

Docker was running in Debian buster so it did not have the keys for bookworm.


root@erp:/srv/frappe_docker# git diff
diff --git a/images/custom/Containerfile b/images/custom/Containerfile
index eed82e5..c0d9db4 100644
--- a/images/custom/Containerfile
+++ b/images/custom/Containerfile
@@ -4,6 +4,7 @@ FROM python:${PYTHON_VERSION}-slim-${DEBIAN_BASE} AS base
 
 COPY resources/nginx-template.conf /templates/nginx/frappe.conf.template
 COPY resources/nginx-entrypoint.sh /usr/local/bin/nginx-entrypoint.sh
+COPY resources/debian-archive-keyring*.deb /root
 
 ARG WKHTMLTOPDF_VERSION=0.12.6.1-3
 ARG WKHTMLTOPDF_DISTRO=bookworm
@@ -11,6 +12,7 @@ ARG NODE_VERSION=18.18.2
 ENV NVM_DIR=/home/frappe/.nvm
 ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH}
 
+RUN dpkg -i /root/debian-archive-keyring*.deb
 RUN useradd -ms /bin/bash frappe \
     && apt-get update \
     && apt-get install --no-install-recommends -y \

Installing the deb with RUN dpkg -i failed with Debian bookworm: apt-get update fails with member 'control.tar': lzma error: Cannot allocate memory - Unix & Linux Stack Exchange which was fixed by updating docker-ce packages to latest.

But these worker containers are still in a reboot loop, though with a different error now.

root@erp:/srv/erp.puri.sm# docker-compose -p erp -f docker-compose-common.yml--f docker-compose-erpnext.yml ps
        Name                 Command            State             Ports       
------------------------------------------------------------------------------
erp_erpnext-nginx_1    nginx-entrypoint.sh    Up           0.0.0.0:8080->8080/
                                                           tcp,:::8080->8080/t
                                                           cp                 
erp_erpnext-python_1   /home/frappe/frappe-   Restarting                      
                       bench/ ...                                             
erp_frappe-            bench schedule         Restarting                      
schedule_1                                                                    
erp_frappe-            node /home/frappe/fr   Up                              
socketio_1             appe-b ...                                             
erp_frappe-worker-     bench worker --queue   Restarting                      
default_1              default                                                
erp_frappe-worker-     bench worker --queue   Restarting                      
long_1                 long                                                   
erp_frappe-worker-     bench worker --queue   Restarting                      
short_1                short                                                  
erp_mariadb_1          docker-entrypoint.sh   Up           3306/tcp           
                       mysqld                                                 
erp_redis-cache_1      docker-entrypoint.sh   Up           6379/tcp           
                       redis ...                                              
erp_redis-queue_1      docker-entrypoint.sh   Up           6379/tcp           
                       redis ...                                              
erp_redis-socketio_1   docker-entrypoint.sh   Up           6379/tcp           
                       redis ...
root@erp:/srv/erp.puri.sm# docker container ls
CONTAINER ID   IMAGE                                COMMAND                  CREATED              STATUS                          PORTS                                       NAMES
29125cf755e1   ghcr.io/user/repo/erpnext:v12.12.0   "nginx-entrypoint.sh"    59 seconds ago       Up 58 seconds                   0.0.0.0:8080->8080/tcp, :::8080->8080/tcp   erp_erpnext-nginx_1
2b818b133627   ghcr.io/user/repo/erpnext:v12.12.0   "bench schedule"         About a minute ago   Restarting (2) 24 seconds ago                                               erp_frappe-schedule_1
74b423868ca9   ghcr.io/user/repo/erpnext:v12.12.0   "node /home/frappe/f…"   About a minute ago   Up 58 seconds                                                               erp_frappe-socketio_1
80d1c62a2615   ghcr.io/user/repo/erpnext:v12.12.0   "bench worker --queu…"   About a minute ago   Restarting (2) 24 seconds ago                                               erp_frappe-worker-short_1
6952748ba9b6   ghcr.io/user/repo/erpnext:v12.12.0   "bench worker --queu…"   About a minute ago   Restarting (2) 23 seconds ago                                               erp_frappe-worker-default_1
717201c62fcf   ghcr.io/user/repo/erpnext:v12.12.0   "bench worker --queu…"   About a minute ago   Restarting (2) 24 seconds ago                                               erp_frappe-worker-long_1
db00a97f629d   redis:latest                         "docker-entrypoint.s…"   About a minute ago   Up About a minute               6379/tcp                                    erp_redis-socketio_1
7160e51eb45a   redis:latest                         "docker-entrypoint.s…"   About a minute ago   Up About a minute               6379/tcp                                    erp_redis-cache_1
a49e1fe755f6   mariadb:10.3                         "docker-entrypoint.s…"   About a minute ago   Up About a minute               3306/tcp                                    erp_mariadb_1
814116724c69   ghcr.io/user/repo/erpnext:v12.12.0   "/home/frappe/frappe…"   About a minute ago   Restarting (1) 19 seconds ago                                               erp_erpnext-python_1
eb4657e9fd9e   redis:latest                         "docker-entrypoint.s…"   About a minute ago   Up About a minute               6379/tcp                                    erp_redis-queue_1
root@erp:/srv/erp.puri.sm# docker logs 6952748ba9b6
Usage: bench [OPTIONS] COMMAND [ARGS]...
Try "bench --help" for help.

Error: No such command "worker".
INFO: A newer version of bench is available: 5.4.1 → 5.22.3

The same error for other containers too.

I tried other versions of frappe-bench as well, but none worked.

For 5.22.3 and 5.7.0, docker build itself failed,


#13 185.3 $ bench build
#13 188.9 Usage: bench [OPTIONS] COMMAND [ARGS]...
#13 188.9 Try 'bench --help' for help.
#13 188.9 
#13 188.9 Error: No such command 'build'.
#13 189.9 Traceback (most recent call last):
#13 189.9   File "/usr/local/lib/python3.7/site-packages/bench/commands/make.py", line 83, in init
#13 189.9     verbose=verbose,
#13 189.9   File "/usr/local/lib/python3.7/site-packages/bench/utils/render.py", line 95, in wrapper_fn
#13 189.9     return fn(*args, **kwargs)
#13 189.9   File "/usr/local/lib/python3.7/site-packages/bench/utils/system.py", line 93, in init
#13 189.9     build_assets(bench_path=path)
#13 189.9   File "/usr/local/lib/python3.7/site-packages/bench/utils/bench.py", line 294, in build_assets
#13 189.9     exec_cmd(command, cwd=bench_path, env={"BENCH_DEVELOPER": "1"})
#13 189.9   File "/usr/local/lib/python3.7/site-packages/bench/utils/__init__.py", line 126, in exec_cmd
#13 189.9     raise CommandFailedError
#13 189.9 bench.exceptions.CommandFailedError
#13 189.9

For 5.5.0, docker build succeeded but then some containers still in reboot loop.

Updated tag in build command and in docker-compose-erpnext.yml

root@erp:/srv/erp.puri.sm# docker-compose -p erp -f docker-compose-common.yml -f docker-compose-erpnext.yml ps
        Name                 Command            State             Ports       
------------------------------------------------------------------------------
erp_erpnext-nginx_1    nginx-entrypoint.sh    Up           0.0.0.0:8080->8080/
                                                           tcp,:::8080->8080/t
                                                           cp                 
erp_erpnext-python_1   /home/frappe/frappe-   Restarting                      
                       bench/ ...                                             
erp_frappe-            bench schedule         Restarting                      
schedule_1                                                                    
erp_frappe-            node /home/frappe/fr   Up                              
socketio_1             appe-b ...                                             
erp_frappe-worker-     bench worker --queue   Restarting                      
default_1              default                                                
erp_frappe-worker-     bench worker --queue   Restarting                      
long_1                 long                                                   
erp_frappe-worker-     bench worker --queue   Restarting                      
short_1                short                                                  
erp_mariadb_1          docker-entrypoint.sh   Up           3306/tcp           
                       mysqld                                                 
erp_redis-cache_1      docker-entrypoint.sh   Up           6379/tcp           
                       redis ...                                              
erp_redis-queue_1      docker-entrypoint.sh   Up           6379/tcp           
                       redis ...                                              
erp_redis-socketio_1   docker-entrypoint.sh   Up           6379/tcp           
                       redis ...                                              
root@erp:/srv/erp.puri.sm# docker container ps
CONTAINER ID   IMAGE                                           COMMAND                  CREATED          STATUS                         PORTS                                       NAMES
c86e8768174c   source.puri.sm/purism/frappe/erpnext:v12.12.0   "nginx-entrypoint.sh"    41 seconds ago   Up 40 seconds                  0.0.0.0:8080->8080/tcp, :::8080->8080/tcp   erp_erpnext-nginx_1
a1b9dec14fee   source.puri.sm/purism/frappe/erpnext:v12.12.0   "bench worker --queu…"   42 seconds ago   Restarting (2) 5 seconds ago                                               erp_frappe-worker-default_1
02e0f11e0b61   source.puri.sm/purism/frappe/erpnext:v12.12.0   "bench worker --queu…"   42 seconds ago   Restarting (2) 5 seconds ago                                               erp_frappe-worker-long_1
a0fb110e6f14   source.puri.sm/purism/frappe/erpnext:v12.12.0   "bench schedule"         42 seconds ago   Restarting (2) 6 seconds ago                                               erp_frappe-schedule_1
d98a2ce817e9   source.puri.sm/purism/frappe/erpnext:v12.12.0   "bench worker --queu…"   42 seconds ago   Restarting (2) 6 seconds ago                                               erp_frappe-worker-short_1
8d0c9b8a3721   source.puri.sm/purism/frappe/erpnext:v12.12.0   "node /home/frappe/f…"   42 seconds ago   Up 41 seconds                                                              erp_frappe-socketio_1
4040d90eb982   redis:latest                                    "docker-entrypoint.s…"   43 seconds ago   Up 42 seconds                  6379/tcp                                    erp_redis-socketio_1
2d89e4b85f62   redis:latest                                    "docker-entrypoint.s…"   43 seconds ago   Up 42 seconds                  6379/tcp                                    erp_redis-queue_1
3feab454d90e   redis:latest                                    "docker-entrypoint.s…"   43 seconds ago   Up 42 seconds                  6379/tcp                                    erp_redis-cache_1
1c2aae33dac8   source.puri.sm/purism/frappe/erpnext:v12.12.0   "/home/frappe/frappe…"   43 seconds ago   Restarting (1) 1 second ago                                                erp_erpnext-python_1
6b0a836071a8   mariadb:10.3                                    "docker-entrypoint.s…"   43 seconds ago   Up 42 seconds                  3306/tcp                                    erp_mariadb_1

ImportError: cannot import name 'soft_unicode' from 'markupsafe' (/home/frappe/frappe-bench/env/lib/python3.7/site-packages/markupsafe/__init__.py)
Traceback (most recent call last):
  File "/home/frappe/frappe-bench/env/bin/gunicorn", line 8, in <module>
    sys.exit(run())
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 61, in run
    WSGIApplication("%(prog)s [OPTIONS] [APP_MODULE]").run()
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/gunicorn/app/base.py", line 223, in run
    super(Application, self).run()
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/gunicorn/app/base.py", line 72, in run
    Arbiter(self).run()
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 60, in __init__
    self.setup(app)
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 120, in setup
    self.app.wsgi()
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/gunicorn/app/base.py", line 67, in wsgi
    self.callable = self.load()
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 52, in load
    return self.load_wsgiapp()
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 41, in load_wsgiapp
    return util.import_app(self.app_uri)
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/gunicorn/util.py", line 350, in import_app
    __import__(module)
  File "/home/frappe/frappe-bench/apps/frappe/frappe/app.py", line 17, in <module>
    import frappe.handler
  File "/home/frappe/frappe-bench/apps/frappe/frappe/handler.py", line 8, in <module>
    import frappe.sessions
  File "/home/frappe/frappe-bench/apps/frappe/frappe/sessions.py", line 17, in <module>
    import frappe.translate
  File "/home/frappe/frappe-bench/apps/frappe/frappe/translate.py", line 18, in <module>
    from jinja2 import TemplateError
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/jinja2/__init__.py", line 33, in <module>
    from jinja2.environment import Environment, Template
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/jinja2/environment.py", line 15, in <module>
    from jinja2 import nodes
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/jinja2/nodes.py", line 19, in <module>
    from jinja2.utils import Markup
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/jinja2/utils.py", line 642, in <module>
    from markupsafe import Markup, escape, soft_unicode
ImportError: cannot import name 'soft_unicode' from 'markupsafe' (/home/frappe/frappe-bench/env/lib/python3.7/site-packages/markupsafe/__init__.py)

Does frappe-bench 5.4.1 support bench worker command? Or does it need installing some other package as well?

To debug this further, I changes command for worker container to just sleep 100000 so I could exec bash.


frappe@fd5e417f866a:~/frappe-bench/sites$ bench backup erp.puri.sm
Traceback (most recent call last):
  File "/home/frappe/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", line 68, in get_app_commands
    app_command_module = importlib.import_module(app + '.commands')
  File "/usr/local/lib/python3.7/importlib/__init__.py", line 127, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
  File "<frozen importlib._bootstrap>", line 983, in _find_and_load
  File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 728, in exec_module
  File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
  File "/home/frappe/frappe-bench/apps/frappe/frappe/commands/__init__.py", line 87, in <module>
    commands = get_commands()
  File "/home/frappe/frappe-bench/apps/frappe/frappe/commands/__init__.py", line 81, in get_commands
    from .site import commands as site_commands
  File "/home/frappe/frappe-bench/apps/frappe/frappe/commands/site.py", line 19, in <module>
    from frappe.installer import update_site_config
  File "/home/frappe/frappe-bench/apps/frappe/frappe/installer.py", line 17, in <module>
    from frappe.website import render
  File "/home/frappe/frappe-bench/apps/frappe/frappe/website/render.py", line 7, in <module>
    import frappe.sessions
  File "/home/frappe/frappe-bench/apps/frappe/frappe/sessions.py", line 17, in <module>
    import frappe.translate
  File "/home/frappe/frappe-bench/apps/frappe/frappe/translate.py", line 18, in <module>
    from jinja2 import TemplateError
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/jinja2/__init__.py", line 33, in <module>
    from jinja2.environment import Environment, Template
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/jinja2/environment.py", line 15, in <module>
    from jinja2 import nodes
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/jinja2/nodes.py", line 19, in <module>
    from jinja2.utils import Markup
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/jinja2/utils.py", line 642, in <module>
    from markupsafe import Markup, escape, soft_unicode
ImportError: cannot import name 'soft_unicode' from 'markupsafe' (/home/frappe/frappe-bench/env/lib/python3.7/site-packages/markupsafe/__init__.py)
Usage: bench frappe [OPTIONS] COMMAND [ARGS]...
Try "bench frappe --help" for help.

Error: No such command "backup".

So the root cause for bench worker failing could also be this

ImportError: cannot import name 'soft_unicode' from 'markupsafe' (/home/frappe/frappe-bench/env/lib/python3.7/site-packages/markupsafe/__init__.py)

Trying out python - ImportError: cannot import name 'soft_unicode' from 'markupsafe' - Stack Overflow

Forcing a newer Jinja2 and hoping that fixes it.

-    && pip3 install frappe-bench \
+    && pip3 install frappe-bench==5.4.1 \
+    && pip3 install Jinja2==3.1.2 \

Update: this did not fix the problem

I tried forking frappe-bench and locking MarkupSafe==2.0.1 but that did not fix this.

while looking to fork frappe itself, I found there is a fixed version already so I need to change the docker build command to use this branch

--build-arg=FRAPPE_BRANCH=version-12-hotfix

With this change bench backup is working. But bench worker now fails with

ModuleNotFoundError: No module named 'erpnext'



frappe@d974a8d48308:~/frappe-bench$ bench worker --queue default
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/local/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/frappe/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", line 99, in <module>
    main()
  File "/home/frappe/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", line 18, in main
    click.Group(commands=commands)(prog_name='bench')
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/click/core.py", line 764, in __call__
    return self.main(*args, **kwargs)
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/click/core.py", line 717, in main
    rv = self.invoke(ctx)
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/click/core.py", line 1137, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/click/core.py", line 1137, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/click/core.py", line 956, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/click/core.py", line 555, in invoke
    return callback(*args, **kwargs)
  File "/home/frappe/frappe-bench/apps/frappe/frappe/commands/scheduler.py", line 167, in start_worker
    start_worker(queue, quiet = quiet)
  File "/home/frappe/frappe-bench/apps/frappe/frappe/utils/background_jobs.py", line 139, in start_worker
    with frappe.init_site():
  File "/home/frappe/frappe-bench/apps/frappe/frappe/__init__.py", line 248, in __enter__
    init(self.site)
  File "/home/frappe/frappe-bench/apps/frappe/frappe/__init__.py", line 175, in init
    setup_module_map()
  File "/home/frappe/frappe-bench/apps/frappe/frappe/__init__.py", line 1049, in setup_module_map
    for module in get_module_list(app):
  File "/home/frappe/frappe-bench/apps/frappe/frappe/__init__.py", line 912, in get_module_list
    return get_file_items(os.path.join(os.path.dirname(get_module(app_name).__file__), "modules.txt"))
  File "/home/frappe/frappe-bench/apps/frappe/frappe/__init__.py", line 870, in get_module
    return importlib.import_module(modulename)
  File "/usr/local/lib/python3.7/importlib/__init__.py", line 127, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
  File "<frozen importlib._bootstrap>", line 983, in _find_and_load
  File "<frozen importlib._bootstrap>", line 965, in _find_and_load_unlocked
ModuleNotFoundError: No module named 'erpnext'

as per ModuleNotFoundError! - #11 by NCP

frappe@d974a8d48308:~/frappe-bench$ bench update --reset
INFO: A newer version of bench is available: 5.4.1 → 5.22.3
frappe@d974a8d48308:~/frappe-bench$ bench setup requirements
Installing applications...
$ ./env/bin/python -m pip install -q -U pip

Installing frappe
$ ./env/bin/python -m pip install -q -U -e ./apps/frappe 
$ yarn install
yarn install v1.22.22
[1/4] Resolving packages...
success Already up-to-date.
Done in 0.52s.

Installing erpnext
$ ./env/bin/python -m pip install -q -U -e ./apps/erpnext 
ERROR: ./apps/erpnext is not a valid editable requirement. It should either be a path to a local project or a VCS URL (beginning with bzr+http, bzr+https, bzr+ssh, bzr+sftp, bzr+ftp, bzr+lp, bzr+file, git+http, git+https, git+ssh, git+git, git+file, hg+file, hg+http, hg+https, hg+ssh, hg+static-http, svn+ssh, svn+http, svn+https, svn+svn, svn+file).
INFO: A newer version of bench is available: 5.4.1 → 5.22.3

I had to remove erpnext from sites/apps.txt and sites/app_info.json for all containers to start. But now it gives an internal server error. apps directory only has frappe and no erpnext though. Can I reinstall just the code (bench reinstall wipes db as well).

You can try refer this guide Setup Production ERPNext behind Nginx reverse proxy (Docker) | by Poong Ka Vui | Mar, 2024 | Medium

@pravi judging from all the back and forth, it might be better to just backup the data directly from mariadb container and restore it on new installation.