Permission Error on Initialize Bench (Frappe Press)

flexy2ky · December 19, 2023, 9:09pm

When Initializing bench on a new server, the following error is returned:
`

Data
{
    "command": "docker run --rm --net none -v /home/frappe/benches/bench-0001-000002-appsvr1/config:/home/frappe/frappe-bench/configmount registry.digitalocean.com/test/ftest-cloud/cloud.testurl.net/bench-0001:deploy-0001-000002 cp -LR config/. configmount",
    "directory": "/home/frappe/benches/bench-0001-000002-appsvr1",
    "duration": "0:00:00.211595",
    "end": "2023-12-19 17:53:46.946393",
    "output": "cp: cannot create directory 'configmount/./pids': Permission denied\ncp: cannot create regular file 'configmount/./redis_cache.acl': Permission denied\ncp: cannot create regular file 'configmount/./redis_cache.conf': Permission denied\ncp: cannot create regular file 'configmount/./redis_queue.acl': Permission denied\ncp: cannot create regular file 'configmount/./redis_queue.conf': Permission denied\ncp: cannot create regular file 'configmount/./redis_socketio.acl': Permission denied\ncp: cannot create regular file 'configmount/./redis_socketio.conf': Permission denied\ncp: cannot create regular file 'configmount/./bench_build.lock': Permission denied\ncp: cannot create regular file 'configmount/./redis-cache.conf': Permission denied\ncp: cannot create regular file 'configmount/./redis-queue.conf': Permission denied\ncp: cannot create directory 'configmount/./ssh': Permission denied\ncp: cannot create regular file 'configmount/./supervisor.conf': Permission denied",
    "returncode": 1,
    "start": "2023-12-19 17:53:46.734798",
    "traceback": "Traceback (most recent call last):\n  File \"/home/frappe/agent/repo/agent/base.py\", line 32, in execute\n    process = subprocess.run(\n  File \"/usr/lib/python3.10/subprocess.py\", line 526, in run\n    raise CalledProcessError(retcode, process.args,\nsubprocess.CalledProcessError: Command 'docker run --rm --net none -v /home/frappe/benches/bench-0001-000002-appsvr1/config:/home/frappe/frappe-bench/configmount registry.digitalocean.com/test/ftest-cloud/cloud.testurl.net/bench-0001:deploy-0001-000002 cp -LR config/. configmount' returned non-zero exit status 1.\n"

Output
cp: cannot create directory 'configmount/./pids': Permission denied
cp: cannot create regular file 'configmount/./redis_cache.acl': Permission denied
cp: cannot create regular file 'configmount/./redis_cache.conf': Permission denied
cp: cannot create regular file 'configmount/./redis_queue.acl': Permission denied
cp: cannot create regular file 'configmount/./redis_queue.conf': Permission denied
cp: cannot create regular file 'configmount/./redis_socketio.acl': Permission denied
cp: cannot create regular file 'configmount/./redis_socketio.conf': Permission denied
cp: cannot create regular file 'configmount/./bench_build.lock': Permission denied
cp: cannot create regular file 'configmount/./redis-cache.conf': Permission denied
cp: cannot create regular file 'configmount/./redis-queue.conf': Permission denied
cp: cannot create directory 'configmount/./ssh': Permission denied
cp: cannot create regular file 'configmount/./supervisor.conf': Permission denied

Traceback
Traceback (most recent call last):
  File "/home/frappe/agent/repo/agent/base.py", line 32, in execute
    process = subprocess.run(
  File "/usr/lib/python3.10/subprocess.py", line 526, in run
    raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command 'docker run --rm --net none -v /home/frappe/benches/bench-0001-000002-appsvr1/config:/home/frappe/frappe-bench/configmount Output
cp: cannot create directory 'configmount/./pids': Permission denied
cp: cannot create regular file 'configmount/./redis_cache.acl': Permission denied
cp: cannot create regular file 'configmount/./redis_cache.conf': Permission denied
cp: cannot create regular file 'configmount/./redis_queue.acl': Permission denied
cp: cannot create regular file 'configmount/./redis_queue.conf': Permission denied
cp: cannot create regular file 'configmount/./redis_socketio.acl': Permission denied
cp: cannot create regular file 'configmount/./redis_socketio.conf': Permission denied
cp: cannot create regular file 'configmount/./bench_build.lock': Permission denied
cp: cannot create regular file 'configmount/./redis-cache.conf': Permission denied
cp: cannot create regular file 'configmount/./redis-queue.conf': Permission denied
cp: cannot create directory 'configmount/./ssh': Permission denied
cp: cannot create regular file 'configmount/./supervisor.conf': Permission denied
Traceback
Traceback (most recent call last):
  File "/home/frappe/agent/repo/agent/base.py", line 32, in execute
    process = subprocess.run(
  File "/usr/lib/python3.10/subprocess.py", line 526, in run
    raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command 'docker run --rm --net none -v /home/frappe/benches/bench-0001-000002-appsvr1/config:/home/frappe/frappe-bench/configmount registry.digitalocean.com/test/ftest-cloud/cloud.testurl.net/bench-0001:deploy-0001-000002 cp -LR config/. configmount' returned non-zero exit status 1. cp -LR config/. configmount' returned non-zero exit status 1.
}`

And the offending function is

@step("Initialize Bench")
  def bench_init(self, name, config):
       bench_directory = os.path.join(self.benches_directory, name)
       os.mkdir(bench_directory)
       directories = ["logs", "sites", "config"]
       for directory in directories:
           os.mkdir(os.path.join(bench_directory, directory))

       bench_config_file = os.path.join(bench_directory, "config.json")
       with open(bench_config_file, "w") as f:
           json.dump(config, f, indent=1, sort_keys=True)

       config.update({"directory": bench_directory, "name": name})
       docker_compose = os.path.join(bench_directory, "docker-compose.yml")
       self._render_template(
           "bench/docker-compose.yml.jinja2", config, docker_compose
       )

       config_directory = os.path.join(bench_directory, "config")
       command = (
           f"docker run --rm --net none "
           f"-v {config_directory}:/home/frappe/frappe-bench/configmount "
           f"{config['docker_image']} cp -LR config/. configmount"
       )
       self.execute(command, directory=bench_directory)

       sites_directory = os.path.join(bench_directory, "sites")
       # Copy sites directory from image to host system
       command = (
           f"docker run --rm --net none "
           f"-v {sites_directory}:/home/frappe/frappe-bench/sitesmount "
           f"{config['docker_image']} cp -LR sites/. sitesmount"
       )
       return self.execute(command, directory=bench_directory)

   def dump(self):
       return {
           "name": self.name,
           "benches": {
               name: bench.dump() for name, bench in self.benches.items()
           },
           "config": self.config,
       }`

The way to overcome this error is to modify this block like this:

`       uid = os.getuid()
        gid = os.getgid()

        config_directory = os.path.join(bench_directory, "config")
        command = (
            f"docker run --rm --net none -u {uid}:{gid}"
            f"-v {config_directory}:/home/frappe/frappe-bench/configmount "
            f"{config['docker_image']} cp -LR config/. configmount"
        )
        self.execute(command, directory=bench_directory)

        sites_directory = os.path.join(bench_directory, "sites")
        # Copy sites directory from image to host system
        command = (
            f"docker run --rm --net none -u {uid}:{gid}"
            f"-v {sites_directory}:/home/frappe/frappe-bench/sitesmount "
            f"{config['docker_image']} cp -LR sites/. sitesmount"
        )
        return self.execute(command, directory=bench_directory)`.

However, the relevant directories and files are not created even though the Initialize Bench job is then marked as Complete. Is this a unique situation or it’s a bug someone needs to look into?

Github issue raised here:

github.com/frappe/press

Permission Error on Initialize Bench

opened 06:23PM - 19 Dec 23 UTC

flexcomng

When Initializing bench on a new server, the following error is returned: ` …``` Data { "command": "docker run --rm --net none -v /home/frappe/benches/bench-0001-000002-appsvr1/config:/home/frappe/frappe-bench/configmount registry.digitalocean.com/test/ftest-cloud/cloud.testurl.net/bench-0001:deploy-0001-000002 cp -LR config/. configmount", "directory": "/home/frappe/benches/bench-0001-000002-appsvr1", "duration": "0:00:00.211595", "end": "2023-12-19 17:53:46.946393", "output": "cp: cannot create directory 'configmount/./pids': Permission denied\ncp: cannot create regular file 'configmount/./redis_cache.acl': Permission denied\ncp: cannot create regular file 'configmount/./redis_cache.conf': Permission denied\ncp: cannot create regular file 'configmount/./redis_queue.acl': Permission denied\ncp: cannot create regular file 'configmount/./redis_queue.conf': Permission denied\ncp: cannot create regular file 'configmount/./redis_socketio.acl': Permission denied\ncp: cannot create regular file 'configmount/./redis_socketio.conf': Permission denied\ncp: cannot create regular file 'configmount/./bench_build.lock': Permission denied\ncp: cannot create regular file 'configmount/./redis-cache.conf': Permission denied\ncp: cannot create regular file 'configmount/./redis-queue.conf': Permission denied\ncp: cannot create directory 'configmount/./ssh': Permission denied\ncp: cannot create regular file 'configmount/./supervisor.conf': Permission denied", "returncode": 1, "start": "2023-12-19 17:53:46.734798", "traceback": "Traceback (most recent call last):\n File \"/home/frappe/agent/repo/agent/base.py\", line 32, in execute\n process = subprocess.run(\n File \"/usr/lib/python3.10/subprocess.py\", line 526, in run\n raise CalledProcessError(retcode, process.args,\nsubprocess.CalledProcessError: Command 'docker run --rm --net none -v /home/frappe/benches/bench-0001-000002-appsvr1/config:/home/frappe/frappe-bench/configmount registry.digitalocean.com/test/ftest-cloud/cloud.testurl.net/bench-0001:deploy-0001-000002 cp -LR config/. configmount' returned non-zero exit status 1.\n" Output cp: cannot create directory 'configmount/./pids': Permission denied cp: cannot create regular file 'configmount/./redis_cache.acl': Permission denied cp: cannot create regular file 'configmount/./redis_cache.conf': Permission denied cp: cannot create regular file 'configmount/./redis_queue.acl': Permission denied cp: cannot create regular file 'configmount/./redis_queue.conf': Permission denied cp: cannot create regular file 'configmount/./redis_socketio.acl': Permission denied cp: cannot create regular file 'configmount/./redis_socketio.conf': Permission denied cp: cannot create regular file 'configmount/./bench_build.lock': Permission denied cp: cannot create regular file 'configmount/./redis-cache.conf': Permission denied cp: cannot create regular file 'configmount/./redis-queue.conf': Permission denied cp: cannot create directory 'configmount/./ssh': Permission denied cp: cannot create regular file 'configmount/./supervisor.conf': Permission denied Traceback Traceback (most recent call last): File "/home/frappe/agent/repo/agent/base.py", line 32, in execute process = subprocess.run( File "/usr/lib/python3.10/subprocess.py", line 526, in run raise CalledProcessError(retcode, process.args, subprocess.CalledProcessError: Command 'docker run --rm --net none -v /home/frappe/benches/bench-0001-000002-appsvr1/config:/home/frappe/frappe-bench/configmount Output cp: cannot create directory 'configmount/./pids': Permission denied cp: cannot create regular file 'configmount/./redis_cache.acl': Permission denied cp: cannot create regular file 'configmount/./redis_cache.conf': Permission denied cp: cannot create regular file 'configmount/./redis_queue.acl': Permission denied cp: cannot create regular file 'configmount/./redis_queue.conf': Permission denied cp: cannot create regular file 'configmount/./redis_socketio.acl': Permission denied cp: cannot create regular file 'configmount/./redis_socketio.conf': Permission denied cp: cannot create regular file 'configmount/./bench_build.lock': Permission denied cp: cannot create regular file 'configmount/./redis-cache.conf': Permission denied cp: cannot create regular file 'configmount/./redis-queue.conf': Permission denied cp: cannot create directory 'configmount/./ssh': Permission denied cp: cannot create regular file 'configmount/./supervisor.conf': Permission denied Traceback Traceback (most recent call last): File "/home/frappe/agent/repo/agent/base.py", line 32, in execute process = subprocess.run( File "/usr/lib/python3.10/subprocess.py", line 526, in run raise CalledProcessError(retcode, process.args, subprocess.CalledProcessError: Command 'docker run --rm --net none -v /home/frappe/benches/bench-0001-000002-appsvr1/config:/home/frappe/frappe-bench/configmount registry.digitalocean.com/test/ftest-cloud/cloud.testurl.net/bench-0001:deploy-0001-000002 cp -LR config/. configmount' returned non-zero exit status 1. cp -LR config/. configmount' returned non-zero exit status 1. }` ``` And the offending function is ``` @step("Initialize Bench") def bench_init(self, name, config): bench_directory = os.path.join(self.benches_directory, name) os.mkdir(bench_directory) directories = ["logs", "sites", "config"] for directory in directories: os.mkdir(os.path.join(bench_directory, directory)) bench_config_file = os.path.join(bench_directory, "config.json") with open(bench_config_file, "w") as f: json.dump(config, f, indent=1, sort_keys=True) config.update({"directory": bench_directory, "name": name}) docker_compose = os.path.join(bench_directory, "docker-compose.yml") self._render_template( "bench/docker-compose.yml.jinja2", config, docker_compose ) config_directory = os.path.join(bench_directory, "config") command = ( f"docker run --rm --net none " f"-v {config_directory}:/home/frappe/frappe-bench/configmount " f"{config['docker_image']} cp -LR config/. configmount" ) self.execute(command, directory=bench_directory) sites_directory = os.path.join(bench_directory, "sites") # Copy sites directory from image to host system command = ( f"docker run --rm --net none " f"-v {sites_directory}:/home/frappe/frappe-bench/sitesmount " f"{config['docker_image']} cp -LR sites/. sitesmount" ) return self.execute(command, directory=bench_directory) def dump(self): return { "name": self.name, "benches": { name: bench.dump() for name, bench in self.benches.items() }, "config": self.config, }` ``` The way to overcome this error is to modify this block like this: ``` ` uid = os.getuid() gid = os.getgid() config_directory = os.path.join(bench_directory, "config") command = ( f"docker run --rm --net none -u {uid}:{gid}" f"-v {config_directory}:/home/frappe/frappe-bench/configmount " f"{config['docker_image']} cp -LR config/. configmount" ) self.execute(command, directory=bench_directory) sites_directory = os.path.join(bench_directory, "sites") # Copy sites directory from image to host system command = ( f"docker run --rm --net none -u {uid}:{gid}" f"-v {sites_directory}:/home/frappe/frappe-bench/sitesmount " f"{config['docker_image']} cp -LR sites/. sitesmount" ) return self.execute(command, directory=bench_directory)`. ``` However, the relevant directories and files are not created even though the Initialize Bench job is then marked as Complete. Is this a unique situation or it's a bug someone needs to look into?