After one successful deploy and site creation, when restring to deploy an update to the bench, new bench, or make another site agent jobs is stuck forever. I have checked the error log nothing is there.
1 Like
@buildwithhussain @ankush
Press Instance is running on Ubuntu 22.04.3 LTS with frappe 15 and Python 3.10.12, other Servers (n,f,m) running on Ubuntu 20.04.4 LTS, when I go to /home/frappe/agent and see jobs.sqlite3 I got success messages like:
Initialize BenchSuccess{“command”: "docker run --rm --net none -v /home/frappe/benches/bench-0001-000001-f1 "…etc.
But when I go to logs and see the content of worker.error.log I get:
Exception in thread Thread-1:
Traceback (most recent call last):
File “/usr/lib/python3.8/threading.py”, line 932, in _bootstrap_inner
self.run()
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 1151, in run
pubsub.get_message(ignore_subscribe_messages=True, timeout=sleep_time)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 1013, in get_message
response = self.parse_response(block=(timeout is None), timeout=timeout)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 824, in parse_response
response = self._execute(conn, try_read)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 800, in _execute
return conn.retry.call_with_retry(
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/retry.py”, line 49, in call_with_retry
fail(error)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 802, in
lambda error: self._disconnect_raise_connect(conn, error),
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 789, in _disconnect_raise_connect
raise error
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/retry.py”, line 46, in call_with_retry
return do()
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 801, in
lambda: command(*args, **kwargs),
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 818, in try_read
if not conn.can_read(timeout=timeout):
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/connection.py”, line 478, in can_read
return self._parser.can_read(timeout)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/_parsers/base.py”, line 128, in can_read
return self._buffer and self._buffer.can_read(timeout)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/_parsers/socket.py”, line 95, in can_read
return bool(self.unread_bytes()) or self._read_from_socket(
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/_parsers/socket.py”, line 68, in _read_from_socket
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
redis.exceptions.ConnectionError: Connection closed by server.
20:30:18 Could not connect to Redis instance: Connection closed by server. Retrying in 1 seconds…
Exception in thread Thread-1:
Traceback (most recent call last):
File “/usr/lib/python3.8/threading.py”, line 932, in _bootstrap_inner
20:30:18 Could not connect to Redis instance: Connection closed by server. Retrying in 1 seconds…
Exception in thread Thread-1:
Traceback (most recent call last):
File “/usr/lib/python3.8/threading.py”, line 932, in _bootstrap_inner
self.run()
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 1151, in run
self.run()
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 1151, in run
pubsub.get_message(ignore_subscribe_messages=True, timeout=sleep_time)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 1013, in get_message
response = self.parse_response(block=(timeout is None), timeout=timeout)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 824, in parse_response
response = self._execute(conn, try_read)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 800, in _execute
return conn.retry.call_with_retry(
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/retry.py”, line 49, in call_with_retry
pubsub.get_message(ignore_subscribe_messages=True, timeout=sleep_time)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 1013, in get_message
response = self.parse_response(block=(timeout is None), timeout=timeout)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 824, in parse_response
response = self._execute(conn, try_read)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 800, in _execute
return conn.retry.call_with_retry(
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/retry.py”, line 49, in call_with_retry
fail(error)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 802, in
fail(error)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 802, in
lambda error: self._disconnect_raise_connect(conn, error),
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 789, in _disconnect_raise_connect
raise error
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/retry.py”, line 46, in call_with_retry
lambda error: self._disconnect_raise_connect(conn, error),
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 789, in _disconnect_raise_connect
raise error
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/retry.py”, line 46, in call_with_retry
return do()
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 801, in
lambda: command(*args, **kwargs),
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 818, in try_read
if not conn.can_read(timeout=timeout):
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/connection.py”, line 478, in can_read
return do()
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 801, in
lambda: command(*args, **kwargs),
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/client.py”, line 818, in try_read
if not conn.can_read(timeout=timeout):
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/connection.py”, line 478, in can_read
return self._parser.can_read(timeout)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/_parsers/base.py”, line 128, in can_read
return self._parser.can_read(timeout)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/_parsers/base.py”, line 128, in can_read
return self._buffer and self._buffer.can_read(timeout)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/_parsers/socket.py”, line 95, in can_read
return self._buffer and self._buffer.can_read(timeout)
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/_parsers/socket.py”, line 95, in can_read
return bool(self.unread_bytes()) or self._read_from_socket(
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/_parsers/socket.py”, line 68, in _read_from_socket
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
redis.exceptions.ConnectionError: Connection closed by server.
return bool(self.unread_bytes()) or self._read_from_socket(
File “/home/frappe/agent/env/lib/python3.8/site-packages/redis/_parsers/socket.py”, line 68, in _read_from_socket
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
redis.exceptions.ConnectionError: Connection closed by server.
I have to manually make agent job sucess in order to get the deploy working, how could we fix that?
1 Like
This looks like redis issue. Check if your redis workers are running.
Please help in this regard, same issue is with my setup