Intermittent Postgres Concurrency Issue in frappe.desk.form.assign_to.remove during Workflow action

We are facing intermittent issues when changing the assignee of a particular document using workflow actions (approve/reject). Specifically, we’re using the frappe.desk.form.assign_to.remove method in our custom client script to remove the assignee of the document. However, this often results in a PostgreSQL concurrency issue.

Details:

  • We investigated our database activity but didn’t find any concurrent locks on the table.
  • Using Frappe’s recording feature, we analyzed the SQL queries executed during the /api/method/frappe.desk.form.assign_to.remove API call. The logs indicated that the update query failed due to concurrent updates, causing a rollback.

Error Traceback :

Traceback with variables (most recent call last):
  File "apps/frappe/frappe/app.py", line 115, in application
    response = frappe.api.handle(request)
      request = <Request 'http://frappe.domain.co/api/method/frappe.desk.form.assign_to.remove' [POST]>
      response = None
      rollback = True
      e = SerializationFailure('could not serialize access due to concurrent update\n')
  File "apps/frappe/frappe/api/__init__.py", line 49, in handle
    data = endpoint(**arguments)
      request = <Request 'http://frappe.domain.co/api/method/frappe.desk.form.assign_to.remove' [POST]>
      endpoint = <function handle_rpc_call at 0x7fddaa06a700>
      arguments = {'method': 'frappe.desk.form.assign_to.remove'}
  File "apps/frappe/frappe/api/v1.py", line 36, in handle_rpc_call
    return frappe.handler.handle()
      method = 'frappe.desk.form.assign_to.remove'
      frappe = <module 'frappe' from 'apps/frappe/frappe/__init__.py'>
  File "apps/frappe/frappe/handler.py", line 49, in handle
    data = execute_cmd(cmd)
      cmd = 'frappe.desk.form.assign_to.remove'
      data = None
  File "apps/frappe/frappe/handler.py", line 85, in execute_cmd
    return frappe.call(method, **frappe.form_dict)
      cmd = 'frappe.desk.form.assign_to.remove'
      from_async = False
      server_script = None
      method = <function remove at 0x7fdda454bba0>
  File "apps/frappe/frappe/__init__.py", line 1768, in call
    return fn(*args, **newargs)
      fn = <function remove at 0x7fdda454bba0>
      args = ()
      kwargs = {'doctype': 'vmsTripv2', 'name': 'Bahraich_04_10_24_0987', 'assign_to': 'devesh.kumar@domain.in', 'ignore_permissions': 'true', 'cmd': 'frappe.desk.form.assign_to.remove'}
      newargs = {'doctype': 'vmsTripv2', 'name': 'Bahraich_04_10_24_0987', 'assign_to': 'devesh.kumar@domain.in'}
  File "apps/frappe/frappe/utils/typing_validations.py", line 31, in wrapper
    return func(*args, **kwargs)
      args = ()
      kwargs = {'doctype': 'vmsTripv2', 'name': 'Bahraich_04_10_24_0987', 'assign_to': 'devesh.kumar@domain.in'}
      apply_condition = <function whitelist.<locals>.innerfn.<locals>.<lambda> at 0x7fdda454bb00>
      func = <function remove at 0x7fdda454ba60>
  File "apps/frappe/frappe/desk/form/assign_to.py", line 175, in remove
    return set_status(doctype, name, "", assign_to, status="Cancelled", ignore_permissions=ignore_permissions)
      doctype = 'vmsTripv2'
      name = 'Bahraich_04_10_24_0987'
      assign_to = 'devesh.kumar@domain.in'
      ignore_permissions = False
  File "apps/frappe/frappe/desk/form/assign_to.py", line 205, in set_status
    todo.save(ignore_permissions=True)
      doctype = 'vmsTripv2'
      name = 'Bahraich_04_10_24_0987'
      todo = <ToDo: cke35ui8qi>
      assign_to = 'devesh.kumar@domain.in'
      status = 'Cancelled'
      ignore_permissions = False
  File "apps/frappe/frappe/model/document.py", line 337, in save
    return self._save(*args, **kwargs)
      self = <ToDo: cke35ui8qi>
      args = ()
      kwargs = {'ignore_permissions': True}
  File "apps/frappe/frappe/model/document.py", line 390, in _save
    self.run_post_save_methods()
      self = <ToDo: cke35ui8qi>
      ignore_permissions = True
      ignore_version = None
  File "apps/frappe/frappe/model/document.py", line 1128, in run_post_save_methods
    self.run_method("on_update")
      self = <ToDo: cke35ui8qi>
  File "apps/frappe/frappe/model/document.py", line 962, in run_method
    out = Document.hook(fn)(self, *args, **kwargs)
      self = <ToDo: cke35ui8qi>
      method = 'on_update'
      args = ()
      kwargs = {}
      fn = <function Document.run_method.<locals>.fn at 0x7fdd9c588fe0>
  File "apps/frappe/frappe/model/document.py", line 1322, in composer
    return composed(self, method, *args, **kwargs)
      self = <ToDo: cke35ui8qi>
      args = ()
      kwargs = {}
      hooks = [<function clear_doctype_notifications at 0x7fdda8395c60>, <function process_workflow_actions at 0x7fdda45494e0>, <function attach_files_to_document at 0x7fddaab3a200>, <function apply at 0x7fdda4494c20>, <function update_due_date at 0x7fdda4494cc0>, <function apply_permissions_for_non_standard_user_type at 0x7fdda83954e0>]
      method = 'on_update'
      doc_events = {'*': {'on_update': ['frappe.desk.notifications.clear_doctype_notifications', 'frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions', 'frappe.core.doctype.file.utils.attach_files_to_document', 'frappe.automation.doctype.assignment_rule.assignment_rule.apply', 'frappe.automation.doctype.assignment_rule.assignment_rule.update_due_date', 'frappe.core.doctype.user_type.user_type.apply_permissions_for_non_standard_user_type'], 'after_rename': ['frappe.desk.notifications.clear_doctype_notifications'], 'on_cancel': ['frappe.desk.notifications.clear_doctype_notifications', 'frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions', 'frappe.automation.doctype.assignment_rule.assignment_rule.apply'], 'on_trash': ['frappe.desk.notifications.clear_doctype_notifications', 'frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions'], 'on_update_after_submit': ['frappe.workflow.doctype.workflow_action.workflow_action.process_w...
      handler = 'frappe.core.doctype.user_type.user_type.apply_permissions_for_non_standard_user_type'
      composed = <function Document.hook.<locals>.compose.<locals>.runner at 0x7fdd9c58b600>
      compose = <function Document.hook.<locals>.compose at 0x7fdd9c588cc0>
      f = <function Document.run_method.<locals>.fn at 0x7fdd9c588fe0>
  File "apps/frappe/frappe/model/document.py", line 1304, in runner
    add_to_return_value(self, fn(self, *args, **kwargs))
      self = <ToDo: cke35ui8qi>
      method = 'on_update'
      args = ()
      kwargs = {}
      add_to_return_value = <function Document.hook.<locals>.add_to_return_value at 0x7fdd9c58ab60>
      fn = <function Document.run_method.<locals>.fn at 0x7fdd9c588fe0>
      hooks = (<function clear_doctype_notifications at 0x7fdda8395c60>, <function process_workflow_actions at 0x7fdda45494e0>, <function attach_files_to_document at 0x7fddaab3a200>, <function apply at 0x7fdda4494c20>, <function update_due_date at 0x7fdda4494cc0>, <function apply_permissions_for_non_standard_user_type at 0x7fdda83954e0>)
  File "apps/frappe/frappe/model/document.py", line 959, in fn
    return method_object(*args, **kwargs)
      self = <ToDo: cke35ui8qi>
      args = ()
      kwargs = {}
      method_object = <bound method ToDo.on_update of <ToDo: cke35ui8qi>>
      method = 'on_update'
  File "apps/frappe/frappe/desk/doctype/todo/todo.py", line 71, in on_update
    self.update_in_reference()
      self = <ToDo: cke35ui8qi>
  File "apps/frappe/frappe/desk/doctype/todo/todo.py", line 112, in update_in_reference
    frappe.db.set_value(
      self = <ToDo: cke35ui8qi>
      assignments = ['abhishek.chaurasia@domain.in']
  File "apps/frappe/frappe/database/database.py", line 994, in set_value
    query.run(debug=debug)
      self = <frappe.database.postgres.database.PostgresDatabase object at 0x7fdda67c05d0>
      dt = 'vmsTripv2'
      dn = 'Bahraich_04_10_24_0987'
      field = '_assign'
      val = '["abhishek.chaurasia@domain.in"]'
      modified = None
      modified_by = None
      update_modified = False
      debug = False
      is_single_doctype = <function is_single_doctype at 0x7fddaab58400>
      to_update = {'_assign': '["abhishek.chaurasia@domain.in"]'}
      query = UPDATE "tabvmsTripv2" SET "_assign"='["abhishek.chaurasia@domain.in"]' WHERE "name"='Bahraich_04_10_24_0987'
      column = '_assign'
      value = '["abhishek.chaurasia@domain.in"]'
  File "apps/frappe/frappe/query_builder/utils.py", line 87, in execute_query
    result = frappe.db.sql(query, params, *args, **kwargs)  # nosemgrep
      query = 'UPDATE "tabvmsTripv2" SET "_assign"=%(param1)s WHERE "name"=%(param2)s'
      args = ()
      kwargs = {'debug': False}
      child_queries = []
      params = {'param1': '["abhishek.chaurasia@domain.in"]', 'param2': 'Bahraich_04_10_24_0987'}
      execute_child_queries = <function patch_query_execute.<locals>.execute_child_queries at 0x7fdda41cd940>
      prepare_query = <function patch_query_execute.<locals>.prepare_query at 0x7fdda41cf380>
  File "apps/frappe/frappe/recorder.py", line 59, in record_sql
    result = frappe.db._sql(*args, **kwargs)
      args = ('UPDATE "tabvmsTripv2" SET "_assign"=%(param1)s WHERE "name"=%(param2)s', {'param1': '["abhishek.chaurasia@domain.in"]', 'param2': 'Bahraich_04_10_24_0987'})
      kwargs = {'debug': False}
      start_time = 86551.913561766
  File "apps/frappe/frappe/database/postgres/database.py", line 213, in sql
    return super().sql(modify_query(query), modify_values(values), *args, **kwargs)
      self = <frappe.database.postgres.database.PostgresDatabase object at 0x7fdda67c05d0>
      query = 'UPDATE "tabvmsTripv2" SET "_assign"=%(param1)s WHERE "name"=%(param2)s'
      values = {'param1': '["abhishek.chaurasia@domain.in"]', 'param2': 'Bahraich_04_10_24_0987'}
      args = ()
      kwargs = {'debug': False}
      __class__ = <class 'frappe.database.postgres.database.PostgresDatabase'>
  File "apps/frappe/frappe/database/database.py", line 234, in sql
    self._cursor.execute(query, values)
      self = <frappe.database.postgres.database.PostgresDatabase object at 0x7fdda67c05d0>
      query = 'UPDATE "tabvmsTripv2" SET "_assign"=%(param1)s WHERE "name"=%(param2)s'
      values = {'param1': '["abhishek.chaurasia@domain.in"]', 'param2': 'Bahraich_04_10_24_0987'}
      as_dict = 0
      as_list = 0
      debug = False
      ignore_ddl = 0
      auto_commit = 0
      update = None
      explain = False
      run = True
      pluck = False
      as_iterator = False
      trace_id = None
psycopg2.errors.SerializationFailure: could not serialize access due to concurrent update

Pls suggest. @NCP @Abdeali @peterg

Hello,

Don’t know the actual reason(code), causing this issue.

Have you modified your database configuration?

Or In your custom query have you used for_update?

Also refer this stack overflow thread: