For custom functions/endpoints you can use werkzeug Response object
import json
from werkzeug.wrappers import Response
@frappe.whitelist()
def my_custom_endpoint():
data = {"foo": "bar"}
response = Response(json.dumps(data), content_type='application/json')
response.status_code = 200
return response