Hi sir,
I installed lanchain in my bench. I trying to summarize bunch of lines using langchain for that i installed.
After installing i got an error called no module named tiktoken, In pip list i already had tiktoken.
Error:
Traceback (most recent call last):
File “apps/frappe/frappe/handler.py”, line 75, in execute_cmd
method = get_attr(cmd)
File “apps/frappe/frappe/handler.py”, line 261, in get_attr
method = frappe.get_attr(cmd)
File “apps/frappe/frappe/init.py”, line 1581, in get_attr
return getattr(get_module(modulename), methodname)
File “apps/frappe/frappe/init.py”, line 1311, in get_module
return importlib.import_module(modulename)
File “/usr/lib/python3.10/importlib/init.py”, line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File “”, line 1050, in _gcd_import
File “”, line 1027, in _find_and_load
File “”, line 1006, in _find_and_load_unlocked
File “”, line 688, in _load_unlocked
File “”, line 883, in exec_module
File “”, line 241, in _call_with_frames_removed
File “apps/turiya_app/turiya_app/trial.py”, line 34, in
from tiktoken import Tokenizer
ModuleNotFoundError: No module named ‘tiktoken’
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “apps/frappe/frappe/app.py”, line 53, in application
response = frappe.api.handle()
File “apps/frappe/frappe/api.py”, line 53, in handle
return RESTAPIHandler(call, doctype, name).get_response()
File “apps/frappe/frappe/api.py”, line 69, in get_response
return self.handle_method()
File “apps/frappe/frappe/api.py”, line 79, in handle_method
return frappe.handler.handle()
File “apps/frappe/frappe/handler.py”, line 48, in handle
data = execute_cmd(cmd)
File “apps/frappe/frappe/handler.py”, line 77, in execute_cmd
frappe.throw((“Failed to get method for command {0} with {1}”).format(cmd, e))
File “apps/frappe/frappe/init.py”, line 523, in throw
msgprint(
File “apps/frappe/frappe/init.py”, line 491, in msgprint
_raise_exception()
File “apps/frappe/frappe/init.py”, line 440, in _raise_exception
raise raise_exception(msg)
frappe.exceptions.ValidationError: Failed to get method for command turiya_app.trial.arjunpdfread with No module named ‘tiktoken’
This is my code:
from PyPDF2 import PdfFileReader
from langchain.llms import OpenAI
from langchain.chains.summarize import load_summarize_chain
from langchain.docstore.document import Document
from tiktoken import Tokenizer
import frappe
import os
import re
@frappe.whitelist(allow_guest=True)
def arjunpdfread():
if ‘file’ in frappe.request.files:
file = frappe.request.files[‘file’]
# Read the PDF and extract data
text = read_pdf(file)
# Split the text into chunks
chunks = split_text_into_chunks(text)
# Summarize the chunks
summary = summarize_chunks(chunks)
return summary
else:
frappe.throw(_(“No file found”))
@frappe.whitelist(allow_guest=True)
def read_pdf(file):
pdf = PdfFileReader(file.stream)
text = ‘’
for page_num in range(pdf.getNumPages()):
page_obj = pdf.getPage(page_num)
text += page_obj.extractText()
return text
def split_text_into_chunks(text, chunk_size=200):
# Split the text into chunks of specified size
chunks = re.findall(‘.{1,%d}’ % chunk_size, text)
return chunks
def summarize_chunks(chunks):
# Set your OpenAI API key from Frappe configuration
os.environ[“OPENAI_API_KEY”] = frappe.conf.get(“openai_api_key”)
# Initialize the OpenAI module
llm = OpenAI()
# Load the summarize chain
chain = load_summarize_chain(llm, chain_type=“map_reduce”)
# Convert chunks to Document objects
docs = [Document(page_content=chunk) for chunk in chunks]
# Run the summarize chain on the documents
summary = chain.run(docs)
return summary
@frappe.whitelist(allow_guest=True)
def arjunpdfread():
if ‘file’ in frappe.request.files:
file = frappe.request.files[‘file’]
# Read the PDF and extract data
text = read_pdf(file)
# Clean the extracted text
cleaned_text = clean_data(text)
return cleaned_text
else:
frappe.throw(_(“No file found”))
@frappe.whitelist(allow_guest=True)
def read_pdf(file):
pdf = PdfFileReader(file.stream)
text = ‘’
for page_num in range(pdf.getNumPages()):
page_obj = pdf.getPage(page_num)
text += page_obj.extractText()
return text
def clean_data(text):
# Remove non-alphanumeric characters
text = re.sub(r’\W+', ’ ', text)
# Convert to lowercase
text = text.lower()
# Remove extra spaces
text = re.sub(' +', ' ', text)
# Strip leading and trailing spaces
text = text.strip()
return text
Many times if i install external package in bench it doesn’t recognize. How to solve.
Thankyou