OiO.lk Community platform!

Oio.lk is an excellent forum for developers, providing a wide range of resources, discussions, and support for those in the developer community. Join oio.lk today to connect with like-minded professionals, share insights, and stay updated on the latest trends and technologies in the development field.
  You need to log in or register to access the solved answers to this problem.
  • You have reached the maximum number of guest views allowed
  • Please register below to remove this limitation

Why is my JSON object not inserting correctly in ChromaDB using Langchain and Python?

  • Thread starter Thread starter Ken Tola
  • Start date Start date
K

Ken Tola

Guest
I am trying to enter a series of Topics into ChromaDB, those topics can be found here There are 35 total topics, 34 of which are unique. The topics are in valid JSON format - I can add and remove them from MongoDB, use JSON dumps and loads on them, and they were created from a langchain openai call where the output comes from a JSON formatter.

No matter what I try, however, only 7 of them are entered into ChromaDB AND trying to use a RecursiveJsonSplitter always results in an error.

Here are the two methods I am using (the write_object_to_prompt call simply removes all curly brackets and add in line breaks/indents):

Code:
def add_to_chroma(database_name: str, collection_name: str, json_object: json, user_directory: str, state, meta_data: json, doc_ids: list[str] = None):
    try:
        db_directory = os.path.join(user_directory, database_name + ".db")
        embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
        # embedding_function = OpenAIEmbeddings(model="text-embedding-3-small")
        chroma_db = Chroma(persist_directory=db_directory, collection_name=collection_name, embedding_function=embedding_function,
                           collection_metadata={"hnsw:space": "cosine"})
        # embed_object = write_object_to_prompt(json_object)
        embed_object = json_object
        # text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=100)
        json_splitter = RecursiveJsonSplitter(max_chunk_size=2000)
        json_docs = json_splitter.split_json(embed_object, True)
        meta_list = []
        for json_doc in json_docs:
            meta_list.append(meta_data)
        docs = json_splitter.create_documents(texts=json_docs, metadatas=meta_list)

        if doc_ids is None:
            doc_ids = [str(uuid.uuid4()) for i in range(1, len(docs) + 1)]
        else:
            # We look to see if the document exists:
            result = chroma_db.get(doc_ids)
            if result is not None and len(result) > 0:
                # This is an update:
                state["persistent_logs"].append("Updating " + meta_data["topic_id"] + " in Chroma")
                chroma_db.update_documents(doc_ids, docs)
                return doc_ids, state
        state["persistent_logs"].append("Adding " + meta_data["topic_id"] + " to Chroma")
        chroma_db.from_documents(docs, embedding_function, ids=doc_ids)
    except:
        trace_back = traceback.format_exc()
        logging.error("An unexpected error occurred attempting to add document to Chroma: " + database_name + ", to the collection: " + collection_name +
                      "\nHere is the document that failed: " + write_object_to_prompt(json_object) + " \nWith the error:\n " + trace_back)
        state["persistent_logs"].append(
            "An unexpected error occurred attempting to add document to Chroma: " + database_name + ", to the collection: " + collection_name +
            "\nHere is the document that failed: " + write_object_to_prompt(json_object) + " \nWith the error:\n " + trace_back)
    return doc_ids, state

Code:
def get_current_count(database_name: str, collection_name: str, user_directory: str) -> int:
    db_directory = os.path.join(user_directory, database_name + ".db")
    # embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
    embedding_function = OpenAIEmbeddings(model="text-embedding-3-small")
    # Cosine will keep te similarity scores between zero and one
    chroma_db = Chroma(persist_directory=db_directory, collection_name=collection_name, embedding_function=embedding_function,
                       collection_metadata={"hnsw:space": "cosine"})
    results = chroma_db.get()
    total_count = 0
    if results is not None:
        total_count = len(results)
    return total_count

And here is my testing script:

Code:
user_directory = "../UserData/user-x"
with open("Topics.txt", "r") as f:
    Topics = json.load(f)
print("Total topics: " + str(len(Topics)))
state = {
        "errors": "",
        "persistent_logs": [],
    }
unique_ids = []
for this_topic in Topics:
    meta = {"topic_id": this_topic["topic_id"]}
    doc_ids, state = add_to_chroma("user-x", "conversations", this_topic, user_directory, state, meta)
    if this_topic["topic_id"] not in unique_ids:
        unique_ids.append(this_topic["topic_id"])
print("Number of unique Ids: " + str(len(unique_ids)))

if len(state["persistent_logs"]) > 0:
    for log in state["persistent_logs"]:
        print(log)
print("Total Topics in Chroma " + str(get_current_count("user-x", "topics", user_directory)))

I never get any errors back from Chroma and the output shows 35 entries.

I have tried to use OpenAi embeddings, HuggingFace embeddings, JSON and text splitters, creating ChomraDB Documents manually. Entering the JSON as plaintext, entering it as JSON. Nothing is working.

Does anybody have any ideas?

Sorry - here are the includes:

Code:
import json
import uuid
from langchain_chroma import Chroma
from langchain.docstore.document import Document
from langchain_text_splitters import RecursiveCharacterTextSplitter, RecursiveJsonSplitter
from langchain_community.embeddings.sentence_transformer import SentenceTransformerEmbeddings
from langchain_openai import OpenAIEmbeddings
import logging
import os
import traceback
<p>I am trying to enter a series of Topics into ChromaDB, those topics <a href="https://www.dropbox.com/scl/fi/6bcs...ey=xfznwo7pwtrwixcs2cnwcqx1b&st=bwcpunmp&dl=0" rel="nofollow noreferrer">can be found here</a> There are 35 total topics, 34 of which are unique. The topics are in valid JSON format - I can add and remove them from MongoDB, use JSON dumps and loads on them, and they were created from a langchain openai call where the output comes from a JSON formatter.</p>
<p>No matter what I try, however, only 7 of them are entered into ChromaDB AND trying to use a RecursiveJsonSplitter always results in an error.</p>
<p>Here are the two methods I am using (the write_object_to_prompt call simply removes all curly brackets and add in line breaks/indents):</p>
<pre><code>def add_to_chroma(database_name: str, collection_name: str, json_object: json, user_directory: str, state, meta_data: json, doc_ids: list[str] = None):
try:
db_directory = os.path.join(user_directory, database_name + ".db")
embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
# embedding_function = OpenAIEmbeddings(model="text-embedding-3-small")
chroma_db = Chroma(persist_directory=db_directory, collection_name=collection_name, embedding_function=embedding_function,
collection_metadata={"hnsw:space": "cosine"})
# embed_object = write_object_to_prompt(json_object)
embed_object = json_object
# text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=100)
json_splitter = RecursiveJsonSplitter(max_chunk_size=2000)
json_docs = json_splitter.split_json(embed_object, True)
meta_list = []
for json_doc in json_docs:
meta_list.append(meta_data)
docs = json_splitter.create_documents(texts=json_docs, metadatas=meta_list)

if doc_ids is None:
doc_ids = [str(uuid.uuid4()) for i in range(1, len(docs) + 1)]
else:
# We look to see if the document exists:
result = chroma_db.get(doc_ids)
if result is not None and len(result) > 0:
# This is an update:
state["persistent_logs"].append("Updating " + meta_data["topic_id"] + " in Chroma")
chroma_db.update_documents(doc_ids, docs)
return doc_ids, state
state["persistent_logs"].append("Adding " + meta_data["topic_id"] + " to Chroma")
chroma_db.from_documents(docs, embedding_function, ids=doc_ids)
except:
trace_back = traceback.format_exc()
logging.error("An unexpected error occurred attempting to add document to Chroma: " + database_name + ", to the collection: " + collection_name +
"\nHere is the document that failed: " + write_object_to_prompt(json_object) + " \nWith the error:\n " + trace_back)
state["persistent_logs"].append(
"An unexpected error occurred attempting to add document to Chroma: " + database_name + ", to the collection: " + collection_name +
"\nHere is the document that failed: " + write_object_to_prompt(json_object) + " \nWith the error:\n " + trace_back)
return doc_ids, state
</code></pre>
<pre><code>def get_current_count(database_name: str, collection_name: str, user_directory: str) -> int:
db_directory = os.path.join(user_directory, database_name + ".db")
# embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
embedding_function = OpenAIEmbeddings(model="text-embedding-3-small")
# Cosine will keep te similarity scores between zero and one
chroma_db = Chroma(persist_directory=db_directory, collection_name=collection_name, embedding_function=embedding_function,
collection_metadata={"hnsw:space": "cosine"})
results = chroma_db.get()
total_count = 0
if results is not None:
total_count = len(results)
return total_count
</code></pre>
<p>And here is my testing script:</p>
<pre><code>user_directory = "../UserData/user-x"
with open("Topics.txt", "r") as f:
Topics = json.load(f)
print("Total topics: " + str(len(Topics)))
state = {
"errors": "",
"persistent_logs": [],
}
unique_ids = []
for this_topic in Topics:
meta = {"topic_id": this_topic["topic_id"]}
doc_ids, state = add_to_chroma("user-x", "conversations", this_topic, user_directory, state, meta)
if this_topic["topic_id"] not in unique_ids:
unique_ids.append(this_topic["topic_id"])
print("Number of unique Ids: " + str(len(unique_ids)))

if len(state["persistent_logs"]) > 0:
for log in state["persistent_logs"]:
print(log)
print("Total Topics in Chroma " + str(get_current_count("user-x", "topics", user_directory)))
</code></pre>
<p>I never get any errors back from Chroma and the output shows 35 entries.</p>
<p>I have tried to use OpenAi embeddings, HuggingFace embeddings, JSON and text splitters, creating ChomraDB Documents manually. Entering the JSON as plaintext, entering it as JSON. Nothing is working.</p>
<p>Does anybody have any ideas?</p>
<p>Sorry - here are the includes:</p>
<pre><code>import json
import uuid
from langchain_chroma import Chroma
from langchain.docstore.document import Document
from langchain_text_splitters import RecursiveCharacterTextSplitter, RecursiveJsonSplitter
from langchain_community.embeddings.sentence_transformer import SentenceTransformerEmbeddings
from langchain_openai import OpenAIEmbeddings
import logging
import os
import traceback
</code></pre>
 

Online statistics

Members online
0
Guests online
4
Total visitors
4
Top