You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hugegraph.apache.org by ji...@apache.org on 2024/09/05 11:45:05 UTC
(incubator-hugegraph-ai) 01/01: refact: fix some bugs & query logic
This is an automated email from the ASF dual-hosted git repository.
jin pushed a commit to branch graphdata
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-ai.git
commit 7f671b625e1685ce2b855c0dba9360d52dc3ef51
Author: imbajin <ji...@apache.org>
AuthorDate: Thu Sep 5 19:09:44 2024 +0800
refact: fix some bugs & query logic
bugfix
---
.../src/hugegraph_llm/demo/rag_web_demo.py | 14 ++++----
.../operators/hugegraph_op/graph_rag_query.py | 40 ++++++++++++++--------
.../operators/hugegraph_op/schema_manager.py | 6 ++--
.../operators/index_op/semantic_id_query.py | 2 ++
.../operators/llm_op/answer_synthesize.py | 21 +++++++-----
.../operators/llm_op/disambiguate_data.py | 2 +-
.../src/hugegraph_llm/utils/decorators.py | 2 +-
.../src/hugegraph_llm/utils/graph_index_utils.py | 6 ++--
8 files changed, 57 insertions(+), 36 deletions(-)
diff --git a/hugegraph-llm/src/hugegraph_llm/demo/rag_web_demo.py b/hugegraph-llm/src/hugegraph_llm/demo/rag_web_demo.py
index 3cedc50..ceb2475 100644
--- a/hugegraph-llm/src/hugegraph_llm/demo/rag_web_demo.py
+++ b/hugegraph-llm/src/hugegraph_llm/demo/rag_web_demo.py
@@ -529,12 +529,14 @@ def init_rag_ui() -> gr.Interface:
input_file = gr.File(
value=[os.path.join(resource_path, "demo", "test.txt")],
label="Docs (multi-files can be selected together)",
- file_count="multiple"
+ file_count="multiple",
)
with gr.Tab("text") as tab_upload_text:
- input_text = gr.Textbox(value="", label="Doc(s)")
- input_schema = gr.Textbox(value=schema, label="Schema")
- info_extract_template = gr.Textbox(value=SCHEMA_EXAMPLE_PROMPT, label="Info extract head")
+ input_text = gr.Textbox(value="", label="Doc(s)", lines=20, show_copy_button=True)
+ input_schema = gr.Textbox(value=schema, label="Schema", lines=15, show_copy_button=True)
+ info_extract_template = gr.Textbox(value=SCHEMA_EXAMPLE_PROMPT, label="Info extract head", lines=15,
+ show_copy_button=True)
+ out = gr.Textbox(label="Output", lines=15, show_copy_button=True)
with gr.Column(visible=False, elem_classes="modal-box") as preview_box:
with gr.Row():
import_btn = gr.Button("Import Into Graph")
@@ -552,8 +554,8 @@ def init_rag_ui() -> gr.Interface:
graph_index_btn1 = gr.Button("Clear Graph Index")
graph_index_btn2 = gr.Button("Extract Graph", variant="primary")
graph_index_btn3 = gr.Button("Fit Vid Index")
- with gr.Row():
- out = gr.Textbox(label="Output", show_copy_button=True)
+ # with gr.Row():
+ # out = gr.Textbox(label="Output", show_copy_button=True)
vector_index_btn0.click(get_vector_index_info, outputs=out) # pylint: disable=no-member
vector_index_btn1.click(clean_vector_index) # pylint: disable=no-member
vector_index_btn2.click(build_vector_index, inputs=[input_file, input_text], outputs=out) # pylint: disable=no-member
diff --git a/hugegraph-llm/src/hugegraph_llm/operators/hugegraph_op/graph_rag_query.py b/hugegraph-llm/src/hugegraph_llm/operators/hugegraph_op/graph_rag_query.py
index 1cb7c17..f4229b6 100644
--- a/hugegraph-llm/src/hugegraph_llm/operators/hugegraph_op/graph_rag_query.py
+++ b/hugegraph-llm/src/hugegraph_llm/operators/hugegraph_op/graph_rag_query.py
@@ -20,6 +20,7 @@ import re
from typing import Any, Dict, Optional, List, Set, Tuple
from hugegraph_llm.config import settings
+from hugegraph_llm.utils.log import log
from pyhugegraph.client import PyHugeClient
@@ -117,37 +118,48 @@ class GraphRAGQuery:
edge_labels_str = ",".join("'" + label + "'" for label in edge_labels)
use_id_to_match = self._prop_to_match is None
- if not use_id_to_match:
- assert keywords is not None, "No keywords for graph query."
- keywords_str = ",".join("'" + kw + "'" for kw in keywords)
- gremlin_query = PROPERTY_QUERY_NEIGHBOR_TPL.format(
- prop=self._prop_to_match,
- keywords=keywords_str,
+ if use_id_to_match:
+ if not entrance_vids: return context
+
+ gremlin_query = VERTEX_QUERY_TPL.format(keywords=entrance_vids)
+ result: List[Any] = self._client.gremlin().exec(gremlin=gremlin_query)["data"]
+ log.debug(f"Vids query: {gremlin_query}")
+
+ vertex_knowledge = self._format_graph_from_vertex(query_result=result)
+ gremlin_query = ID_QUERY_NEIGHBOR_TPL.format(
+ keywords=entrance_vids,
max_deep=self._max_deep,
max_items=self._max_items,
edge_labels=edge_labels_str,
)
+ log.debug(f"Kneighbor query: {gremlin_query}")
+
result: List[Any] = self._client.gremlin().exec(gremlin=gremlin_query)["data"]
graph_chain_knowledge, vertex_degree_list, knowledge_with_degree = self._format_graph_from_query_result(
query_result=result
)
+ graph_chain_knowledge.update(vertex_knowledge)
+ if vertex_degree_list:
+ vertex_degree_list[0].update(vertex_knowledge)
+ else:
+ vertex_degree_list.append(vertex_knowledge)
else:
- assert entrance_vids is not None, "No entrance vertices for query."
- gremlin_query = VERTEX_QUERY_TPL.format(keywords=entrance_vids)
- result: List[Any] = self._client.gremlin().exec(gremlin=gremlin_query)["data"]
- vertex_knowledge = self._format_graph_from_vertex(query_result=result)
- gremlin_query = ID_QUERY_NEIGHBOR_TPL.format(
- keywords=entrance_vids,
+ # When will the query enter here?
+ assert keywords, "No related property(keywords) for graph query."
+ keywords_str = ",".join("'" + kw + "'" for kw in keywords)
+ gremlin_query = PROPERTY_QUERY_NEIGHBOR_TPL.format(
+ prop=self._prop_to_match,
+ keywords=keywords_str,
max_deep=self._max_deep,
max_items=self._max_items,
edge_labels=edge_labels_str,
)
+ log.warning("Unable to find vid, downgraded to property query, please confirm if it meets expectation.")
+
result: List[Any] = self._client.gremlin().exec(gremlin=gremlin_query)["data"]
graph_chain_knowledge, vertex_degree_list, knowledge_with_degree = self._format_graph_from_query_result(
query_result=result
)
- graph_chain_knowledge.update(vertex_knowledge)
- vertex_degree_list[0].update(vertex_knowledge)
context["graph_result"] = list(graph_chain_knowledge)
context["vertex_degree_list"] = [list(vertex_degree) for vertex_degree in vertex_degree_list]
diff --git a/hugegraph-llm/src/hugegraph_llm/operators/hugegraph_op/schema_manager.py b/hugegraph-llm/src/hugegraph_llm/operators/hugegraph_op/schema_manager.py
index 5c002ae..a215bb4 100644
--- a/hugegraph-llm/src/hugegraph_llm/operators/hugegraph_op/schema_manager.py
+++ b/hugegraph-llm/src/hugegraph_llm/operators/hugegraph_op/schema_manager.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+from typing import Dict, Any
from hugegraph_llm.config import settings
from pyhugegraph.client import PyHugeClient
@@ -33,7 +33,9 @@ class SchemaManager:
)
self.schema = self.client.schema()
- def run(self):
+ # FIXME: This method is not working as expected
+ # def run(self, context: Dict[str, Any]) -> Dict[str, Any]:
+ def run(self) -> Dict[str, Any]:
schema = self.schema.getSchema()
vertices = []
for vl in schema["vertexlabels"]:
diff --git a/hugegraph-llm/src/hugegraph_llm/operators/index_op/semantic_id_query.py b/hugegraph-llm/src/hugegraph_llm/operators/index_op/semantic_id_query.py
index ef036d2..c3fa560 100644
--- a/hugegraph-llm/src/hugegraph_llm/operators/index_op/semantic_id_query.py
+++ b/hugegraph-llm/src/hugegraph_llm/operators/index_op/semantic_id_query.py
@@ -54,5 +54,7 @@ class SemanticIdQuery:
results = self.vector_index.search(keyword_vector, top_k=self.topk_per_keyword)
if results:
graph_query_entrance.extend(results[:self.topk_per_keyword])
+ from hugegraph_llm.utils.log import log
context["entrance_vids"] = list(set(graph_query_entrance))
+ log.debug(f"graph_query_entrance: {graph_query_entrance}")
return context
diff --git a/hugegraph-llm/src/hugegraph_llm/operators/llm_op/answer_synthesize.py b/hugegraph-llm/src/hugegraph_llm/operators/llm_op/answer_synthesize.py
index 2cb7264..fbba4d6 100644
--- a/hugegraph-llm/src/hugegraph_llm/operators/llm_op/answer_synthesize.py
+++ b/hugegraph-llm/src/hugegraph_llm/operators/llm_op/answer_synthesize.py
@@ -21,6 +21,7 @@ from typing import Any, Dict, Optional
from hugegraph_llm.models.llms.base import BaseLLM
from hugegraph_llm.models.llms.init_llm import LLMs
+from hugegraph_llm.utils.log import log
# TODO: we need enhance the template to answer the question (put it in a separate file)
DEFAULT_ANSWER_TEMPLATE = """You are an expert in knowledge graphs and natural language processing.
@@ -88,25 +89,27 @@ class AnswerSynthesize:
response = self._llm.generate(prompt=prompt)
return {"answer": response}
- vector_result = context.get("vector_result", [])
- if len(vector_result) == 0:
- vector_result_context = "No (vector)phrase related to the query."
- else:
+ vector_result = context.get("vector_result")
+ if vector_result:
vector_result_context = "Phrases related to the query:\n" + "\n".join(
f"{i + 1}. {res}" for i, res in enumerate(vector_result)
)
- graph_result = context.get("graph_result", [])
- if len(graph_result) == 0:
- graph_result_context = "No knowledge found in HugeGraph for the query."
else:
+ vector_result_context = "No (vector)phrase related to the query."
+
+ graph_result = context.get("graph_result")
+ if graph_result:
graph_context_head = context.get("graph_context_head",
- "The following are knowledge from HugeGraph related to the query:\n")
+ "Knowledge from graph related to the query:\n")
graph_result_context = graph_context_head + "\n".join(
f"{i + 1}. {res}" for i, res in enumerate(graph_result)
)
+ else:
+ graph_result_context = "No related graph data found for current query."
+ log.info(graph_result_context)
+
context = asyncio.run(self.async_generate(context, context_head_str, context_tail_str,
vector_result_context, graph_result_context))
-
return context
async def async_generate(self, context: Dict[str, Any], context_head_str: str,
diff --git a/hugegraph-llm/src/hugegraph_llm/operators/llm_op/disambiguate_data.py b/hugegraph-llm/src/hugegraph_llm/operators/llm_op/disambiguate_data.py
index fffa03c..dcc0ab6 100644
--- a/hugegraph-llm/src/hugegraph_llm/operators/llm_op/disambiguate_data.py
+++ b/hugegraph-llm/src/hugegraph_llm/operators/llm_op/disambiguate_data.py
@@ -46,7 +46,7 @@ class DisambiguateData:
# only disambiguate triples
if "triples" in data:
# TODO: ensure the logic here
- log.debug(data)
+ # log.debug(data)
triples = data["triples"]
prompt = generate_disambiguate_prompt(triples)
llm_output = self.llm.generate(prompt=prompt)
diff --git a/hugegraph-llm/src/hugegraph_llm/utils/decorators.py b/hugegraph-llm/src/hugegraph_llm/utils/decorators.py
index 173c1f7..8cbe475 100644
--- a/hugegraph-llm/src/hugegraph_llm/utils/decorators.py
+++ b/hugegraph-llm/src/hugegraph_llm/utils/decorators.py
@@ -72,7 +72,7 @@ def log_operator_time(func: Callable) -> Callable:
# Only record time ≥ 0.01s (10ms)
if op_time >= 0.01:
log.debug("Operator %s finished in %.2f seconds", operator.__class__.__name__, op_time)
- log.debug("Context:\n%s", result)
+ log.debug("Current context: %s", result)
return result
return wrapper
diff --git a/hugegraph-llm/src/hugegraph_llm/utils/graph_index_utils.py b/hugegraph-llm/src/hugegraph_llm/utils/graph_index_utils.py
index 0db406e..10bf72e 100644
--- a/hugegraph-llm/src/hugegraph_llm/utils/graph_index_utils.py
+++ b/hugegraph-llm/src/hugegraph_llm/utils/graph_index_utils.py
@@ -64,7 +64,7 @@ def extract_graph(input_file, input_text, schema, example_prompt):
(builder
.chunk_split(texts, "paragraph", "zh")
.extract_info(example_prompt, "property_graph"))
- log.debug(builder.operators)
+ log.debug("Operators: %s", builder.operators)
try:
context = builder.run()
return (
@@ -80,7 +80,7 @@ def extract_graph(input_file, input_text, schema, example_prompt):
def fit_vid_index():
builder = KgBuilder(LLMs().get_llm(), Embeddings().get_embedding(), get_hg_client())
builder.fetch_graph_data().build_vertex_id_semantic_index()
- log.debug(builder.operators)
+ log.debug("Operators: %s", builder.operators)
try:
context = builder.run()
removed_num = context["removed_vid_vector_num"]
@@ -109,7 +109,7 @@ def build_graph_index(input_file, input_text, schema, example_prompt):
.extract_info(example_prompt, "property_graph")
.commit_to_hugegraph()
.build_vertex_id_semantic_index())
- log.debug(builder.operators)
+ log.debug("Operators: %s", builder.operators)
try:
context = builder.run()
return json.dumps(context, ensure_ascii=False, indent=2)