You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@answer.apache.org by li...@apache.org on 2024/09/03 02:47:11 UTC

(incubator-answer-plugins) 01/02: Add sync to Elasticsearch

This is an automated email from the ASF dual-hosted git repository.

linkinstar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git

commit 70bb63de0773550d70c1ca162e340958e9c84cf4
Author: sy-records <52...@qq52o.cn>
AuthorDate: Sat Aug 31 16:44:32 2024 +0800

    Add sync to Elasticsearch
---
 search-elasticsearch/es.go   |  2 +-
 search-elasticsearch/sync.go | 84 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/search-elasticsearch/es.go b/search-elasticsearch/es.go
index 8ea9a8a..d04c732 100644
--- a/search-elasticsearch/es.go
+++ b/search-elasticsearch/es.go
@@ -143,7 +143,7 @@ func (s *SearchEngine) DeleteContent(ctx context.Context, contentID string) erro
 
 func (s *SearchEngine) RegisterSyncer(ctx context.Context, syncer plugin.SearchSyncer) {
 	s.syncer = syncer
-	// TODO: Synchronization of already existing data through some strategy
+	s.sync()
 }
 
 func (s *SearchEngine) warpResult(resp *elastic.SearchResult) ([]plugin.SearchResult, int64, error) {
diff --git a/search-elasticsearch/sync.go b/search-elasticsearch/sync.go
new file mode 100644
index 0000000..76a137f
--- /dev/null
+++ b/search-elasticsearch/sync.go
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package es
+
+import (
+	"context"
+	"github.com/apache/incubator-answer/plugin"
+	"github.com/segmentfault/pacman/log"
+)
+
+func (s *SearchEngine) sync() {
+	var page, pageSize = 1, 100
+	go func() {
+		log.Info("es: start sync questions...")
+		page = 1
+		for {
+			log.Infof("es: sync question page %d, page size %d", page, pageSize)
+			questionList, err := s.syncer.GetQuestionsPage(context.TODO(), page, pageSize)
+			if err != nil {
+				log.Error("es: sync questions error", err)
+				break
+			}
+			if len(questionList) == 0 {
+				break
+			}
+			err = s.batchUpdateContent(context.TODO(), questionList)
+			if err != nil {
+				log.Error("es: sync questions error", err)
+			}
+			page += 1
+		}
+
+		log.Info("es: start sync answers...")
+		page = 1
+		for {
+			log.Infof("es: sync answer page %d, page size %d", page, pageSize)
+			answerList, err := s.syncer.GetAnswersPage(context.TODO(), page, pageSize)
+			if err != nil {
+				log.Error("es: sync answers error", err)
+				break
+			}
+
+			if len(answerList) == 0 {
+				break
+			}
+
+			err = s.batchUpdateContent(context.TODO(), answerList)
+			if err != nil {
+				log.Error("es: sync answers error", err)
+			}
+
+			page += 1
+		}
+		log.Info("es: sync done")
+	}()
+}
+
+func (s *SearchEngine) batchUpdateContent(ctx context.Context, contents []*plugin.SearchContent) (err error) {
+	for _, content := range contents {
+		err = s.Operator.SaveDoc(ctx, s.getIndexName(), content.ObjectID, CreateDocFromSearchContent(content.ObjectID, content))
+		if err != nil {
+			return
+		}
+	}
+
+	return nil
+}