You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@answer.apache.org by li...@apache.org on 2024/09/03 02:47:11 UTC
(incubator-answer-plugins) 01/02: Add sync to Elasticsearch
This is an automated email from the ASF dual-hosted git repository.
linkinstar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git
commit 70bb63de0773550d70c1ca162e340958e9c84cf4
Author: sy-records <52...@qq52o.cn>
AuthorDate: Sat Aug 31 16:44:32 2024 +0800
Add sync to Elasticsearch
---
search-elasticsearch/es.go | 2 +-
search-elasticsearch/sync.go | 84 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 85 insertions(+), 1 deletion(-)
diff --git a/search-elasticsearch/es.go b/search-elasticsearch/es.go
index 8ea9a8a..d04c732 100644
--- a/search-elasticsearch/es.go
+++ b/search-elasticsearch/es.go
@@ -143,7 +143,7 @@ func (s *SearchEngine) DeleteContent(ctx context.Context, contentID string) erro
func (s *SearchEngine) RegisterSyncer(ctx context.Context, syncer plugin.SearchSyncer) {
s.syncer = syncer
- // TODO: Synchronization of already existing data through some strategy
+ s.sync()
}
func (s *SearchEngine) warpResult(resp *elastic.SearchResult) ([]plugin.SearchResult, int64, error) {
diff --git a/search-elasticsearch/sync.go b/search-elasticsearch/sync.go
new file mode 100644
index 0000000..76a137f
--- /dev/null
+++ b/search-elasticsearch/sync.go
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package es
+
+import (
+ "context"
+ "github.com/apache/incubator-answer/plugin"
+ "github.com/segmentfault/pacman/log"
+)
+
+func (s *SearchEngine) sync() {
+ var page, pageSize = 1, 100
+ go func() {
+ log.Info("es: start sync questions...")
+ page = 1
+ for {
+ log.Infof("es: sync question page %d, page size %d", page, pageSize)
+ questionList, err := s.syncer.GetQuestionsPage(context.TODO(), page, pageSize)
+ if err != nil {
+ log.Error("es: sync questions error", err)
+ break
+ }
+ if len(questionList) == 0 {
+ break
+ }
+ err = s.batchUpdateContent(context.TODO(), questionList)
+ if err != nil {
+ log.Error("es: sync questions error", err)
+ }
+ page += 1
+ }
+
+ log.Info("es: start sync answers...")
+ page = 1
+ for {
+ log.Infof("es: sync answer page %d, page size %d", page, pageSize)
+ answerList, err := s.syncer.GetAnswersPage(context.TODO(), page, pageSize)
+ if err != nil {
+ log.Error("es: sync answers error", err)
+ break
+ }
+
+ if len(answerList) == 0 {
+ break
+ }
+
+ err = s.batchUpdateContent(context.TODO(), answerList)
+ if err != nil {
+ log.Error("es: sync answers error", err)
+ }
+
+ page += 1
+ }
+ log.Info("es: sync done")
+ }()
+}
+
+func (s *SearchEngine) batchUpdateContent(ctx context.Context, contents []*plugin.SearchContent) (err error) {
+ for _, content := range contents {
+ err = s.Operator.SaveDoc(ctx, s.getIndexName(), content.ObjectID, CreateDocFromSearchContent(content.ObjectID, content))
+ if err != nil {
+ return
+ }
+ }
+
+ return nil
+}