You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@answer.apache.org by li...@apache.org on 2026/07/09 07:58:42 UTC
(answer-plugins) 01/12: feat(pgvector): add PostgreSQL pgvector vector search plugin with configuration and sync functionality
This is an automated email from the ASF dual-hosted git repository.
LinkinStars pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/answer-plugins.git
commit b8244a32ec9d00ab9e870a1a248502ae5b9bb1a5
Author: hgaol <dh...@hotmail.com>
AuthorDate: Mon May 11 22:20:46 2026 +0800
feat(pgvector): add PostgreSQL pgvector vector search plugin with configuration and sync functionality
---
vector-search-pgvector/README.md | 49 ++++
vector-search-pgvector/go.mod | 11 +
vector-search-pgvector/i18n/en_US.yaml | 61 +++++
vector-search-pgvector/i18n/translation.go | 45 ++++
vector-search-pgvector/i18n/zh_CN.yaml | 61 +++++
vector-search-pgvector/info.yaml | 22 ++
vector-search-pgvector/pgvector.go | 398 +++++++++++++++++++++++++++++
vector-search-pgvector/sync.go | 105 ++++++++
8 files changed, 752 insertions(+)
diff --git a/vector-search-pgvector/README.md b/vector-search-pgvector/README.md
new file mode 100644
index 0000000..ce07404
--- /dev/null
+++ b/vector-search-pgvector/README.md
@@ -0,0 +1,49 @@
+# Apache Answer Plugin: pgvector Vector Search
+
+This plugin enables semantic/vector search in [Apache Answer](https://github.com/apache/answer) using [PostgreSQL](https://www.postgresql.org/) with the [pgvector](https://github.com/pgvector/pgvector) extension.
+
+## Prerequisites
+
+- PostgreSQL 12+ with the [pgvector](https://github.com/pgvector/pgvector) extension installed
+- An OpenAI-compatible embedding API (e.g., OpenAI, Azure OpenAI, or any compatible provider)
+
+## Installation
+
+Build Apache Answer with this plugin:
+
+```bash
+./answer build --with github.com/apache/answer-plugins/vector-search-pgvector
+```
+
+## Configuration
+
+After enabling the plugin in the Admin UI (**Admin > Plugins > Vector Search**), configure the following fields:
+
+| Field | Description | Example |
+|---|---|---|
+| **DSN** | PostgreSQL connection string | `postgres://user:pass@localhost:5432/answer?sslmode=disable` |
+| **Embedding API Host** | OpenAI-compatible API base URL | `https://api.openai.com` |
+| **Embedding API Key** | API key for the embedding service | `sk-...` |
+| **Embedding Model** | Model name for generating embeddings | `text-embedding-3-small` |
+| **Embedding Level** | `question` embeds question + all answers + comments together; `answer` embeds each answer separately | `question` |
+| **Similarity Threshold** | Minimum cosine similarity score (0-1). Default `0` means no filtering | `0.5` |
+
+## How It Works
+
+- **Embedding dimensions** are auto-detected from the configured model. No manual dimension configuration is needed.
+- On first configuration, the plugin creates a table `answer_vector_embeddings` with a `vector` column matching the detected dimensions.
+- If the embedding model changes and produces different dimensions, the table is automatically dropped and recreated.
+- Uses cosine similarity (`<=>` operator) for vector search.
+- A full sync of all questions/answers is triggered when the plugin starts.
+
+## Database Setup
+
+Ensure the pgvector extension is installed in your PostgreSQL database:
+
+```sql
+CREATE EXTENSION IF NOT EXISTS vector;
+```
+
+## License
+
+[Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0)
diff --git a/vector-search-pgvector/go.mod b/vector-search-pgvector/go.mod
new file mode 100644
index 0000000..e0fbfe3
--- /dev/null
+++ b/vector-search-pgvector/go.mod
@@ -0,0 +1,11 @@
+module github.com/apache/answer-plugins/vector-search-pgvector
+
+go 1.23.0
+
+require (
+ github.com/apache/answer v1.7.0
+ github.com/apache/answer-plugins/util v1.0.3-0.20250107030257-cf94ebc70954
+ github.com/lib/pq v1.10.9
+ github.com/pgvector/pgvector-go v0.2.2
+ github.com/segmentfault/pacman v1.0.5-0.20230822083413-c0075a2d401f
+)
diff --git a/vector-search-pgvector/i18n/en_US.yaml b/vector-search-pgvector/i18n/en_US.yaml
new file mode 100644
index 0000000..6059eb3
--- /dev/null
+++ b/vector-search-pgvector/i18n/en_US.yaml
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+plugin:
+ pgvector_search:
+ backend:
+ info:
+ name:
+ other: PostgreSQL pgvector Search
+ description:
+ other: Use PostgreSQL with pgvector extension for semantic/vector search
+ config:
+ dsn:
+ title:
+ other: PostgreSQL DSN
+ description:
+ other: "PostgreSQL connection string (e.g. postgres://user:pass@localhost:5432/dbname?sslmode=disable)"
+ api_host:
+ title:
+ other: Embedding API Host
+ description:
+ other: "OpenAI-compatible API base URL for embedding generation (e.g. https://api.openai.com)"
+ api_key:
+ title:
+ other: Embedding API Key
+ description:
+ other: API key for the embedding service
+ embedding_model:
+ title:
+ other: Embedding Model
+ description:
+ other: "Model name for embedding generation (e.g. text-embedding-3-small)"
+ embedding_level:
+ title:
+ other: Embedding Level
+ description:
+ other: "Choose 'question' to embed question + all answers + comments, or 'answer' to embed each answer separately"
+ options:
+ question:
+ other: Question
+ answer:
+ other: Answer
+ similarity_threshold:
+ title:
+ other: Similarity Threshold
+ description:
+ other: "Minimum cosine similarity score (0-1) for search results. Default 0 (no filtering)."
diff --git a/vector-search-pgvector/i18n/translation.go b/vector-search-pgvector/i18n/translation.go
new file mode 100644
index 0000000..254a19a
--- /dev/null
+++ b/vector-search-pgvector/i18n/translation.go
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package i18n
+
+const (
+ InfoName = "plugin.pgvector_search.backend.info.name"
+ InfoDescription = "plugin.pgvector_search.backend.info.description"
+
+ ConfigDSNTitle = "plugin.pgvector_search.backend.config.dsn.title"
+ ConfigDSNDescription = "plugin.pgvector_search.backend.config.dsn.description"
+
+ ConfigAPIHostTitle = "plugin.pgvector_search.backend.config.api_host.title"
+ ConfigAPIHostDescription = "plugin.pgvector_search.backend.config.api_host.description"
+
+ ConfigAPIKeyTitle = "plugin.pgvector_search.backend.config.api_key.title"
+ ConfigAPIKeyDescription = "plugin.pgvector_search.backend.config.api_key.description"
+
+ ConfigEmbeddingModelTitle = "plugin.pgvector_search.backend.config.embedding_model.title"
+ ConfigEmbeddingModelDescription = "plugin.pgvector_search.backend.config.embedding_model.description"
+
+ ConfigEmbeddingLevelTitle = "plugin.pgvector_search.backend.config.embedding_level.title"
+ ConfigEmbeddingLevelDescription = "plugin.pgvector_search.backend.config.embedding_level.description"
+ ConfigEmbeddingLevelOptionQuestion = "plugin.pgvector_search.backend.config.embedding_level.options.question"
+ ConfigEmbeddingLevelOptionAnswer = "plugin.pgvector_search.backend.config.embedding_level.options.answer"
+
+ ConfigSimilarityThresholdTitle = "plugin.pgvector_search.backend.config.similarity_threshold.title"
+ ConfigSimilarityThresholdDescription = "plugin.pgvector_search.backend.config.similarity_threshold.description"
+)
diff --git a/vector-search-pgvector/i18n/zh_CN.yaml b/vector-search-pgvector/i18n/zh_CN.yaml
new file mode 100644
index 0000000..66887d8
--- /dev/null
+++ b/vector-search-pgvector/i18n/zh_CN.yaml
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+plugin:
+ pgvector_search:
+ backend:
+ info:
+ name:
+ other: PostgreSQL pgvector 搜索
+ description:
+ other: 使用 PostgreSQL pgvector 扩展进行语义/向量搜索
+ config:
+ dsn:
+ title:
+ other: PostgreSQL 连接字符串
+ description:
+ other: "PostgreSQL 连接字符串（如 postgres://user:pass@localhost:5432/dbname?sslmode=disable）"
+ api_host:
+ title:
+ other: Embedding API 地址
+ description:
+ other: "用于生成 Embedding 的 OpenAI 兼容 API 地址（如 https://api.openai.com）"
+ api_key:
+ title:
+ other: Embedding API 密钥
+ description:
+ other: Embedding 服务的 API 密钥
+ embedding_model:
+ title:
+ other: Embedding 模型
+ description:
+ other: "Embedding 模型名称（如 text-embedding-3-small）"
+ embedding_level:
+ title:
+ other: Embedding 级别
+ description:
+ other: "选择 'question' 将问题+所有回答+评论合并嵌入，或 'answer' 将每个回答单独嵌入"
+ options:
+ question:
+ other: 问题
+ answer:
+ other: 回答
+ similarity_threshold:
+ title:
+ other: 相似度阈值
+ description:
+ other: "搜索结果的最低余弦相似度分数（0-1）。默认为 0（不过滤）。"
diff --git a/vector-search-pgvector/info.yaml b/vector-search-pgvector/info.yaml
new file mode 100644
index 0000000..b9b2f60
--- /dev/null
+++ b/vector-search-pgvector/info.yaml
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+slug_name: pgvector_search
+type: vector_search
+version: 0.1.0
+author: answerdev
+link: https://github.com/apache/answer-plugins/tree/main/vector-search-pgvector
diff --git a/vector-search-pgvector/pgvector.go b/vector-search-pgvector/pgvector.go
new file mode 100644
index 0000000..dc50601
--- /dev/null
+++ b/vector-search-pgvector/pgvector.go
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package pgvector_search
+
+import (
+ "context"
+ "database/sql"
+ "embed"
+ "encoding/json"
+ "fmt"
+ "strconv"
+ "sync"
+
+ "github.com/apache/answer-plugins/util"
+ "github.com/apache/answer-plugins/vector-search-pgvector/i18n"
+ "github.com/apache/answer/plugin"
+ _ "github.com/lib/pq"
+ pgv "github.com/pgvector/pgvector-go"
+ "github.com/segmentfault/pacman/log"
+)
+
+//go:embed info.yaml
+var Info embed.FS
+
+const tableName = "answer_vector_embeddings"
+
+// VectorSearchEngine implements plugin.VectorSearch using PostgreSQL + pgvector.
+type VectorSearchEngine struct {
+ Config *VectorSearchConfig
+ db *sql.DB
+ syncer plugin.VectorSearchSyncer
+ syncing bool
+ lock sync.Mutex
+ embeddingDimensions int // auto-detected from embedding model
+}
+
+// VectorSearchConfig holds all plugin configuration.
+type VectorSearchConfig struct {
+ DSN string `json:"dsn"`
+ APIHost string `json:"api_host"`
+ APIKey string `json:"api_key"`
+ EmbeddingModel string `json:"embedding_model"`
+ EmbeddingLevel string `json:"embedding_level"`
+ SimilarityThreshold float64 `json:"similarity_threshold"`
+}
+
+func init() {
+ plugin.Register(&VectorSearchEngine{
+ Config: &VectorSearchConfig{},
+ lock: sync.Mutex{},
+ })
+}
+
+func (e *VectorSearchEngine) Info() plugin.Info {
+ info := &util.Info{}
+ info.GetInfo(Info)
+
+ return plugin.Info{
+ Name: plugin.MakeTranslator(i18n.InfoName),
+ SlugName: info.SlugName,
+ Description: plugin.MakeTranslator(i18n.InfoDescription),
+ Author: info.Author,
+ Version: info.Version,
+ Link: info.Link,
+ }
+}
+
+// Description returns metadata about this vector search engine.
+func (e *VectorSearchEngine) Description() plugin.VectorSearchDesc {
+ return plugin.VectorSearchDesc{
+ Icon: "",
+ Link: "https://github.com/pgvector/pgvector",
+ }
+}
+
+// RegisterSyncer stores the syncer and triggers a full sync.
+func (e *VectorSearchEngine) RegisterSyncer(ctx context.Context, syncer plugin.VectorSearchSyncer) {
+ log.Debugf("pgvector: RegisterSyncer called, db=%v, embeddingLevel=%s", e.db != nil, e.Config.EmbeddingLevel)
+ e.syncer = syncer
+ if e.db != nil {
+ e.sync()
+ }
+}
+
+// SearchSimilar performs a cosine similarity search using pgvector.
+func (e *VectorSearchEngine) SearchSimilar(ctx context.Context, query string, topK int) ([]plugin.VectorSearchResult, error) {
+ if e.db == nil {
+ return nil, fmt.Errorf("pgvector: database not initialized")
+ }
+ if topK <= 0 {
+ topK = 10
+ }
+
+ log.Debugf("pgvector: SearchSimilar query=%q topK=%d model=%s", query, topK, e.Config.EmbeddingModel)
+
+ embedding, err := plugin.GenerateEmbedding(ctx, e.Config.APIHost, e.Config.APIKey, e.Config.EmbeddingModel, query)
+ if err != nil {
+ return nil, fmt.Errorf("generate query embedding: %w", err)
+ }
+
+ log.Debugf("pgvector: search embedding generated, dimensions=%d", len(embedding))
+
+ vec := pgv.NewVector(embedding)
+
+ // Use cosine distance operator (<=>). Score = 1 - distance.
+ sqlQuery := fmt.Sprintf(
+ `SELECT object_id, object_type, metadata, 1 - (embedding <=> $1) AS score
+ FROM %s
+ ORDER BY embedding <=> $1
+ LIMIT $2`, tableName)
+
+ rows, err := e.db.QueryContext(ctx, sqlQuery, vec, topK)
+ if err != nil {
+ return nil, fmt.Errorf("pgvector search failed: %w", err)
+ }
+ defer rows.Close()
+
+ results := make([]plugin.VectorSearchResult, 0, topK)
+ for rows.Next() {
+ var r plugin.VectorSearchResult
+ if err := rows.Scan(&r.ObjectID, &r.ObjectType, &r.Metadata, &r.Score); err != nil {
+ log.Warnf("pgvector: scan row: %v", err)
+ continue
+ }
+ if e.Config.SimilarityThreshold > 0 && r.Score < e.Config.SimilarityThreshold {
+ log.Debugf("pgvector: skipping result %s score=%.4f below threshold=%.4f", r.ObjectID, r.Score, e.Config.SimilarityThreshold)
+ continue
+ }
+ results = append(results, r)
+ }
+ log.Debugf("pgvector: SearchSimilar returning %d results", len(results))
+ return results, rows.Err()
+}
+
+// UpdateContent upserts a single document into the pgvector table.
+func (e *VectorSearchEngine) UpdateContent(ctx context.Context, content *plugin.VectorSearchContent) error {
+ if e.db == nil {
+ return fmt.Errorf("pgvector: database not initialized")
+ }
+
+ log.Debugf("pgvector: UpdateContent objectID=%s objectType=%s titleLen=%d contentLen=%d",
+ content.ObjectID, content.ObjectType, len(content.Title), len(content.Content))
+
+ embedding, err := plugin.GenerateEmbedding(ctx, e.Config.APIHost, e.Config.APIKey, e.Config.EmbeddingModel, content.Content)
+ if err != nil {
+ return fmt.Errorf("generate embedding for %s: %w", content.ObjectID, err)
+ }
+
+ log.Debugf("pgvector: embedding generated for %s, dimensions=%d", content.ObjectID, len(embedding))
+
+ vec := pgv.NewVector(embedding)
+
+ query := fmt.Sprintf(
+ `INSERT INTO %s (object_id, object_type, title, content, metadata, embedding)
+ VALUES ($1, $2, $3, $4, $5, $6)
+ ON CONFLICT (object_id) DO UPDATE SET
+ object_type = EXCLUDED.object_type,
+ title = EXCLUDED.title,
+ content = EXCLUDED.content,
+ metadata = EXCLUDED.metadata,
+ embedding = EXCLUDED.embedding`, tableName)
+
+ _, err = e.db.ExecContext(ctx, query,
+ content.ObjectID, content.ObjectType, content.Title, content.Content, content.Metadata, vec)
+ if err != nil {
+ return fmt.Errorf("upsert document %s: %w", content.ObjectID, err)
+ }
+ log.Debugf("pgvector: upserted document %s successfully", content.ObjectID)
+ return nil
+}
+
+// DeleteContent removes a document by object ID.
+func (e *VectorSearchEngine) DeleteContent(ctx context.Context, objectID string) error {
+ if e.db == nil {
+ return fmt.Errorf("pgvector: database not initialized")
+ }
+ log.Debugf("pgvector: DeleteContent objectID=%s", objectID)
+ _, err := e.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM %s WHERE object_id = $1`, tableName), objectID)
+ if err != nil {
+ return fmt.Errorf("delete document %s: %w", objectID, err)
+ }
+ log.Debugf("pgvector: deleted document %s", objectID)
+ return nil
+}
+
+// ConfigFields returns the plugin configuration form fields.
+func (e *VectorSearchEngine) ConfigFields() []plugin.ConfigField {
+ return []plugin.ConfigField{
+ {
+ Name: "dsn",
+ Type: plugin.ConfigTypeInput,
+ Title: plugin.MakeTranslator(i18n.ConfigDSNTitle),
+ Description: plugin.MakeTranslator(i18n.ConfigDSNDescription),
+ Required: true,
+ UIOptions: plugin.ConfigFieldUIOptions{
+ InputType: plugin.InputTypeText,
+ },
+ Value: e.Config.DSN,
+ },
+ {
+ Name: "api_host",
+ Type: plugin.ConfigTypeInput,
+ Title: plugin.MakeTranslator(i18n.ConfigAPIHostTitle),
+ Description: plugin.MakeTranslator(i18n.ConfigAPIHostDescription),
+ Required: true,
+ UIOptions: plugin.ConfigFieldUIOptions{
+ InputType: plugin.InputTypeText,
+ },
+ Value: e.Config.APIHost,
+ },
+ {
+ Name: "api_key",
+ Type: plugin.ConfigTypeInput,
+ Title: plugin.MakeTranslator(i18n.ConfigAPIKeyTitle),
+ Description: plugin.MakeTranslator(i18n.ConfigAPIKeyDescription),
+ Required: true,
+ UIOptions: plugin.ConfigFieldUIOptions{
+ InputType: plugin.InputTypePassword,
+ },
+ Value: e.Config.APIKey,
+ },
+ {
+ Name: "embedding_model",
+ Type: plugin.ConfigTypeInput,
+ Title: plugin.MakeTranslator(i18n.ConfigEmbeddingModelTitle),
+ Description: plugin.MakeTranslator(i18n.ConfigEmbeddingModelDescription),
+ Required: true,
+ UIOptions: plugin.ConfigFieldUIOptions{
+ InputType: plugin.InputTypeText,
+ },
+ Value: e.Config.EmbeddingModel,
+ },
+ {
+ Name: "embedding_level",
+ Type: plugin.ConfigTypeSelect,
+ Title: plugin.MakeTranslator(i18n.ConfigEmbeddingLevelTitle),
+ Description: plugin.MakeTranslator(i18n.ConfigEmbeddingLevelDescription),
+ Required: true,
+ Options: []plugin.ConfigFieldOption{
+ {Label: plugin.MakeTranslator(i18n.ConfigEmbeddingLevelOptionQuestion), Value: "question"},
+ {Label: plugin.MakeTranslator(i18n.ConfigEmbeddingLevelOptionAnswer), Value: "answer"},
+ },
+ Value: e.Config.EmbeddingLevel,
+ },
+ {
+ Name: "similarity_threshold",
+ Type: plugin.ConfigTypeInput,
+ Title: plugin.MakeTranslator(i18n.ConfigSimilarityThresholdTitle),
+ Description: plugin.MakeTranslator(i18n.ConfigSimilarityThresholdDescription),
+ Required: false,
+ UIOptions: plugin.ConfigFieldUIOptions{
+ InputType: plugin.InputTypeText,
+ },
+ Value: fmt.Sprintf("%.2f", e.Config.SimilarityThreshold),
+ },
+ }
+}
+
+// ConfigReceiver applies configuration from the admin UI.
+func (e *VectorSearchEngine) ConfigReceiver(config []byte) error {
+ log.Debugf("pgvector: ConfigReceiver called, raw config=%s", string(config))
+
+ // Pre-process: convert string numbers to actual numbers before unmarshalling,
+ // because the admin UI sends all form values as strings.
+ var raw map[string]interface{}
+ if err := json.Unmarshal(config, &raw); err != nil {
+ return fmt.Errorf("unmarshal config: %w", err)
+ }
+ if threshStr, ok := raw["similarity_threshold"].(string); ok {
+ if thresh, err := strconv.ParseFloat(threshStr, 64); err == nil {
+ raw["similarity_threshold"] = thresh
+ }
+ }
+ fixed, _ := json.Marshal(raw)
+
+ conf := &VectorSearchConfig{}
+ if err := json.Unmarshal(fixed, conf); err != nil {
+ return fmt.Errorf("unmarshal config: %w", err)
+ }
+
+ e.Config = conf
+
+ log.Debugf("pgvector: config parsed: dsn=%s model=%s level=%s threshold=%.2f",
+ conf.DSN, conf.EmbeddingModel, conf.EmbeddingLevel, conf.SimilarityThreshold)
+
+ if !plugin.StatusManager.IsEnabled("pgvector_search") {
+ log.Debugf("pgvector: plugin not active, skipping initialization")
+ return nil
+ }
+
+ // Auto-detect embedding dimensions by generating a probe embedding.
+ log.Debugf("pgvector: detecting embedding dimensions via probe call")
+ probeEmbedding, err := plugin.GenerateEmbedding(context.Background(), conf.APIHost, conf.APIKey, conf.EmbeddingModel, "dimension probe")
+ if err != nil {
+ return fmt.Errorf("detect embedding dimensions: %w", err)
+ }
+ e.embeddingDimensions = len(probeEmbedding)
+ log.Infof("pgvector: auto-detected embedding dimensions=%d for model=%s", e.embeddingDimensions, conf.EmbeddingModel)
+
+ log.Debugf("pgvector: connecting to PostgreSQL")
+
+ db, err := sql.Open("postgres", conf.DSN)
+ if err != nil {
+ return fmt.Errorf("open postgres: %w", err)
+ }
+ if err := db.Ping(); err != nil {
+ db.Close()
+ return fmt.Errorf("ping postgres: %w", err)
+ }
+
+ log.Debugf("pgvector: connected to PostgreSQL successfully")
+
+ // Close previous connection if any.
+ if e.db != nil {
+ e.db.Close()
+ }
+ e.db = db
+
+ if err := e.ensureTable(context.Background()); err != nil {
+ return fmt.Errorf("ensure table: %w", err)
+ }
+
+ log.Debugf("pgvector: ConfigReceiver completed successfully")
+ return nil
+}
+
+// ensureTable creates the pgvector extension and the embeddings table if they don't exist.
+// If the table exists but the embedding dimensions differ, it drops and recreates the table.
+func (e *VectorSearchEngine) ensureTable(ctx context.Context) error {
+ // Enable the pgvector extension.
+ if _, err := e.db.ExecContext(ctx, `CREATE EXTENSION IF NOT EXISTS vector`); err != nil {
+ return fmt.Errorf("create vector extension: %w", err)
+ }
+
+ dims := e.embeddingDimensions
+ if dims <= 0 {
+ dims = 1536
+ }
+
+ log.Debugf("pgvector: ensureTable with dimensions=%d", dims)
+
+ // Check if table exists and verify embedding dimensions match.
+ var existingDims int
+ err := e.db.QueryRowContext(ctx, fmt.Sprintf(
+ `SELECT atttypmod FROM pg_attribute
+ WHERE attrelid = '%s'::regclass AND attname = 'embedding'`, tableName)).Scan(&existingDims)
+ if err == nil && existingDims > 0 && existingDims != dims {
+ log.Warnf("pgvector: embedding dimensions changed from %d to %d, recreating table", existingDims, dims)
+ if _, err := e.db.ExecContext(ctx, fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tableName)); err != nil {
+ return fmt.Errorf("drop table for dimension change: %w", err)
+ }
+ } else if err != nil {
+ log.Debugf("pgvector: table does not exist yet or dimension check skipped: %v", err)
+ } else {
+ log.Debugf("pgvector: existing table dimensions=%d, matches configured=%d", existingDims, dims)
+ }
+
+ createSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
+ object_id TEXT PRIMARY KEY,
+ object_type TEXT NOT NULL,
+ title TEXT NOT NULL DEFAULT '',
+ content TEXT NOT NULL DEFAULT '',
+ metadata TEXT NOT NULL DEFAULT '',
+ embedding vector(%d) NOT NULL
+ )`, tableName, dims)
+
+ if _, err := e.db.ExecContext(ctx, createSQL); err != nil {
+ return fmt.Errorf("create table: %w", err)
+ }
+
+ // Create an IVFFlat index for faster cosine similarity search if it doesn't exist.
+ indexSQL := fmt.Sprintf(
+ `CREATE INDEX IF NOT EXISTS idx_%s_embedding ON %s USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100)`,
+ tableName, tableName)
+ if _, err := e.db.ExecContext(ctx, indexSQL); err != nil {
+ // IVFFlat requires data in the table to build the index; log and continue if empty.
+ log.Warnf("pgvector: create ivfflat index (may need data first): %v", err)
+ }
+
+ return nil
+}
diff --git a/vector-search-pgvector/sync.go b/vector-search-pgvector/sync.go
new file mode 100644
index 0000000..e4a3063
--- /dev/null
+++ b/vector-search-pgvector/sync.go
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package pgvector_search
+
+import (
+ "context"
+
+ "github.com/apache/answer/plugin"
+ "github.com/segmentfault/pacman/log"
+)
+
+// sync triggers a full bulk sync of all Q&A content into the pgvector table.
+// It runs in a background goroutine and is guarded by the syncing flag.
+func (e *VectorSearchEngine) sync() {
+ if e.syncer == nil {
+ log.Warn("pgvector: syncer not registered, skip sync")
+ return
+ }
+ if e.syncing {
+ log.Warn("pgvector: sync already running, skip")
+ return
+ }
+
+ go func() {
+ e.lock.Lock()
+ defer e.lock.Unlock()
+ if e.syncing {
+ return
+ }
+ e.syncing = true
+ defer func() { e.syncing = false }()
+
+ page, pageSize := 1, 100
+
+ if e.Config.EmbeddingLevel == "" || e.Config.EmbeddingLevel == "question" {
+ log.Info("pgvector: starting question sync...")
+ page = 1
+ for {
+ log.Infof("pgvector: sync questions page %d", page)
+ questions, err := e.syncer.GetQuestionsPage(context.TODO(), page, pageSize)
+ if err != nil {
+ log.Errorf("pgvector: get questions page failed: %v", err)
+ break
+ }
+ if len(questions) == 0 {
+ break
+ }
+ e.bulkIndex(context.TODO(), questions)
+ page++
+ }
+ }
+
+ if e.Config.EmbeddingLevel == "answer" {
+ log.Info("pgvector: starting answer sync...")
+ page = 1
+ for {
+ log.Infof("pgvector: sync answers page %d", page)
+ answers, err := e.syncer.GetAnswersPage(context.TODO(), page, pageSize)
+ if err != nil {
+ log.Errorf("pgvector: get answers page failed: %v", err)
+ break
+ }
+ if len(answers) == 0 {
+ break
+ }
+ e.bulkIndex(context.TODO(), answers)
+ page++
+ }
+ }
+
+ log.Info("pgvector: sync complete")
+ }()
+}
+
+// bulkIndex indexes a batch of documents, computing embeddings for each.
+func (e *VectorSearchEngine) bulkIndex(ctx context.Context, contents []*plugin.VectorSearchContent) {
+ log.Debugf("pgvector: bulkIndex batch size=%d", len(contents))
+ success, failed := 0, 0
+ for _, c := range contents {
+ if err := e.UpdateContent(ctx, c); err != nil {
+ log.Warnf("pgvector: index %s failed: %v", c.ObjectID, err)
+ failed++
+ } else {
+ success++
+ }
+ }
+ log.Debugf("pgvector: bulkIndex batch done, success=%d failed=%d", success, failed)
+}