You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2024/10/31 13:43:38 UTC
(incubator-baremaps) branch overturemaps-lucene created (now f871323a6)
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a change to branch overturemaps-lucene
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
at f871323a6 Add consumer, mapper and query for datatable abstration
This branch includes the following new commits:
new d7a4b6396 Refactor the geocoder package
new f871323a6 Add consumer, mapper and query for datatable abstration
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-baremaps) 01/02: Refactor the geocoder package
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch overturemaps-lucene
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit d7a4b63963271ac4485995ebf02b1d0a9bbf07e2
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Wed Oct 30 18:58:29 2024 +0100
Refactor the geocoder package
---
.../org/apache/baremaps/cli/geocoder/Search.java | 2 +-
.../{ => geonames}/GeonamesDocumentMapper.java | 2 +-
.../geocoder/{ => geonames}/GeonamesEntry.java | 2 +-
.../{ => geonames}/GeonamesQueryBuilder.java | 4 ++-
.../geocoder/{ => geonames}/GeonamesReader.java | 2 +-
.../OpenStreetMapDocumentMapper.java} | 6 ++---
.../OpenStreetMapEntityConsumer.java} | 12 ++++-----
.../openstreetmap/OpenStreetMapQuery.java} | 6 ++---
.../openstreetmap}/OsmTags.java | 2 +-
.../org/apache/baremaps/iploc/IpLocMapper.java | 2 +-
.../tasks/CreateGeocoderOpenStreetMap.java | 6 ++---
.../workflow/tasks/CreateGeonamesIndex.java | 4 +--
.../geocoder/{ => geonames}/GeonamesIndexTest.java | 3 ++-
.../{ => geonames}/GeonamesReaderTest.java | 3 ++-
.../openstreetmap/OpenStreetMapIndexTest.java} | 7 ++---
baremaps-geocoder/pom.xml | 30 ++++++++++++++++++++++
.../apache/baremaps/server/GeocoderResource.java | 2 +-
pom.xml | 1 +
18 files changed, 66 insertions(+), 30 deletions(-)
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Search.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Search.java
index e14fbc2a8..49eefe598 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Search.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Search.java
@@ -21,7 +21,7 @@ package org.apache.baremaps.cli.geocoder;
import java.nio.file.Path;
import java.util.concurrent.Callable;
-import org.apache.baremaps.geocoder.GeonamesQueryBuilder;
+import org.apache.baremaps.geocoder.geonames.GeonamesQueryBuilder;
import org.apache.lucene.search.*;
import org.apache.lucene.store.FSDirectory;
import org.slf4j.Logger;
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesDocumentMapper.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesDocumentMapper.java
similarity index 98%
rename from baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesDocumentMapper.java
rename to baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesDocumentMapper.java
index 7f704aff2..24b262138 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesDocumentMapper.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesDocumentMapper.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.baremaps.geocoder;
+package org.apache.baremaps.geocoder.geonames;
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesEntry.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesEntry.java
similarity index 99%
rename from baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesEntry.java
rename to baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesEntry.java
index 601cc3b05..6393537b0 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesEntry.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesEntry.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.baremaps.geocoder;
+package org.apache.baremaps.geocoder.geonames;
/**
* A record from the Geonames database.
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesQueryBuilder.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesQueryBuilder.java
similarity index 97%
rename from baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesQueryBuilder.java
rename to baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesQueryBuilder.java
index 8b4958ac2..d1cd32db2 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesQueryBuilder.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesQueryBuilder.java
@@ -15,13 +15,15 @@
* limitations under the License.
*/
-package org.apache.baremaps.geocoder;
+package org.apache.baremaps.geocoder.geonames;
import com.google.common.base.Strings;
import java.text.ParseException;
import java.util.Map;
+
+import org.apache.baremaps.geocoder.GeocoderConstants;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.expressions.Expression;
import org.apache.lucene.expressions.SimpleBindings;
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesReader.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesReader.java
similarity index 98%
rename from baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesReader.java
rename to baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesReader.java
index b35c4c45f..aa1b04271 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesReader.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesReader.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.baremaps.geocoder;
+package org.apache.baremaps.geocoder.geonames;
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmDocumentMapper.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapDocumentMapper.java
similarity index 94%
rename from baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmDocumentMapper.java
rename to baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapDocumentMapper.java
index 60e318ecc..cc478c01f 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmDocumentMapper.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapDocumentMapper.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.baremaps.geocoderosm;
+package org.apache.baremaps.geocoder.openstreetmap;
@@ -35,8 +35,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class GeocoderOsmDocumentMapper implements Function<Element, Document> {
- private static final Logger logger = LoggerFactory.getLogger(GeocoderOsmDocumentMapper.class);
+public class OpenStreetMapDocumentMapper implements Function<Element, Document> {
+ private static final Logger logger = LoggerFactory.getLogger(OpenStreetMapDocumentMapper.class);
@Override
public Document apply(Element element) {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmConsumerEntity.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapEntityConsumer.java
similarity index 79%
rename from baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmConsumerEntity.java
rename to baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapEntityConsumer.java
index ea7f6616b..ab29a2954 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmConsumerEntity.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapEntityConsumer.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.baremaps.geocoderosm;
+package org.apache.baremaps.geocoder.openstreetmap;
import java.util.function.Consumer;
import org.apache.baremaps.openstreetmap.model.Element;
@@ -24,14 +24,14 @@ import org.apache.lucene.index.IndexWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class GeocoderOsmConsumerEntity implements Consumer<Entity> {
+public class OpenStreetMapEntityConsumer implements Consumer<Entity> {
private final IndexWriter indexWriter;
- private final GeocoderOsmDocumentMapper geocoderOsmDocumentMapper =
- new GeocoderOsmDocumentMapper();
+ private final OpenStreetMapDocumentMapper geocoderOsmDocumentMapper =
+ new OpenStreetMapDocumentMapper();
- private static final Logger logger = LoggerFactory.getLogger(GeocoderOsmConsumerEntity.class);
+ private static final Logger logger = LoggerFactory.getLogger(OpenStreetMapEntityConsumer.class);
- public GeocoderOsmConsumerEntity(IndexWriter indexWriter) {
+ public OpenStreetMapEntityConsumer(IndexWriter indexWriter) {
this.indexWriter = indexWriter;
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmQuery.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapQuery.java
similarity index 92%
rename from baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmQuery.java
rename to baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapQuery.java
index bd326369c..435917539 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmQuery.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapQuery.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.baremaps.geocoderosm;
+package org.apache.baremaps.geocoder.openstreetmap;
import org.apache.baremaps.geocoder.GeocoderConstants;
import org.apache.lucene.queryparser.classic.QueryParserBase;
@@ -24,11 +24,11 @@ import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
-public class GeocoderOsmQuery {
+public class OpenStreetMapQuery {
private final String query;
- public GeocoderOsmQuery(String query) {
+ public OpenStreetMapQuery(String query) {
this.query = query;
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/OsmTags.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OsmTags.java
similarity index 95%
rename from baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/OsmTags.java
rename to baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OsmTags.java
index 3be62f7b4..7385b4a8d 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/OsmTags.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OsmTags.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.baremaps.geocoderosm;
+package org.apache.baremaps.geocoder.openstreetmap;
public enum OsmTags {
NAME("name"),
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocMapper.java b/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocMapper.java
index 5602f09cd..23ee3e5a4 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocMapper.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocMapper.java
@@ -27,7 +27,7 @@ import java.util.Optional;
import java.util.function.Function;
import java.util.regex.Pattern;
import net.ripe.ipresource.IpResourceRange;
-import org.apache.baremaps.geocoder.GeonamesQueryBuilder;
+import org.apache.baremaps.geocoder.geonames.GeonamesQueryBuilder;
import org.apache.baremaps.utils.IsoCountriesUtils;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SearcherManager;
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java
index 1b258922b..c4fcf0c82 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeocoderOpenStreetMap.java
@@ -24,7 +24,7 @@ import java.util.List;
import java.util.Map;
import java.util.StringJoiner;
import org.apache.baremaps.geocoder.GeocoderConstants;
-import org.apache.baremaps.geocoderosm.GeocoderOsmConsumerEntity;
+import org.apache.baremaps.geocoder.openstreetmap.OpenStreetMapEntityConsumer;
import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader;
import org.apache.baremaps.openstreetmap.stream.StreamUtils;
import org.apache.baremaps.workflow.Task;
@@ -75,7 +75,7 @@ public class CreateGeocoderOpenStreetMap implements Task {
var config = new IndexWriterConfig(GeocoderConstants.ANALYZER);
try (var indexWriter = new IndexWriter(directory, config)) {
- var importer = new GeocoderOsmConsumerEntity(indexWriter);
+ var importer = new OpenStreetMapEntityConsumer(indexWriter);
execute(
path,
coordinateMap,
@@ -88,7 +88,7 @@ public class CreateGeocoderOpenStreetMap implements Task {
Path path,
Map<Long, Coordinate> coordinateMap,
Map<Long, List<Long>> referenceMap,
- GeocoderOsmConsumerEntity importer) throws IOException {
+ OpenStreetMapEntityConsumer importer) throws IOException {
// configure the block reader
var reader = new PbfEntityReader()
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java
index 07655c37e..f15160483 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java
@@ -22,8 +22,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.StringJoiner;
import org.apache.baremaps.geocoder.GeocoderConstants;
-import org.apache.baremaps.geocoder.GeonamesDocumentMapper;
-import org.apache.baremaps.geocoder.GeonamesReader;
+import org.apache.baremaps.geocoder.geonames.GeonamesDocumentMapper;
+import org.apache.baremaps.geocoder.geonames.GeonamesReader;
import org.apache.baremaps.workflow.Task;
import org.apache.baremaps.workflow.WorkflowContext;
import org.apache.lucene.document.Document;
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/geocoder/GeonamesIndexTest.java b/baremaps-core/src/test/java/org/apache/baremaps/geocoder/geonames/GeonamesIndexTest.java
similarity index 98%
rename from baremaps-core/src/test/java/org/apache/baremaps/geocoder/GeonamesIndexTest.java
rename to baremaps-core/src/test/java/org/apache/baremaps/geocoder/geonames/GeonamesIndexTest.java
index b9d392ff2..0effcf9ad 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/geocoder/GeonamesIndexTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/geocoder/geonames/GeonamesIndexTest.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.baremaps.geocoder;
+package org.apache.baremaps.geocoder.geonames;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -24,6 +24,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
+
import org.apache.baremaps.testing.TestFiles;
import org.apache.baremaps.utils.FileUtils;
import org.apache.baremaps.workflow.WorkflowContext;
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/geocoder/GeonamesReaderTest.java b/baremaps-core/src/test/java/org/apache/baremaps/geocoder/geonames/GeonamesReaderTest.java
similarity index 96%
rename from baremaps-core/src/test/java/org/apache/baremaps/geocoder/GeonamesReaderTest.java
rename to baremaps-core/src/test/java/org/apache/baremaps/geocoder/geonames/GeonamesReaderTest.java
index efc90e6d9..3f3170f95 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/geocoder/GeonamesReaderTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/geocoder/geonames/GeonamesReaderTest.java
@@ -15,12 +15,13 @@
* limitations under the License.
*/
-package org.apache.baremaps.geocoder;
+package org.apache.baremaps.geocoder.geonames;
import static org.junit.jupiter.api.Assertions.*;
import java.io.IOException;
import java.nio.file.Files;
+
import org.apache.baremaps.testing.TestFiles;
import org.junit.jupiter.api.Test;
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/geocoderosm/OSMIndexTest.java b/baremaps-core/src/test/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapIndexTest.java
similarity index 97%
rename from baremaps-core/src/test/java/org/apache/baremaps/geocoderosm/OSMIndexTest.java
rename to baremaps-core/src/test/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapIndexTest.java
index f267f6007..4d74343e6 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/geocoderosm/OSMIndexTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/geocoder/openstreetmap/OpenStreetMapIndexTest.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.baremaps.geocoderosm;
+package org.apache.baremaps.geocoder.openstreetmap;
import static org.apache.baremaps.testing.TestFiles.SAMPLE_OSM_PBF;
import static org.junit.jupiter.api.Assertions.*;
@@ -27,6 +27,7 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+
import org.apache.baremaps.utils.FileUtils;
import org.apache.baremaps.workflow.WorkflowContext;
import org.apache.baremaps.workflow.tasks.CreateGeocoderOpenStreetMap;
@@ -49,7 +50,7 @@ import org.junit.jupiter.api.Test;
@Disabled("prototype implementation")
-public class OSMIndexTest {
+public class OpenStreetMapIndexTest {
private static Path directory;
private static IndexSearcher searcher;
@@ -76,7 +77,7 @@ public class OSMIndexTest {
@Test
void testCreateIndex() throws Exception {
var query =
- new GeocoderOsmQuery("vaduz").build();
+ new OpenStreetMapQuery("vaduz").build();
var topDocs = searcher.search(query, 1);
var doc = searcher.doc(Arrays.stream(topDocs.scoreDocs).findFirst().get().doc);
assertEquals("Vaduz", doc.getField("name").stringValue());
diff --git a/baremaps-geocoder/pom.xml b/baremaps-geocoder/pom.xml
new file mode 100644
index 000000000..8214f6b70
--- /dev/null
+++ b/baremaps-geocoder/pom.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.baremaps</groupId>
+ <artifactId>baremaps</artifactId>
+ <version>0.7.4-SNAPSHOT</version>
+ </parent>
+ <artifactId>baremaps-geocoder</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-expressions</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-spatial-extras</artifactId>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/baremaps-server/src/main/java/org/apache/baremaps/server/GeocoderResource.java b/baremaps-server/src/main/java/org/apache/baremaps/server/GeocoderResource.java
index 5a6d595da..071e6521e 100644
--- a/baremaps-server/src/main/java/org/apache/baremaps/server/GeocoderResource.java
+++ b/baremaps-server/src/main/java/org/apache/baremaps/server/GeocoderResource.java
@@ -33,7 +33,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.baremaps.geocoder.GeonamesQueryBuilder;
+import org.apache.baremaps.geocoder.geonames.GeonamesQueryBuilder;
import org.apache.baremaps.openstreetmap.stream.StreamException;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
diff --git a/pom.xml b/pom.xml
index 07acbfcdf..ffabeb341 100644
--- a/pom.xml
+++ b/pom.xml
@@ -59,6 +59,7 @@ limitations under the License.
<module>baremaps-pmtiles</module>
<module>baremaps-server</module>
<module>baremaps-testing</module>
+ <module>baremaps-geocoder</module>
</modules>
<scm>
(incubator-baremaps) 02/02: Add consumer, mapper and query for datatable abstration
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch overturemaps-lucene
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit f871323a69fe3edfdf80a7edd5e25866adf28952
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Thu Oct 31 14:43:23 2024 +0100
Add consumer, mapper and query for datatable abstration
---
.../apache/baremaps/geocoder/DataRowConsumer.java | 48 +++++
.../apache/baremaps/geocoder/DataRowMapper.java | 232 +++++++++++++++++++++
.../apache/baremaps/geocoder/DataTableQuery.java | 45 ++++
baremaps-geocoder/pom.xml | 30 ---
4 files changed, 325 insertions(+), 30 deletions(-)
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowConsumer.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowConsumer.java
new file mode 100644
index 000000000..175afe89a
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowConsumer.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.geocoder;
+
+import org.apache.baremaps.data.storage.DataRow;
+import org.apache.lucene.index.IndexWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.function.Consumer;
+
+public class DataRowConsumer implements Consumer<DataRow> {
+
+ private static final Logger logger = LoggerFactory.getLogger(DataRowConsumer.class);
+
+ private final IndexWriter indexWriter;
+
+ private final DataRowMapper dataRowMapper = new DataRowMapper();
+
+ public DataRowConsumer(IndexWriter indexWriter) {
+ this.indexWriter = indexWriter;
+ }
+
+ @Override
+ public void accept(DataRow row) {
+ try {
+ var document = dataRowMapper.apply(row);
+ indexWriter.addDocument(document);
+ } catch (Exception e) {
+ logger.warn("The following row ({}) is not processed due to {}", row, e);
+ }
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowMapper.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowMapper.java
new file mode 100644
index 000000000..3938d42cf
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowMapper.java
@@ -0,0 +1,232 @@
+package org.apache.baremaps.geocoder;
+
+import org.apache.baremaps.data.storage.DataColumn;
+import org.apache.baremaps.data.storage.DataRow;
+import org.apache.baremaps.data.storage.DataSchema;
+import org.locationtech.jts.geom.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.FloatPoint;
+import org.apache.lucene.document.DoublePoint;
+import org.apache.lucene.document.LatLonPoint;
+import org.apache.lucene.document.LatLonShape;
+
+import java.net.InetAddress;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.function.Function;
+
+public class DataRowMapper implements Function<DataRow, Document> {
+
+ private static final Logger logger = LoggerFactory.getLogger(DataRowMapper.class);
+
+ @Override
+ public Document apply(DataRow dataRow) {
+ Document doc = new Document();
+ DataSchema schema = dataRow.schema();
+ List<DataColumn> columns = schema.columns();
+ for (int i = 0; i < columns.size(); i++) {
+ DataColumn column = columns.get(i);
+ String columnName = column.name();
+ Object value = dataRow.get(i);
+ if (value == null) continue;
+ DataColumn.Type type = column.type();
+
+ try {
+ switch (type) {
+ case BINARY:
+ doc.add(new StoredField(columnName, (byte[]) value));
+ break;
+ case BYTE:
+ doc.add(new IntPoint(columnName, ((Byte) value).intValue()));
+ doc.add(new StoredField(columnName, ((Byte) value).intValue()));
+ break;
+ case BOOLEAN:
+ doc.add(new StringField(columnName, value.toString(), Field.Store.YES));
+ break;
+ case SHORT:
+ doc.add(new IntPoint(columnName, ((Short) value).intValue()));
+ doc.add(new StoredField(columnName, ((Short) value).intValue()));
+ break;
+ case INTEGER:
+ doc.add(new IntPoint(columnName, (Integer) value));
+ doc.add(new StoredField(columnName, (Integer) value));
+ break;
+ case LONG:
+ doc.add(new LongPoint(columnName, (Long) value));
+ doc.add(new StoredField(columnName, (Long) value));
+ break;
+ case FLOAT:
+ doc.add(new FloatPoint(columnName, (Float) value));
+ doc.add(new StoredField(columnName, (Float) value));
+ break;
+ case DOUBLE:
+ doc.add(new DoublePoint(columnName, (Double) value));
+ doc.add(new StoredField(columnName, (Double) value));
+ break;
+ case STRING:
+ doc.add(new TextField(columnName, (String) value, Field.Store.YES));
+ break;
+ case COORDINATE:
+ Coordinate coord = (Coordinate) value;
+ double lat = coord.getY();
+ double lon = coord.getX();
+ doc.add(new LatLonPoint(columnName, lat, lon));
+ doc.add(new StoredField(columnName + "_lat", lat));
+ doc.add(new StoredField(columnName + "_lon", lon));
+ break;
+ case POINT:
+ Point point = (Point) value;
+ double pointLat = point.getY();
+ double pointLon = point.getX();
+ doc.add(new LatLonPoint(columnName, pointLat, pointLon));
+ doc.add(new StoredField(columnName + "_lat", pointLat));
+ doc.add(new StoredField(columnName + "_lon", pointLon));
+ break;
+ case LINESTRING:
+ case POLYGON:
+ case MULTIPOINT:
+ case MULTILINESTRING:
+ case MULTIPOLYGON:
+ case GEOMETRYCOLLECTION:
+ case GEOMETRY:
+ Geometry geometry = (Geometry) value;
+ if (geometry != null) {
+ Field[] shapeFields = createShapeFields(columnName, geometry);
+ for (Field field : shapeFields) {
+ doc.add(field);
+ }
+ doc.add(new StoredField(columnName + "_wkt", geometry.toText()));
+ }
+ break;
+ case ENVELOPE:
+ Envelope envelope = (Envelope) value;
+ String envelopeStr = envelope.toString();
+ doc.add(new StringField(columnName, envelopeStr, Field.Store.YES));
+ break;
+ case INET_ADDRESS:
+ case INET4_ADDRESS:
+ case INET6_ADDRESS:
+ InetAddress addr = (InetAddress) value;
+ doc.add(new StringField(columnName, addr.getHostAddress(), Field.Store.YES));
+ break;
+ case LOCAL_DATE:
+ LocalDate date = (LocalDate) value;
+ doc.add(new StringField(columnName, date.toString(), Field.Store.YES));
+ break;
+ case LOCAL_TIME:
+ LocalTime time = (LocalTime) value;
+ doc.add(new StringField(columnName, time.toString(), Field.Store.YES));
+ break;
+ case LOCAL_DATE_TIME:
+ LocalDateTime dateTime = (LocalDateTime) value;
+ doc.add(new StringField(columnName, dateTime.toString(), Field.Store.YES));
+ break;
+ case NESTED:
+ Map<String, Object> map = (Map<String, Object>) value;
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ String nestedKey = columnName + "." + entry.getKey();
+ Object nestedValue = entry.getValue();
+ if (nestedValue != null) {
+ doc.add(new TextField(nestedKey, nestedValue.toString(), Field.Store.YES));
+ }
+ }
+ break;
+ default:
+ doc.add(new StringField(columnName, value.toString(), Field.Store.YES));
+ break;
+ }
+ } catch (Exception e) {
+ logger.error("Error processing column '{}' with value '{}': {}", columnName, value, e.getMessage());
+ }
+ }
+ return doc;
+ }
+
+ private Field[] createShapeFields(String fieldName, Geometry geometry) {
+ if (geometry instanceof Point point) {
+ double lat = point.getY();
+ double lon = point.getX();
+ return new Field[] { new LatLonPoint(fieldName, lat, lon) };
+ } else if (geometry instanceof LineString lineString) {
+ return LatLonShape.createIndexableFields(fieldName, convertToLuceneLine(lineString));
+ } else if (geometry instanceof Polygon polygon) {
+ org.apache.lucene.geo.Polygon lucenePolygon = convertToLucenePolygon(polygon);
+ return LatLonShape.createIndexableFields(fieldName, lucenePolygon);
+ } else if (geometry instanceof MultiPolygon multiPolygon) {
+ return createFieldsFromMultiPolygon(fieldName, multiPolygon);
+ } else if (geometry instanceof GeometryCollection collection) {
+ List<Field> fieldList = new ArrayList<>();
+ for (int i = 0; i < collection.getNumGeometries(); i++) {
+ Geometry geom = collection.getGeometryN(i);
+ Field[] fields = createShapeFields(fieldName, geom);
+ fieldList.addAll(Arrays.asList(fields));
+ }
+ return fieldList.toArray(new Field[0]);
+ } else {
+ logger.warn("Unsupported geometry type '{}' for field '{}'", geometry.getGeometryType(), fieldName);
+ return new Field[0];
+ }
+ }
+
+ private org.apache.lucene.geo.Line convertToLuceneLine(LineString lineString) {
+ Coordinate[] coords = lineString.getCoordinates();
+ double[] lats = new double[coords.length];
+ double[] lons = new double[coords.length];
+ for (int i = 0; i < coords.length; i++) {
+ lats[i] = coords[i].getY();
+ lons[i] = coords[i].getX();
+ }
+ return new org.apache.lucene.geo.Line(lats, lons);
+ }
+
+ private org.apache.lucene.geo.Polygon convertToLucenePolygon(org.locationtech.jts.geom.Polygon jtsPolygon) {
+ LinearRing shell = jtsPolygon.getExteriorRing();
+ Coordinate[] shellCoords = shell.getCoordinates();
+ double[] lats = new double[shellCoords.length];
+ double[] lons = new double[shellCoords.length];
+ for (int i = 0; i < shellCoords.length; i++) {
+ lats[i] = shellCoords[i].getY();
+ lons[i] = shellCoords[i].getX();
+ }
+
+ int numHoles = jtsPolygon.getNumInteriorRing();
+ org.apache.lucene.geo.Polygon[] holes = new org.apache.lucene.geo.Polygon[numHoles];
+ for (int i = 0; i < numHoles; i++) {
+ LinearRing hole = jtsPolygon.getInteriorRingN(i);
+ Coordinate[] holeCoords = hole.getCoordinates();
+ double[] holeLats = new double[holeCoords.length];
+ double[] holeLons = new double[holeCoords.length];
+ for (int j = 0; j < holeCoords.length; j++) {
+ holeLats[j] = holeCoords[j].getY();
+ holeLons[j] = holeCoords[j].getX();
+ }
+ holes[i] = new org.apache.lucene.geo.Polygon(holeLats, holeLons);
+ }
+
+ return new org.apache.lucene.geo.Polygon(lats, lons, holes);
+ }
+
+ private Field[] createFieldsFromMultiPolygon(String fieldName, MultiPolygon multiPolygon) {
+ List<Field> fieldList = new ArrayList<>();
+ for (int i = 0; i < multiPolygon.getNumGeometries(); i++) {
+ org.locationtech.jts.geom.Polygon polygon = (org.locationtech.jts.geom.Polygon) multiPolygon.getGeometryN(i);
+ org.apache.lucene.geo.Polygon lucenePolygon = convertToLucenePolygon(polygon);
+ Field[] fields = LatLonShape.createIndexableFields(fieldName, lucenePolygon);
+ fieldList.addAll(Arrays.asList(fields));
+ }
+ return fieldList.toArray(new Field[0]);
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataTableQuery.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataTableQuery.java
new file mode 100644
index 000000000..0d3c6dfc0
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataTableQuery.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.geocoder;
+
+import org.apache.baremaps.geocoder.openstreetmap.OsmTags;
+import org.apache.lucene.queryparser.classic.QueryParserBase;
+import org.apache.lucene.queryparser.simple.SimpleQueryParser;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Query;
+
+public class DataTableQuery {
+
+ private final String query;
+
+ public DataTableQuery(String query) {
+ this.query = query;
+ }
+
+ public Query build() {
+ var builder = new BooleanQuery.Builder();
+ var queryTextEsc = QueryParserBase.escape(query);
+
+ var parser = new SimpleQueryParser(GeocoderConstants.ANALYZER, OsmTags.NAME.key());
+ var termsQuery = parser.parse(queryTextEsc);
+ // at least one terms of the queryText must be present
+ builder.add(termsQuery, BooleanClause.Occur.MUST);
+ return builder.build();
+ }
+}
diff --git a/baremaps-geocoder/pom.xml b/baremaps-geocoder/pom.xml
deleted file mode 100644
index 8214f6b70..000000000
--- a/baremaps-geocoder/pom.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.baremaps</groupId>
- <artifactId>baremaps</artifactId>
- <version>0.7.4-SNAPSHOT</version>
- </parent>
- <artifactId>baremaps-geocoder</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-expressions</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-queryparser</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-spatial-extras</artifactId>
- </dependency>
- </dependencies>
-</project>
\ No newline at end of file