You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2022/12/22 16:48:54 UTC
[incubator-baremaps] 01/01: Simplify the design of the geocoder
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 562-simplify-geocoder
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 81a09eaf4771b2c498ee34491f5e5fa053df3845
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Thu Dec 22 17:48:39 2022 +0100
Simplify the design of the geocoder
---
.run/geocoder-create.run.xml | 5 +
.run/geocoder-serve.run.xml | 7 +-
.run/iploc-create.run.xml | 5 +
.run/iploc-serve.run.xml | 5 +
.../benchmarks/OpenStreetMapBenchmark.java | 2 +-
.../OpenStreetMapGeometriesBenchmark.java | 2 +-
.../org/apache/baremaps/cli/geocoder/Search.java | 40 ++++--
.../org/apache/baremaps/cli/geocoder/Serve.java | 13 +-
baremaps-core/pom.xml | 10 +-
.../org/apache/baremaps/database/DiffService.java | 2 +-
.../apache/baremaps/database/ImportService.java | 2 +-
.../apache/baremaps/database/UpdateService.java | 2 +-
.../DocumentReader.java} | 9 +-
.../org/apache/baremaps/geocoder/Geocoder.java | 126 ------------------
.../{response/Response.java => QueryBuilder.java} | 10 +-
.../geonames/GeonamesConstants.java} | 11 +-
.../geocoder/geonames/GeonamesDocumentMapper.java | 50 ++++++++
.../geocoder/geonames/GeonamesGeocoder.java | 118 -----------------
.../geocoder/geonames/GeonamesQueryBuilder.java | 77 +++++++++++
.../baremaps/geocoder/geonames/GeonamesReader.java | 50 ++++++++
.../baremaps/geocoder/geonames/GeonamesRecord.java | 2 +-
.../{utils => geonames}/IsoCountriesUtils.java | 2 +-
.../openstreetmap/OpenstreetmapDocumentReader.java | 82 ++++++++++++
.../OpenstreetmapRecord.java} | 6 +-
.../apache/baremaps/geocoder/request/Request.java | 53 --------
.../apache/baremaps/geocoder/response/Data.java | 90 -------------
.../main/java/org/apache/baremaps/iploc/IpLoc.java | 65 ++++++----
.../database/InetnumLocationDaoSqliteImpl.java | 4 -
.../{OsmReader.java => DataReader.java} | 5 +-
.../baremaps/openstreetmap/pbf/PbfBlockReader.java | 6 +-
.../openstreetmap/pbf/PbfEntityReader.java | 8 +-
.../openstreetmap/xml/XmlChangeReader.java | 6 +-
.../openstreetmap/xml/XmlEntityReader.java | 6 +-
.../workflow/tasks/CreateGeonamesIndex.java | 47 +++++--
.../baremaps/workflow/tasks/CreateIplocIndex.java | 28 ++--
.../database/database/BlockImporterTest.java | 2 +-
.../org/apache/baremaps/geocoder/GeocoderTest.java | 79 ------------
.../geocoder/geonames/GeonamesGeocoderTest.java | 73 -----------
.../java/org/apache/baremaps/iploc/IpLocTest.java | 21 +--
.../baremaps/openstreetmap/OpenStreetMapTest.java | 23 ++--
.../geometry/RelationGeometryBuilderTest.java | 2 +-
.../apache/baremaps/server/GeocoderResources.java | 68 +++++++---
.../src/main/resources/geocoder/index.html | 142 +++++++--------------
examples/geocoding/workflow.js | 4 +-
examples/ip-to-location/workflow.js | 73 ++++++++---
pom.xml | 10 +-
46 files changed, 629 insertions(+), 824 deletions(-)
diff --git a/.run/geocoder-create.run.xml b/.run/geocoder-create.run.xml
index d6e2596c..9b1447cb 100644
--- a/.run/geocoder-create.run.xml
+++ b/.run/geocoder-create.run.xml
@@ -3,6 +3,11 @@
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
<option name="PROGRAM_PARAMETERS" value="workflow execute --file examples/geocoding/workflow.js" />
+ <extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+ <option name="credential" />
+ <option name="region" />
+ <option name="useCurrentConnection" value="false" />
+ </extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
diff --git a/.run/geocoder-serve.run.xml b/.run/geocoder-serve.run.xml
index 5132b1e7..97bee42d 100644
--- a/.run/geocoder-serve.run.xml
+++ b/.run/geocoder-serve.run.xml
@@ -1,10 +1,15 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="geocoder-serve" type="Application" factoryName="Application">
- <option name="ALTERNATIVE_JRE_PATH" value="temurin-17" />
+ <option name="ALTERNATIVE_JRE_PATH" value="$USER_HOME$/.sdkman/candidates/java/17.0.2-open" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="true" />
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
<option name="PROGRAM_PARAMETERS" value="geocoder serve --index geocoder-index --port 9000" />
+ <extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+ <option name="credential" />
+ <option name="region" />
+ <option name="useCurrentConnection" value="false" />
+ </extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
diff --git a/.run/iploc-create.run.xml b/.run/iploc-create.run.xml
index 2b434f84..07a0c61a 100644
--- a/.run/iploc-create.run.xml
+++ b/.run/iploc-create.run.xml
@@ -3,6 +3,11 @@
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
<option name="PROGRAM_PARAMETERS" value="workflow execute --file examples/ip-to-location/workflow.js" />
+ <extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+ <option name="credential" />
+ <option name="region" />
+ <option name="useCurrentConnection" value="false" />
+ </extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
diff --git a/.run/iploc-serve.run.xml b/.run/iploc-serve.run.xml
index 26bde019..aa9839c8 100644
--- a/.run/iploc-serve.run.xml
+++ b/.run/iploc-serve.run.xml
@@ -3,6 +3,11 @@
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
<option name="PROGRAM_PARAMETERS" value="iploc serve --database iploc.db --port 9000" />
+ <extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+ <option name="credential" />
+ <option name="region" />
+ <option name="useCurrentConnection" value="false" />
+ </extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/OpenStreetMapBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/OpenStreetMapBenchmark.java
index cb3f9c7f..efefe394 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/OpenStreetMapBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/OpenStreetMapBenchmark.java
@@ -71,7 +71,7 @@ public class OpenStreetMapBenchmark {
AtomicLong relations = new AtomicLong(0);
try (InputStream inputStream = new BufferedInputStream(Files.newInputStream(path))) {
- new PbfEntityReader(new PbfBlockReader()).stream(inputStream).forEach(entity -> {
+ new PbfEntityReader(new PbfBlockReader()).read(inputStream).forEach(entity -> {
if (entity instanceof Node node) {
nodes.incrementAndGet();
} else if (entity instanceof Way way) {
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/OpenStreetMapGeometriesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/OpenStreetMapGeometriesBenchmark.java
index e7e38aff..29a68777 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/OpenStreetMapGeometriesBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/OpenStreetMapGeometriesBenchmark.java
@@ -86,7 +86,7 @@ public class OpenStreetMapGeometriesBenchmark {
AtomicLong relations = new AtomicLong(0);
try (InputStream inputStream = new BufferedInputStream(Files.newInputStream(path))) {
new PbfEntityReader(new PbfBlockReader().coordinateMap(coordinateMap)
- .referenceMap(referenceMap).projection(4326)).stream(inputStream).forEach(entity -> {
+ .referenceMap(referenceMap).projection(4326)).read(inputStream).forEach(entity -> {
if (entity instanceof Node node) {
nodes.incrementAndGet();
} else if (entity instanceof Way way) {
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Search.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Search.java
index 29024ee4..1269323c 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Search.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Search.java
@@ -16,8 +16,9 @@ package org.apache.baremaps.cli.geocoder;
import java.nio.file.Path;
import java.util.concurrent.Callable;
-import org.apache.baremaps.geocoder.geonames.GeonamesGeocoder;
-import org.apache.baremaps.geocoder.request.Request;
+import org.apache.baremaps.geocoder.geonames.GeonamesQueryBuilder;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.MMapDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine.Command;
@@ -30,19 +31,38 @@ public class Search implements Callable<Integer> {
@Option(names = {"--index"}, paramLabel = "INDEX", description = "The path to the lucene index.",
required = true)
- private Path indexPath;
+ private Path indexDirectory;
- @Option(names = {"--search"}, paramLabel = "SEARCH",
+ @Option(names = {"--terms"}, paramLabel = "terms",
description = "The terms to search in the index.", required = true)
- private String search;
+ private String terms;
+
+ @Option(names = {"--country"}, paramLabel = "COUNTRY", description = "The country code filter.",
+ required = false)
+ private String countryCode = "";
+
+ @Option(names = {"--limit"}, paramLabel = "LIMIT",
+ description = "The number of result to return.", required = false)
+ private Integer limit = 10;
@Override
public Integer call() throws Exception {
- var geocoder = new GeonamesGeocoder(indexPath, null);
- geocoder.open();
- var request = new Request(search, 20);
- var response = geocoder.search(request);
- logger.info("{}", response);
+ var directory = MMapDirectory.open(indexDirectory);
+
+ var query = new GeonamesQueryBuilder().queryText(terms).countryCode(countryCode).build();
+
+ var searcherManager = new SearcherManager(directory, new SearcherFactory());
+ var searcher = searcherManager.acquire();
+ try {
+ var result = searcher.search(query, limit);
+ for (var hit : result.scoreDocs) {
+ var document = searcher.doc(hit.doc);
+ logger.info("{}", document);
+ }
+ } finally {
+ searcherManager.release(searcher);
+ }
+
return 0;
}
}
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Serve.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Serve.java
index 097cdb71..e54b2180 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Serve.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/geocoder/Serve.java
@@ -18,10 +18,11 @@ import io.servicetalk.http.netty.HttpServers;
import io.servicetalk.http.router.jersey.HttpJerseyRouterBuilder;
import java.nio.file.Path;
import java.util.concurrent.Callable;
-import org.apache.baremaps.geocoder.Geocoder;
-import org.apache.baremaps.geocoder.geonames.GeonamesGeocoder;
import org.apache.baremaps.server.CorsFilter;
import org.apache.baremaps.server.GeocoderResources;
+import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.store.MMapDirectory;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.server.ResourceConfig;
import org.slf4j.Logger;
@@ -36,7 +37,7 @@ public class Serve implements Callable<Integer> {
@Option(names = {"--index"}, paramLabel = "INDEX", description = "The path to the lucene index.",
required = true)
- private Path indexPath;
+ private Path indexDirectory;
@Option(names = {"--host"}, paramLabel = "HOST", description = "The host of the server.")
private String host = "localhost";
@@ -47,15 +48,15 @@ public class Serve implements Callable<Integer> {
@Override
public Integer call() throws Exception {
- var geocoder = new GeonamesGeocoder(indexPath, null);
- geocoder.open();
+ var directory = MMapDirectory.open(indexDirectory);
+ var searcherManager = new SearcherManager(directory, new SearcherFactory());
// Configure the application
var application = new ResourceConfig().register(CorsFilter.class)
.register(GeocoderResources.class).register(new AbstractBinder() {
@Override
protected void configure() {
- bind(geocoder).to(Geocoder.class).named("geocoder");
+ bind(searcherManager).to(SearcherManager.class).named("searcherManager");
}
});
diff --git a/baremaps-core/pom.xml b/baremaps-core/pom.xml
index c5cc7fb5..6a103615 100644
--- a/baremaps-core/pom.xml
+++ b/baremaps-core/pom.xml
@@ -78,10 +78,6 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers-common</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
@@ -92,11 +88,7 @@
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
- <artifactId>lucene-replicator</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-spatial</artifactId>
+ <artifactId>lucene-spatial-extras</artifactId>
</dependency>
<dependency>
<groupId>org.apache.sis.core</groupId>
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/DiffService.java b/baremaps-core/src/main/java/org/apache/baremaps/database/DiffService.java
index 289c7bdc..bebfdbe6 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/database/DiffService.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/database/DiffService.java
@@ -80,7 +80,7 @@ public class DiffService implements Callable<List<Tile>> {
var projectionTransformer = new ProjectionTransformer(srid, 4326);
try (var changeInputStream =
new GZIPInputStream(new BufferedInputStream(changeUrl.openStream()))) {
- return new XmlChangeReader().stream(changeInputStream).flatMap(this::geometriesForChange)
+ return new XmlChangeReader().read(changeInputStream).flatMap(this::geometriesForChange)
.map(projectionTransformer::transform).flatMap(this::tilesForGeometry).distinct()
.toList();
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/ImportService.java b/baremaps-core/src/main/java/org/apache/baremaps/database/ImportService.java
index ab1b58ad..0745da84 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/database/ImportService.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/database/ImportService.java
@@ -69,7 +69,7 @@ public class ImportService implements Callable<Void> {
var blockImporter =
new BlockImporter(headerRepository, nodeRepository, wayRepository, relationRepository);
try (InputStream inputStream = Files.newInputStream(path)) {
- batch(new PbfBlockReader().stream(inputStream).map(blockMapper)).forEach(blockImporter);
+ batch(new PbfBlockReader().read(inputStream).map(blockMapper)).forEach(blockImporter);
}
return null;
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/UpdateService.java b/baremaps-core/src/main/java/org/apache/baremaps/database/UpdateService.java
index 39fe14cb..6a28f9d6 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/database/UpdateService.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/database/UpdateService.java
@@ -73,7 +73,7 @@ public class UpdateService implements Callable<Void> {
var changeUrl = resolve(replicationUrl, sequenceNumber, "osc.gz");
try (var changeInputStream =
new GZIPInputStream(new BufferedInputStream(changeUrl.openStream()))) {
- new XmlChangeReader().stream(changeInputStream).map(prepareChange).forEach(saveChange);
+ new XmlChangeReader().read(changeInputStream).map(prepareChange).forEach(saveChange);
}
var stateUrl = resolve(replicationUrl, sequenceNumber, "state.txt");
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/OsmReader.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DocumentReader.java
similarity index 74%
copy from baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/OsmReader.java
copy to baremaps-core/src/main/java/org/apache/baremaps/geocoder/DocumentReader.java
index f0356531..31d3757c 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/OsmReader.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DocumentReader.java
@@ -10,14 +10,17 @@
* the License.
*/
-package org.apache.baremaps.openstreetmap;
+package org.apache.baremaps.geocoder;
+import java.io.IOException;
import java.io.InputStream;
import java.util.stream.Stream;
+import org.apache.lucene.document.Document;
-public interface OsmReader<T> {
+public interface DocumentReader {
+
+ Stream<Document> read(InputStream inputStream) throws IOException;
- Stream<T> stream(InputStream inputStream);
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/Geocoder.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/Geocoder.java
deleted file mode 100644
index 2f1fb198..00000000
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/Geocoder.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.baremaps.geocoder;
-
-
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Stream;
-import org.apache.baremaps.geocoder.request.Request;
-import org.apache.baremaps.geocoder.response.Data;
-import org.apache.baremaps.geocoder.response.Response;
-import org.apache.baremaps.geocoder.response.Result;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.SearcherFactory;
-import org.apache.lucene.search.SearcherManager;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
-
-public abstract class Geocoder implements AutoCloseable {
-
- private final Directory directory;
- private SearcherManager searcherManager;
- private Analyzer analyzer = analyzer();
-
- public Geocoder(Path index) throws IOException {
- this.directory = MMapDirectory.open(index);
- }
-
- public boolean indexExists() throws IOException {
- return DirectoryReader.indexExists(directory);
- }
-
- public void open() throws IOException {
- if (!DirectoryReader.indexExists(directory)) {
- throw new IllegalStateException("Invalid Lucene index directory");
- }
- searcherManager = new SearcherManager(directory, new SearcherFactory());
- }
-
- public void build() throws IOException {
- build(documents()::iterator);
- }
-
- private void build(Iterable<Document> documents) throws IOException {
- IndexWriterConfig config = new IndexWriterConfig(analyzer);
- try (IndexWriter indexWriter = new IndexWriter(directory, config)) {
- indexWriter.deleteAll();
- indexWriter.addDocuments(documents);
- } catch (IOException exception) {
- throw new RuntimeException();
- }
- searcherManager = new SearcherManager(directory, new SearcherFactory());
- }
-
- public Response search(Request request) throws IOException, ParseException {
- IndexSearcher searcher = searcherManager.acquire();
- List<Result> results = new ArrayList<>();
- try {
- TopDocs topDocs = searcher.search(query(analyzer, request), request.limit());
- for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
- Document document = searcher.doc(scoreDoc.doc);
- Data data = new Data(document.get("name"), document.get("asciiname"),
- document.get("alternatenames"),
- document.getField("latitude") != null
- ? document.getField("latitude").numericValue().doubleValue()
- : null,
- document.getField("longitude") != null
- ? document.getField("longitude").numericValue().doubleValue()
- : null,
- document.get("featureClass"), document.get("featureCode"), document.get("countryCode"),
- document.get("cc2"), document.get("admin1Code"), document.get("admin2Code"),
- document.get("admin3Code"), document.get("admin4Code"),
- document.getField("population") != null
- ? document.getField("population").numericValue().longValue()
- : null,
- document.getField("elevation") != null
- ? document.getField("elevation").numericValue().intValue()
- : null,
- document.getField("dem") != null ? document.getField("dem").numericValue().intValue()
- : null,
- document.get("timezone"), document.get("modificationDate"));
- results.add(new Result(scoreDoc.score, data));
- }
- return new Response(results);
- } finally {
- searcherManager.release(searcher);
- }
- }
-
- @Override
- public void close() throws IOException {
- analyzer.close();
- directory.close();
- if (searcherManager != null) {
- searcherManager.close();
- }
- }
-
- protected abstract Analyzer analyzer() throws IOException;
-
- protected abstract Stream<Document> documents() throws IOException;
-
- protected abstract Query query(Analyzer analyzer, Request request) throws ParseException;
-}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/response/Response.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/QueryBuilder.java
similarity index 81%
rename from baremaps-core/src/main/java/org/apache/baremaps/geocoder/response/Response.java
rename to baremaps-core/src/main/java/org/apache/baremaps/geocoder/QueryBuilder.java
index e1933dcb..d2490093 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/response/Response.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/QueryBuilder.java
@@ -10,8 +10,12 @@
* the License.
*/
-package org.apache.baremaps.geocoder.response;
+package org.apache.baremaps.geocoder;
-import java.util.List;
-public record Response(List<Result> results) {}
+
+import org.apache.lucene.search.Query;
+
+public interface QueryBuilder {
+ Query build();
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/OsmReader.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesConstants.java
similarity index 68%
copy from baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/OsmReader.java
copy to baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesConstants.java
index f0356531..ba8dafff 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/OsmReader.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesConstants.java
@@ -10,14 +10,15 @@
* the License.
*/
-package org.apache.baremaps.openstreetmap;
+package org.apache.baremaps.geocoder.geonames;
-import java.io.InputStream;
-import java.util.stream.Stream;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
-public interface OsmReader<T> {
+public class GeonamesConstants {
+
+ public static final Analyzer ANALYZER = new StandardAnalyzer();
- Stream<T> stream(InputStream inputStream);
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesDocumentMapper.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesDocumentMapper.java
new file mode 100644
index 00000000..ce6e9407
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesDocumentMapper.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.geocoder.geonames;
+
+
+
+import java.util.function.Function;
+import org.apache.lucene.document.*;
+
+public class GeonamesDocumentMapper implements Function<GeonamesRecord, Document> {
+
+ @Override
+ public Document apply(GeonamesRecord record) {
+ Document document = new Document();
+ document.add(new TextField("name", record.getName(), Field.Store.YES));
+ document.add(new TextField("country", IsoCountriesUtils.getCountry(record.getCountryCode()),
+ Field.Store.YES));
+ document.add(new StringField("countryCode", record.getCountryCode(), Field.Store.YES));
+ document.add(new LatLonPoint("point", record.getLatitude(), record.getLongitude()));
+ document.add(new StoredField("longitude", record.getLongitude()));
+ document.add(new StoredField("latitude", record.getLatitude()));
+ document.add(new StoredField("asciiname", record.getAsciiname()));
+ document.add(new StoredField("alternatenames", record.getAlternatenames()));
+ document.add(new StoredField("featureClass", record.getFeatureClass()));
+ document.add(new StoredField("featureCode", record.getFeatureCode()));
+ document.add(new StoredField("cc2", record.getCc2()));
+ document.add(new StoredField("admin1Code", record.getAdmin1Code()));
+ document.add(new StoredField("admin2Code", record.getAdmin2Code()));
+ document.add(new StoredField("admin3Code", record.getAdmin3Code()));
+ document.add(new StoredField("admin4Code", record.getAdmin4Code()));
+ document.add(new StoredField("population", record.getPopulation()));
+ if (record.getElevation() != null) {
+ document.add(new StoredField("elevation", record.getElevation()));
+ }
+ document.add(new StoredField("dem", record.getDem()));
+ document.add(new StoredField("timezone", record.getTimezone()));
+ document.add(new StoredField("modificationDate", record.getModificationDate()));
+ return document;
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesGeocoder.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesGeocoder.java
deleted file mode 100644
index 7ffdf48b..00000000
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesGeocoder.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.baremaps.geocoder.geonames;
-
-
-
-import com.fasterxml.jackson.databind.MappingIterator;
-import com.fasterxml.jackson.dataformat.csv.CsvMapper;
-import com.fasterxml.jackson.dataformat.csv.CsvSchema;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.Spliterators;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-import org.apache.baremaps.geocoder.Geocoder;
-import org.apache.baremaps.geocoder.request.Request;
-import org.apache.baremaps.geocoder.utils.IsoCountriesUtils;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.queryparser.simple.SimpleQueryParser;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.BooleanQuery.Builder;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-
-public class GeonamesGeocoder extends Geocoder {
-
- private Path data;
-
- public GeonamesGeocoder(Path index, Path data) throws IOException {
- super(index);
- this.data = data;
- }
-
- @Override
- protected Analyzer analyzer() {
- return new StandardAnalyzer();
- }
-
- @Override
- protected Stream<Document> documents() throws IOException {
- CsvMapper mapper = new CsvMapper();
- CsvSchema schema = CsvSchema.builder().addColumn("geonameid").addColumn("name")
- .addColumn("asciiname").addColumn("alternatenames").addColumn("latitude")
- .addColumn("longitude").addColumn("featureClass").addColumn("featureCode")
- .addColumn("countryCode").addColumn("cc2").addColumn("admin1Code").addColumn("admin2Code")
- .addColumn("admin3Code").addColumn("admin4Code").addColumn("population")
- .addColumn("elevation").addColumn("dem").addColumn("timezone").addColumn("modificationDate")
- .build().withColumnSeparator('\t').withoutQuoteChar();
- MappingIterator<GeonamesRecord> it =
- mapper.readerFor(GeonamesRecord.class).with(schema).readValues(data.toFile());
- return StreamSupport.stream(Spliterators.spliteratorUnknownSize(it, 0), false).map(record -> {
- Document document = new Document();
- document.add(new TextField("name", record.getName(), Store.YES));
- document.add(new TextField("country", IsoCountriesUtils.getCountry(record.getCountryCode()),
- Store.YES));
- document.add(new StringField("countryCode", record.getCountryCode(), Store.YES));
- document.add(new StoredField("longitude", record.getLongitude()));
- document.add(new StoredField("latitude", record.getLatitude()));
- document.add(new StoredField("asciiname", record.getAsciiname()));
- document.add(new StoredField("alternatenames", record.getAlternatenames()));
- document.add(new StoredField("featureClass", record.getFeatureClass()));
- document.add(new StoredField("featureCode", record.getFeatureCode()));
- document.add(new StoredField("cc2", record.getCc2()));
- document.add(new StoredField("admin1Code", record.getAdmin1Code()));
- document.add(new StoredField("admin2Code", record.getAdmin2Code()));
- document.add(new StoredField("admin3Code", record.getAdmin3Code()));
- document.add(new StoredField("admin4Code", record.getAdmin4Code()));
- document.add(new StoredField("population", record.getPopulation()));
- if (record.getElevation() != null) {
- document.add(new StoredField("elevation", record.getElevation()));
- }
- document.add(new StoredField("dem", record.getDem()));
- document.add(new StoredField("timezone", record.getTimezone()));
- document.add(new StoredField("modificationDate", record.getModificationDate()));
- return document;
- });
- }
-
- @Override
- protected Query query(Analyzer analyzer, Request request) throws ParseException {
- BooleanQuery.Builder builder = new Builder();
- String query = QueryParser.escape(request.query());
- if (!query.isBlank()) {
- SimpleQueryParser nameQueryParser = new SimpleQueryParser(analyzer, "name");
- builder.add(nameQueryParser.parse(query), Occur.SHOULD);
-
- SimpleQueryParser countryQueryParser = new SimpleQueryParser(analyzer, "country");
- builder.add(countryQueryParser.parse(query), Occur.SHOULD);
-
- if (request.countryCode() != null) {
- builder.add(
- new TermQuery(new Term("countryCode", QueryParser.escape(request.countryCode()))),
- Occur.MUST);
- }
- }
- return builder.build();
- }
-}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesQueryBuilder.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesQueryBuilder.java
new file mode 100644
index 00000000..c2cebb9d
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesQueryBuilder.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.geocoder.geonames;
+
+
+
+import java.util.Map;
+import org.apache.baremaps.geocoder.QueryBuilder;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.queryparser.simple.SimpleQueryParser;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+
+public class GeonamesQueryBuilder implements QueryBuilder {
+
+ private final Analyzer analyzer;
+
+ private String queryText;
+
+ private String countryCode = "";
+
+ public GeonamesQueryBuilder() {
+ this(GeonamesConstants.ANALYZER);
+ }
+
+ public GeonamesQueryBuilder(Analyzer analyzer) {
+ this.analyzer = analyzer;
+ }
+
+ public GeonamesQueryBuilder queryText(String queryText) {
+ this.queryText = queryText;
+ return this;
+ }
+
+ public GeonamesQueryBuilder countryCode(String countryCode) {
+ this.countryCode = countryCode;
+ return this;
+ }
+
+ @Override
+ public Query build() {
+ var builder = new BooleanQuery.Builder();
+
+ if (queryText != null) {
+ var queryTextEsc = QueryParser.escape(queryText);
+ if (!queryTextEsc.isBlank()) {
+ var fieldWeights = Map.of("name", 1f, "country", 1f);
+ var termsQuery = new SimpleQueryParser(analyzer, fieldWeights).parse(queryTextEsc);
+ builder.add(termsQuery, BooleanClause.Occur.SHOULD);
+ }
+ }
+
+ if (countryCode != null) {
+ var countryCodeEsc = QueryParser.escape(countryCode);
+ if (!countryCodeEsc.isBlank()) {
+ var countryCodeQuery = new TermQuery(new Term("countryCode", countryCodeEsc));
+ builder.add(countryCodeQuery, BooleanClause.Occur.MUST);
+ }
+ }
+
+ return builder.build();
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesReader.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesReader.java
new file mode 100644
index 00000000..080b1c57
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesReader.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.geocoder.geonames;
+
+
+
+import com.fasterxml.jackson.databind.MappingIterator;
+import com.fasterxml.jackson.dataformat.csv.CsvMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvSchema;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+import org.apache.baremaps.openstreetmap.DataReader;
+
+public class GeonamesReader implements DataReader<GeonamesRecord> {
+
+ @Override
+ public Stream<GeonamesRecord> read(InputStream inputStream) throws IOException {
+ CsvMapper mapper = new CsvMapper();
+
+ CsvSchema schema = CsvSchema.builder().addColumn("geonameid").addColumn("name")
+ .addColumn("asciiname").addColumn("alternatenames").addColumn("latitude")
+ .addColumn("longitude").addColumn("featureClass").addColumn("featureCode")
+ .addColumn("countryCode").addColumn("cc2").addColumn("admin1Code").addColumn("admin2Code")
+ .addColumn("admin3Code").addColumn("admin4Code").addColumn("population")
+ .addColumn("elevation").addColumn("dem").addColumn("timezone").addColumn("modificationDate")
+ .build().withColumnSeparator('\t').withoutQuoteChar();
+
+ MappingIterator<GeonamesRecord> recordIterator = mapper.readerFor(GeonamesRecord.class)
+ .with(schema).readValues(new InputStreamReader(inputStream));
+ Spliterator<GeonamesRecord> recordSpliterator =
+ Spliterators.spliteratorUnknownSize(recordIterator, 0);
+
+ return StreamSupport.stream(recordSpliterator, false);
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesRecord.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesRecord.java
index 75a964f8..7abe1d34 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesRecord.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/GeonamesRecord.java
@@ -13,7 +13,7 @@
package org.apache.baremaps.geocoder.geonames;
/** Structured of a Geonames record. */
-class GeonamesRecord {
+public class GeonamesRecord {
// integer id of record in geonames database
private Integer geonameid;
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/utils/IsoCountriesUtils.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/IsoCountriesUtils.java
similarity index 96%
rename from baremaps-core/src/main/java/org/apache/baremaps/geocoder/utils/IsoCountriesUtils.java
rename to baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/IsoCountriesUtils.java
index 6b269de7..50de4cf0 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/utils/IsoCountriesUtils.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/geonames/IsoCountriesUtils.java
@@ -10,7 +10,7 @@
* the License.
*/
-package org.apache.baremaps.geocoder.utils;
+package org.apache.baremaps.geocoder.geonames;
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenstreetmapDocumentReader.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenstreetmapDocumentReader.java
new file mode 100644
index 00000000..c5775f7f
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenstreetmapDocumentReader.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.geocoder.openstreetmap;
+
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.stream.Stream;
+import org.apache.baremaps.geocoder.DocumentReader;
+import org.apache.baremaps.openstreetmap.model.Element;
+import org.apache.baremaps.openstreetmap.pbf.PbfBlockReader;
+import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.LatLonShape;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.geo.Line;
+import org.locationtech.jts.geom.LineString;
+import org.locationtech.jts.geom.MultiPolygon;
+import org.locationtech.jts.geom.Point;
+import org.locationtech.jts.geom.Polygon;
+
+
+public class OpenstreetmapDocumentReader implements DocumentReader {
+
+ @Override
+ public Stream<Document> read(InputStream inputStream) throws IOException {
+ var blockReader = new PbfBlockReader().geometries(true);
+ var entityReader = new PbfEntityReader(blockReader);
+ return entityReader.read(inputStream).filter(Element.class::isInstance).map(Element.class::cast)
+ .filter(element -> !element.getTags().isEmpty())
+ .filter(element -> element.getGeometry() != null).map(this::toDocument);
+ }
+
+ private Document toDocument(Element element) {
+ var document = new Document();
+
+ document.add(new StoredField("id", element.getId()));
+
+ element.getTags().forEach((key, value) -> {
+ document.add(new StoredField(key, value));
+ });
+
+ var geometry = element.getGeometry();
+ if (geometry instanceof Point point) {
+ document.add(LatLonShape.createDocValueField("geometry", point.getY(), point.getX()));
+ } else if (geometry instanceof LineString lineString) {
+ var coordinates = lineString.getCoordinates();
+ var lats = new double[coordinates.length];
+ var lons = new double[coordinates.length];
+ for (int i = 0; i < coordinates.length; i++) {
+ lats[i] = coordinates[i].y;
+ lons[i] = coordinates[i].x;
+ }
+ document.add(LatLonShape.createDocValueField("geometry", new Line(lats, lons)));
+ } else if (geometry instanceof Polygon polygon) {
+ var coordinates = polygon.getCoordinates();
+ var lats = new double[coordinates.length];
+ var lons = new double[coordinates.length];
+ for (int i = 0; i < coordinates.length; i++) {
+ lats[i] = coordinates[i].y;
+ lons[i] = coordinates[i].x;
+ }
+ document.add(LatLonShape.createDocValueField("geometry",
+ new org.apache.lucene.geo.Polygon(lats, lons)));
+ } else if (geometry instanceof MultiPolygon) {
+
+ }
+
+ return document;
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/response/Result.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenstreetmapRecord.java
similarity index 85%
rename from baremaps-core/src/main/java/org/apache/baremaps/geocoder/response/Result.java
rename to baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenstreetmapRecord.java
index b0e5bb36..a8e405da 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/response/Result.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/openstreetmap/OpenstreetmapRecord.java
@@ -10,7 +10,7 @@
* the License.
*/
-package org.apache.baremaps.geocoder.response;
+package org.apache.baremaps.geocoder.openstreetmap;
-
-public record Result(float score, Data data) {}
+public class OpenstreetmapRecord {
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/request/Request.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/request/Request.java
deleted file mode 100644
index 2059d451..00000000
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/request/Request.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.baremaps.geocoder.request;
-
-
-
-import org.apache.baremaps.geocoder.utils.IsoCountriesUtils;
-
-public class Request {
-
- private final String query;
- private final String countryCode;
-
- private final int limit;
-
- public Request(String query, int limit) {
- this.query = query;
- this.limit = limit;
- this.countryCode = null;
- }
-
- public Request(String query, int limit, String countryCode) {
- this.query = query;
- this.limit = limit;
- if (IsoCountriesUtils.containsCountry(countryCode)) {
- this.countryCode = countryCode;
- } else {
- this.countryCode = null;
- }
- }
-
- public String query() {
- return query;
- }
-
- public String countryCode() {
- return countryCode;
- }
-
- public int limit() {
- return limit;
- }
-}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/response/Data.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/response/Data.java
deleted file mode 100644
index 11f0e3b1..00000000
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/response/Data.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.baremaps.geocoder.response;
-
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"), you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-/**
- * Structured of a Geonames record.
- */
-public record Data(
-
- // name of geographical point (utf8) varchar(200)
- String name,
-
- // name of geographical point in plain ascii characters, varchar(200)
- String asciiname,
-
- // alternatenames, comma separated, ascii names automatically transliterated,
- // convenience attribute from alternatename table, varchar(10000)
- String alternatenames,
-
- // latitude in decimal degrees (wgs84)
- Double latitude,
-
- // longitude in decimal degrees (wgs84)
- Double longitude,
-
- // see http://www.geonames.org/export/codes.html, char(1)
- String featureClass,
-
- // see http://www.geonames.org/export/codes.html, varchar(10)
- String featureCode,
-
- // ISO-3166 2-letter country code, 2 characters
- String countryCode,
-
- // alternate country codes, comma separated, ISO-3166 2-letter country code, 200 characters
- String cc2,
-
- // fipscode (subject to change to iso code), see exceptions below, see file
- // admin1Codes.txt for display names of this code, varchar(20)
- // in switzerland usually canton code (ex: VD)
- String admin1Code,
-
- // code for the second administrative division, a county in the US, see file
- // admin2Codes.txt, varchar(80)
- String admin2Code,
-
- // code for third level administrative division, varchar(20)
- String admin3Code,
-
- // code for fourth level administrative division, varchar(20)
- String admin4Code,
-
- // population
- Long population,
-
- // elevation in meters, integer
- Integer elevation,
-
- // digital elevation model, srtm3 or gtopo30, average elevation of 3''x3'' (ca 90mx90m) or
- // 30''x30'' (ca 900mx900m) area in meters, integer. srtm processed by cgiar/ciat.
- Integer dem,
-
- // the iana timezone id (see file timeZone.txt) varchar(40)
- String timezone,
-
- // date of last modification in yyyy-MM-dd format
- String modificationDate) {
-}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLoc.java b/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLoc.java
index 6b2fba39..1e8de2f5 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLoc.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLoc.java
@@ -22,10 +22,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.baremaps.geocoder.Geocoder;
-import org.apache.baremaps.geocoder.request.Request;
-import org.apache.baremaps.geocoder.response.Response;
-import org.apache.baremaps.geocoder.utils.IsoCountriesUtils;
+import org.apache.baremaps.geocoder.geonames.GeonamesQueryBuilder;
+import org.apache.baremaps.geocoder.geonames.IsoCountriesUtils;
import org.apache.baremaps.iploc.data.InetnumLocation;
import org.apache.baremaps.iploc.data.IpLocStats;
import org.apache.baremaps.iploc.data.Ipv4Range;
@@ -36,6 +34,8 @@ import org.apache.baremaps.iploc.nic.NicAttribute;
import org.apache.baremaps.iploc.nic.NicObject;
import org.apache.baremaps.stream.StreamUtils;
import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.search.TopDocs;
/** Generating pairs of IP address ranges and their locations into an SQLite database */
public class IpLoc {
@@ -43,19 +43,19 @@ public class IpLoc {
private final float SCORE_THRESHOLD = 0.1f;
private final InetnumLocationDao inetnumLocationDao;
- private final Geocoder geocoder;
+ private final SearcherManager searcherManager;
private IpLocStats iplocStats;
/**
* Create a new IpLoc object
*
* @param databaseUrl the jdbc url to the sqlite database
- * @param geocoder the geocoder that will be used to find the locations of the objects
+ * @param searcherManager the geocoder that will be used to find the locations of the objects
*/
- public IpLoc(String databaseUrl, Geocoder geocoder) {
- inetnumLocationDao = new InetnumLocationDaoSqliteImpl(databaseUrl);
- iplocStats = new IpLocStats();
- this.geocoder = geocoder;
+ public IpLoc(String databaseUrl, SearcherManager searcherManager) {
+ this.inetnumLocationDao = new InetnumLocationDaoSqliteImpl(databaseUrl);
+ this.iplocStats = new IpLocStats();
+ this.searcherManager = searcherManager;
}
/**
@@ -92,7 +92,6 @@ public class IpLoc {
}
NicAttribute firstAttribute = nicObject.attributes().get(0);
-
if (!Objects.equals(firstAttribute.name(), "inetnum")) {
return Optional.empty();
}
@@ -115,7 +114,7 @@ public class IpLoc {
// If there is an address we use that address to query the geocoder
if (attributes.containsKey("address")) {
Optional<Location> location =
- findLocation(new Request(attributes.get("address"), 1, attributes.get("country")));
+ findLocation(attributes.get("address"), attributes.get("country"));
if (location.isPresent()) {
iplocStats.incrementInsertedByAddressCount();
return Optional.of(new InetnumLocation(attributes.get("address"), ipRange, location.get(),
@@ -125,7 +124,7 @@ public class IpLoc {
// If there is a description we use that description to query the geocoder
if (attributes.containsKey("descr")) {
Optional<Location> location =
- findLocation(new Request(attributes.get("descr"), 1, attributes.get("country")));
+ findLocation(attributes.get("descr"), attributes.get("country"));
if (location.isPresent()) {
iplocStats.incrementInsertedByDescrCount();
return Optional.of(new InetnumLocation(attributes.get("descr"), ipRange, location.get(),
@@ -135,7 +134,7 @@ public class IpLoc {
// If there is a name we use that name to query the geocoder
if (attributes.containsKey("name")) {
Optional<Location> location =
- findLocation(new Request(attributes.get("name"), 1, attributes.get("country")));
+ findLocation(attributes.get("name"), attributes.get("country"));
if (location.isPresent()) {
iplocStats.incrementInsertedByDescrCount();
return Optional.of(new InetnumLocation(attributes.get("name"), ipRange, location.get(),
@@ -147,8 +146,8 @@ public class IpLoc {
if (attributes.containsKey("country")
&& IsoCountriesUtils.containsCountry(attributes.get("country").toUpperCase())) {
String countryUppercase = attributes.get("country").toUpperCase();
- Optional<Location> location = findLocation(
- new Request(IsoCountriesUtils.getCountry(countryUppercase), 1, countryUppercase));
+ Optional<Location> location =
+ findLocation(IsoCountriesUtils.getCountry(countryUppercase), countryUppercase);
if (location.isPresent()) {
iplocStats.incrementInsertedByCountryCodeCount();
return Optional.of(new InetnumLocation(IsoCountriesUtils.getCountry(countryUppercase),
@@ -159,7 +158,7 @@ public class IpLoc {
// If there is a country that did not follow the ISO format we will query using the country
// has plain text
if (attributes.containsKey("country")) {
- Optional<Location> location = findLocation(new Request(attributes.get("country"), 1));
+ Optional<Location> location = findLocation(attributes.get("country"), "");
if (location.isPresent()) {
iplocStats.incrementInsertedByCountryCount();
return Optional.of(new InetnumLocation(attributes.get("country"), ipRange, location.get(),
@@ -178,22 +177,32 @@ public class IpLoc {
/**
* Use the geocoder to find a latitude/longitude with the given query.
*
- * @param request for the location
+ * @param searchTerms the search terms
+ * @param countryCode the country code filter
* @return an optional of the location
* @throws IOException
* @throws ParseException
*/
- private Optional<Location> findLocation(Request request) throws IOException, ParseException {
- Response response = geocoder.search(request);
- if (response.results().size() > 0) {
- var bestResult = response.results().get(0);
- if (bestResult.score() > SCORE_THRESHOLD) {
- double latitude = bestResult.data().latitude();
- double longitude = bestResult.data().longitude();
- return Optional.of(new Location(latitude, longitude));
- }
+ private Optional<Location> findLocation(String searchTerms, String countryCode)
+ throws IOException, ParseException {
+ var indexSearcher = searcherManager.acquire();
+ var geonamesQuery =
+ new GeonamesQueryBuilder().queryText(searchTerms).countryCode(countryCode).build();
+
+ TopDocs topDocs = indexSearcher.search(geonamesQuery, 1);
+ if (topDocs.scoreDocs.length == 0) {
+ return Optional.empty();
}
- return Optional.empty();
+
+ var scoreDoc = topDocs.scoreDocs[0];
+ if (scoreDoc.score < SCORE_THRESHOLD) {
+ return Optional.empty();
+ }
+
+ var document = indexSearcher.doc(scoreDoc.doc);
+ double latitude = document.getField("latitude").numericValue().doubleValue();
+ double longitude = document.getField("longitude").numericValue().doubleValue();
+ return Optional.of(new Location(latitude, longitude));
}
/**
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/iploc/database/InetnumLocationDaoSqliteImpl.java b/baremaps-core/src/main/java/org/apache/baremaps/iploc/database/InetnumLocationDaoSqliteImpl.java
index 3437eeb0..cb5c7e93 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/iploc/database/InetnumLocationDaoSqliteImpl.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/iploc/database/InetnumLocationDaoSqliteImpl.java
@@ -23,7 +23,6 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
-import java.util.stream.Collectors;
import org.apache.baremaps.iploc.data.InetnumLocation;
import org.apache.baremaps.iploc.data.Ipv4Range;
import org.apache.baremaps.iploc.data.Location;
@@ -161,7 +160,6 @@ public final class InetnumLocationDaoSqliteImpl implements InetnumLocationDao {
stmt.setString(6, inetnumLocation.getNetwork());
stmt.setString(7, inetnumLocation.getCountry());
stmt.executeUpdate();
- logger.info(String.format("Data Added Successfully %s", inetnumLocation));
} catch (SQLException e) {
logger.error("Unable to save data", e);
}
@@ -185,8 +183,6 @@ public final class InetnumLocationDaoSqliteImpl implements InetnumLocationDao {
}
stmt.executeBatch();
connection.commit();
- logger.info(String.format("Batch executed Successfully \n\t%s", inetnumLocations.stream()
- .map(InetnumLocation::toString).collect(Collectors.joining("\n\t"))));
} catch (SQLException e) {
logger.error("Unable to save data", e);
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/OsmReader.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/DataReader.java
similarity index 84%
rename from baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/OsmReader.java
rename to baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/DataReader.java
index f0356531..0914cd8f 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/OsmReader.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/DataReader.java
@@ -14,10 +14,11 @@ package org.apache.baremaps.openstreetmap;
+import java.io.IOException;
import java.io.InputStream;
import java.util.stream.Stream;
-public interface OsmReader<T> {
+public interface DataReader<T> {
- Stream<T> stream(InputStream inputStream);
+ Stream<T> read(InputStream inputStream) throws IOException;
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfBlockReader.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfBlockReader.java
index 7e74c969..abc12512 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfBlockReader.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfBlockReader.java
@@ -18,14 +18,14 @@ import java.io.InputStream;
import java.util.List;
import java.util.stream.Stream;
import org.apache.baremaps.collection.LongDataMap;
-import org.apache.baremaps.openstreetmap.OsmReader;
+import org.apache.baremaps.openstreetmap.DataReader;
import org.apache.baremaps.openstreetmap.function.*;
import org.apache.baremaps.openstreetmap.model.Block;
import org.apache.baremaps.stream.StreamUtils;
import org.locationtech.jts.geom.Coordinate;
/** A utility class for reading an OpenStreetMap pbf file. */
-public class PbfBlockReader implements OsmReader<Block> {
+public class PbfBlockReader implements DataReader<Block> {
private int buffer = Runtime.getRuntime().availableProcessors();
@@ -143,7 +143,7 @@ public class PbfBlockReader implements OsmReader<Block> {
* @param inputStream an osm pbf {@link InputStream}
* @return a stream of blocks
*/
- public Stream<Block> stream(InputStream inputStream) {
+ public Stream<Block> read(InputStream inputStream) {
var blocks = StreamUtils.bufferInSourceOrder(StreamUtils.stream(new BlobIterator(inputStream)),
new BlobToBlockMapper(), Runtime.getRuntime().availableProcessors());
if (geometry) {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfEntityReader.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfEntityReader.java
index fb173fea..25bb3199 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfEntityReader.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfEntityReader.java
@@ -16,14 +16,14 @@ package org.apache.baremaps.openstreetmap.pbf;
import java.io.InputStream;
import java.util.stream.Stream;
-import org.apache.baremaps.openstreetmap.OsmReader;
+import org.apache.baremaps.openstreetmap.DataReader;
import org.apache.baremaps.openstreetmap.model.DataBlock;
import org.apache.baremaps.openstreetmap.model.Entity;
import org.apache.baremaps.openstreetmap.model.HeaderBlock;
import org.apache.baremaps.stream.StreamException;
/** A utility class for flattening the blocks streamed by a {@link PbfBlockReader}. */
-public class PbfEntityReader implements OsmReader<Entity> {
+public class PbfEntityReader implements DataReader<Entity> {
private final PbfBlockReader reader;
@@ -42,8 +42,8 @@ public class PbfEntityReader implements OsmReader<Entity> {
* @param inputStream an osm pbf {@link InputStream}
* @return a stream of blocks
*/
- public Stream<Entity> stream(InputStream inputStream) {
- return reader.stream(inputStream).flatMap(block -> {
+ public Stream<Entity> read(InputStream inputStream) {
+ return reader.read(inputStream).flatMap(block -> {
try {
Stream.Builder<Entity> entities = Stream.builder();
if (block instanceof HeaderBlock headerBlock) {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlChangeReader.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlChangeReader.java
index e378faf0..dd52d34e 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlChangeReader.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlChangeReader.java
@@ -17,11 +17,11 @@ package org.apache.baremaps.openstreetmap.xml;
import java.io.InputStream;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-import org.apache.baremaps.openstreetmap.OsmReader;
+import org.apache.baremaps.openstreetmap.DataReader;
import org.apache.baremaps.openstreetmap.model.Change;
/** A utility class for parsing an OpenStreetMap change file. */
-public class XmlChangeReader implements OsmReader<Change> {
+public class XmlChangeReader implements DataReader<Change> {
/**
* Creates an ordered stream of OSM changes from an XML file.
@@ -29,7 +29,7 @@ public class XmlChangeReader implements OsmReader<Change> {
* @param input
* @return
*/
- public Stream<Change> stream(InputStream input) {
+ public Stream<Change> read(InputStream input) {
return StreamSupport.stream(new XmlChangeSpliterator(input), false);
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlEntityReader.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlEntityReader.java
index 1c3edaf2..5246a7c5 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlEntityReader.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlEntityReader.java
@@ -17,11 +17,11 @@ package org.apache.baremaps.openstreetmap.xml;
import java.io.InputStream;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-import org.apache.baremaps.openstreetmap.OsmReader;
+import org.apache.baremaps.openstreetmap.DataReader;
import org.apache.baremaps.openstreetmap.model.Entity;
/** A utility class for parsing an OpenStreetMap XML file. */
-public class XmlEntityReader implements OsmReader<Entity> {
+public class XmlEntityReader implements DataReader<Entity> {
/**
* Creates an ordered stream of OSM entities from a XML file.
@@ -29,7 +29,7 @@ public class XmlEntityReader implements OsmReader<Entity> {
* @param input
* @return
*/
- public Stream<Entity> stream(InputStream input) {
+ public Stream<Entity> read(InputStream input) {
return StreamSupport.stream(new XmlEntitySpliterator(input), false);
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java
index 95dd0b78..cd285c63 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateGeonamesIndex.java
@@ -12,30 +12,49 @@
package org.apache.baremaps.workflow.tasks;
-import org.apache.baremaps.geocoder.geonames.GeonamesGeocoder;
+import org.apache.baremaps.geocoder.geonames.GeonamesDocumentMapper;
+import org.apache.baremaps.geocoder.geonames.GeonamesReader;
+import org.apache.baremaps.geocoder.geonames.GeonamesConstants;
import org.apache.baremaps.workflow.Task;
import org.apache.baremaps.workflow.WorkflowContext;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.MMapDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.nio.file.Path;
-public record CreateGeonamesIndex(String geonamesDumpPath, String targetGeonamesIndexPath) implements Task {
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+/**
+ * A task that creates a geonames index.
+ */
+public record CreateGeonamesIndex(String dataFile, String indexDirectory) implements Task {
private static final Logger logger = LoggerFactory.getLogger(CreateGeonamesIndex.class);
@Override
public void execute(WorkflowContext context) throws Exception {
- logger.info("Generating geonames from {}", geonamesDumpPath);
- try (GeonamesGeocoder geocoder =
- new GeonamesGeocoder(Path.of(targetGeonamesIndexPath), Path.of(geonamesDumpPath))) {
- if (!geocoder.indexExists()) {
- logger.info("Building the geocoder index");
- geocoder.build();
- }
- } catch(Exception e) {
- logger.error("Error while creating the geocoder index", e);
- return;
+ logger.info("Indexing {}", dataFile);
+
+ var dataPath = Paths.get(dataFile);
+ var indexPath = Paths.get(indexDirectory);
+ var directory = MMapDirectory.open(indexPath);
+ var config = new IndexWriterConfig(GeonamesConstants.ANALYZER);
+
+ try (var indexWriter = new IndexWriter(directory, config);
+ var inputStream = Files.newInputStream(dataPath)) {
+ indexWriter.deleteAll();
+ var documents = new GeonamesReader()
+ .read(inputStream)
+ .map(new GeonamesDocumentMapper());
+ indexWriter.addDocuments((Iterable<Document>) documents::iterator);
+ } catch (IOException exception) {
+ throw new RuntimeException();
}
- logger.info("Finished creating the Geocoder index {}", targetGeonamesIndexPath);
+
+ logger.info("Finished indexing {}", indexDirectory);
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java
index 7b655917..1034810d 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateIplocIndex.java
@@ -12,7 +12,6 @@
package org.apache.baremaps.workflow.tasks;
-import org.apache.baremaps.geocoder.geonames.GeonamesGeocoder;
import org.apache.baremaps.iploc.IpLoc;
import org.apache.baremaps.iploc.data.IpLocStats;
import org.apache.baremaps.iploc.database.SqliteUtils;
@@ -20,6 +19,9 @@ import org.apache.baremaps.iploc.nic.NicParser;
import org.apache.baremaps.stream.StreamException;
import org.apache.baremaps.workflow.Task;
import org.apache.baremaps.workflow.WorkflowContext;
+import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.store.MMapDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,10 +30,14 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.List;
-public record CreateIplocIndex(String geonamesIndexPath, List<String> nicPaths,
- String targetIplocIndexPath) implements Task {
+public record CreateIplocIndex(
+ String geonamesIndexPath,
+ List<String> nicPaths,
+ String targetIplocIndexPath
+) implements Task {
private static final Logger logger = LoggerFactory.getLogger(CreateIplocIndex.class);
@@ -40,24 +46,14 @@ public record CreateIplocIndex(String geonamesIndexPath, List<String> nicPaths,
logger.info("Generating Iploc from {} {}", geonamesIndexPath, nicPaths);
logger.info("Creating the Geocoder");
- GeonamesGeocoder geocoder;
- try {
- geocoder = new GeonamesGeocoder(Path.of(geonamesIndexPath), null);
- if (!geocoder.indexExists()) {
- logger.error("Geocoder index doesn't exist");
- return;
- }
- geocoder.open();
- } catch (Exception e) {
- logger.error("Error while creating the geocoder index", e);
- return;
- }
+ var directory = MMapDirectory.open(Paths.get(geonamesIndexPath));
+ var searcherManager = new SearcherManager(directory, new SearcherFactory());
logger.info("Creating the Iploc database");
String jdbcUrl = String.format("JDBC:sqlite:%s", targetIplocIndexPath);
SqliteUtils.executeResource(jdbcUrl, "iploc_init.sql");
- IpLoc ipLoc = new IpLoc(jdbcUrl, geocoder);
+ IpLoc ipLoc = new IpLoc(jdbcUrl, searcherManager);
logger.info("Generating NIC objects stream");
nicPaths.stream().parallel().forEach(path -> {
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/database/database/BlockImporterTest.java b/baremaps-core/src/test/java/org/apache/baremaps/database/database/BlockImporterTest.java
index cfd69618..d4f3002e 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/database/database/BlockImporterTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/database/database/BlockImporterTest.java
@@ -67,7 +67,7 @@ class BlockImporterTest extends PostgresContainerTest {
try (InputStream inputStream = Files.newInputStream(TestFiles.resolve("simple/data.osm.pbf"))) {
- new PbfBlockReader().stream(inputStream).forEach(blockImporter);
+ new PbfBlockReader().read(inputStream).forEach(blockImporter);
// Check node importation
assertNull(nodeRepository.get(0l));
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/geocoder/GeocoderTest.java b/baremaps-core/src/test/java/org/apache/baremaps/geocoder/GeocoderTest.java
deleted file mode 100644
index 062164ad..00000000
--- a/baremaps-core/src/test/java/org/apache/baremaps/geocoder/GeocoderTest.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.baremaps.geocoder;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Comparator;
-import java.util.stream.Stream;
-import org.apache.baremaps.geocoder.request.Request;
-import org.apache.baremaps.geocoder.response.Response;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.Query;
-import org.junit.jupiter.api.Test;
-
-class GeocoderTest {
-
- private static final String v1 = "a simple text";
- private static final String v2 = "a simple test";
-
- @Test
- public void buildAndSearch() throws IOException, ParseException {
- Path path = Files.createTempDirectory(Paths.get("."), "geocoder_");
- Geocoder geocoder = new Geocoder(path) {
- @Override
- protected Analyzer analyzer() {
- return new StandardAnalyzer();
- }
-
- @Override
- protected Stream<Document> documents() {
- Document d1 = new Document();
- d1.add(new Field("name", v1, TextField.TYPE_STORED));
- Document d2 = new Document();
- d2.add(new Field("name", v2, TextField.TYPE_STORED));
- return Stream.of(d1, d2);
- }
-
- @Override
- protected Query query(Analyzer analyzer, Request request) throws ParseException {
- return new QueryParser("name", analyzer).parse(request.query());
- }
- };
- geocoder.build();
-
- Response r3 = geocoder.search(new Request("simple", 10));
- assertEquals(2, r3.results().size());
-
- Response r1 = geocoder.search(new Request("text", 10));
- assertEquals(1, r1.results().size());
- assertEquals(v1, r1.results().get(0).data().name());
-
- Response r2 = geocoder.search(new Request("test", 10));
- assertEquals(1, r2.results().size());
- assertEquals(v2, r2.results().get(0).data().name());
-
- Files.walk(path).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
- }
-}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/geocoder/geonames/GeonamesGeocoderTest.java b/baremaps-core/src/test/java/org/apache/baremaps/geocoder/geonames/GeonamesGeocoderTest.java
deleted file mode 100644
index a1830dd0..00000000
--- a/baremaps-core/src/test/java/org/apache/baremaps/geocoder/geonames/GeonamesGeocoderTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.baremaps.geocoder.geonames;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Comparator;
-import org.apache.baremaps.geocoder.request.Request;
-import org.apache.baremaps.testing.TestFiles;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.junit.jupiter.api.Test;
-
-class GeonamesGeocoderTest {
-
- @Test
- public void buildAndSearch() throws IOException, URISyntaxException, ParseException {
- var path = Files.createTempDirectory(Paths.get("."), "geocoder_");
- var data = TestFiles.resolve("geonames/LI.txt");
- var geocoder = new GeonamesGeocoder(path, data);
- geocoder.build();
-
- var response = geocoder.search(new Request("Bim Alta Schloss", 1));
- assertEquals(1, response.results().size());
- assertEquals("Bim Alta Schloss", response.results().get(0).data().name());
-
- Files.walk(path).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
- }
-
- @Test
- public void buildAndSearchWithTheRightCountryCode()
- throws IOException, URISyntaxException, ParseException {
- var path = Files.createTempDirectory(Paths.get("."), "geocoder_");
- var data = TestFiles.resolve("geonames/LI.txt");
- var geocoder = new GeonamesGeocoder(path, data);
- geocoder.build();
-
- var response = geocoder.search(new Request("Bim Alta Schloss", 10, "LI"));
- assertEquals(10, response.results().size());
- assertEquals("Bim Alta Schloss", response.results().get(0).data().name());
-
- Files.walk(path).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
- }
-
- @Test
- public void buildAndSearchWithTheWrongCountryCode()
- throws IOException, URISyntaxException, ParseException {
- var path = Files.createTempDirectory(Paths.get("."), "geocoder_");
- var data = TestFiles.resolve("geonames/LI.txt");
- var geocoder = new GeonamesGeocoder(path, data);
- geocoder.build();
-
- var response = geocoder.search(new Request("Bim Alta Schloss", 10, "CH"));
- assertEquals(0, response.results().size());
-
- Files.walk(path).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
- }
-}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/iploc/IpLocTest.java b/baremaps-core/src/test/java/org/apache/baremaps/iploc/IpLocTest.java
index a38fe88c..489e74f1 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/iploc/IpLocTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/iploc/IpLocTest.java
@@ -15,7 +15,6 @@ package org.apache.baremaps.iploc;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
-import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -23,8 +22,6 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import org.apache.baremaps.collection.utils.FileUtils;
-import org.apache.baremaps.geocoder.Geocoder;
-import org.apache.baremaps.geocoder.geonames.GeonamesGeocoder;
import org.apache.baremaps.iploc.data.InetnumLocation;
import org.apache.baremaps.iploc.data.Ipv4;
import org.apache.baremaps.iploc.data.Ipv4Range;
@@ -35,6 +32,11 @@ import org.apache.baremaps.iploc.database.SqliteUtils;
import org.apache.baremaps.iploc.nic.NicData;
import org.apache.baremaps.iploc.nic.NicObject;
import org.apache.baremaps.testing.TestFiles;
+import org.apache.baremaps.workflow.WorkflowContext;
+import org.apache.baremaps.workflow.tasks.CreateGeonamesIndex;
+import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.store.MMapDirectory;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -53,20 +55,23 @@ class IpLocTest {
private static String databaseUrl;
@BeforeAll
- public static void beforeAll() throws IOException, URISyntaxException {
+ public static void beforeAll() throws Exception {
// Load the NIC sample objects
nicObjects = NicData.sample("ripe/simple_nic_sample.txt");
// Init the geocoderservice
directory = Files.createTempDirectory(Paths.get("."), "geocoder_");
+ // Create the geonames index
var data = TestFiles.resolve("geonames/geocoder_sample.txt");
- Geocoder geocoder = new GeonamesGeocoder(directory, data);
- geocoder.build();
+ var task = new CreateGeonamesIndex(data.toString(), directory.toString());
+ task.execute(new WorkflowContext());
// Create the IPLoc service
databaseUrl = String.format("JDBC:sqlite:%s", directory.resolve("test.db"));
- ipLoc = new IpLoc(databaseUrl, geocoder);
+ var dir = MMapDirectory.open(directory);
+ var searcherManager = new SearcherManager(dir, new SearcherFactory());
+ ipLoc = new IpLoc(databaseUrl, searcherManager);
// Accessor for the database
inetnumLocationDao = new InetnumLocationDaoSqliteImpl(databaseUrl);
@@ -78,7 +83,7 @@ class IpLocTest {
}
@BeforeEach
- public void beforeEach() throws IOException, URISyntaxException, SQLException {
+ public void beforeEach() throws IOException, SQLException {
SqliteUtils.executeResource(databaseUrl, "iploc_init.sql");
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/OpenStreetMapTest.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/OpenStreetMapTest.java
index 441b29a1..bfa3ecc3 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/OpenStreetMapTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/OpenStreetMapTest.java
@@ -52,50 +52,49 @@ class OpenStreetMapTest {
@Test
void dataOsmXml() throws IOException {
try (InputStream input = Files.newInputStream(DATA_OSM_XML)) {
- assertEquals(12, new XmlEntityReader().stream(input).collect(Collectors.toList()).size());
+ assertEquals(12, new XmlEntityReader().read(input).collect(Collectors.toList()).size());
}
}
@Test
void dataOsmXmlNodes() throws IOException {
try (InputStream input = Files.newInputStream(DATA_OSM_XML)) {
- assertEquals(6, new XmlEntityReader().stream(input).filter(e -> e instanceof Node).count());
+ assertEquals(6, new XmlEntityReader().read(input).filter(e -> e instanceof Node).count());
}
}
@Test
void dataOsmXmlWays() throws IOException {
try (InputStream input = Files.newInputStream(DATA_OSM_XML)) {
- assertEquals(3, new XmlEntityReader().stream(input).filter(e -> e instanceof Way).count());
+ assertEquals(3, new XmlEntityReader().read(input).filter(e -> e instanceof Way).count());
}
}
@Test
void dataOsmXmlRelations() throws IOException {
try (InputStream input = Files.newInputStream(DATA_OSM_XML)) {
- assertEquals(1,
- new XmlEntityReader().stream(input).filter(e -> e instanceof Relation).count());
+ assertEquals(1, new XmlEntityReader().read(input).filter(e -> e instanceof Relation).count());
}
}
@Test
void dataOscXml() throws IOException {
try (InputStream input = Files.newInputStream(DATA_OSC_XML)) {
- assertEquals(7, new XmlChangeReader().stream(input).collect(Collectors.toList()).size());
+ assertEquals(7, new XmlChangeReader().read(input).collect(Collectors.toList()).size());
}
}
@Test
void dataOsmPbf() throws IOException {
try (InputStream input = Files.newInputStream(DATA_OSM_PBF)) {
- assertEquals(72002, new PbfEntityReader(new PbfBlockReader()).stream(input).count());
+ assertEquals(72002, new PbfEntityReader(new PbfBlockReader()).read(input).count());
}
}
@Test
void denseNodesOsmPbf() throws IOException {
try (InputStream input = Files.newInputStream(DENSE_NODES_OSM_PBF)) {
- assertEquals(8000, new PbfEntityReader(new PbfBlockReader()).stream(input)
+ assertEquals(8000, new PbfEntityReader(new PbfBlockReader()).read(input)
.filter(e -> e instanceof Node).count());
}
}
@@ -103,7 +102,7 @@ class OpenStreetMapTest {
@Test
void waysOsmPbf() throws IOException {
try (InputStream input = Files.newInputStream(WAYS_OSM_PBF)) {
- assertEquals(8000, new PbfEntityReader(new PbfBlockReader()).stream(input)
+ assertEquals(8000, new PbfEntityReader(new PbfBlockReader()).read(input)
.filter(e -> e instanceof Way).count());
}
}
@@ -111,7 +110,7 @@ class OpenStreetMapTest {
@Test
void relationsOsmPbf() throws IOException {
try (InputStream input = Files.newInputStream(RELATIONS_OSM_PBF)) {
- assertEquals(8000, new PbfEntityReader(new PbfBlockReader()).stream(input)
+ assertEquals(8000, new PbfEntityReader(new PbfBlockReader()).read(input)
.filter(e -> e instanceof Relation).count());
}
}
@@ -128,7 +127,7 @@ class OpenStreetMapTest {
@Test
void monacoOsmPbf() throws IOException, URISyntaxException {
try (InputStream inputStream = Files.newInputStream(MONACO_OSM_PBF)) {
- Stream<Entity> stream = new PbfEntityReader(new PbfBlockReader()).stream(inputStream);
+ Stream<Entity> stream = new PbfEntityReader(new PbfBlockReader()).read(inputStream);
process(stream, 1, 1, 25002, 4018, 243);
}
}
@@ -137,7 +136,7 @@ class OpenStreetMapTest {
void monacoOsmBz2() throws IOException, URISyntaxException {
try (InputStream inputStream =
new BZip2CompressorInputStream(Files.newInputStream(MONACO_OSM_BZ2))) {
- Stream<Entity> stream = new XmlEntityReader().stream(inputStream);
+ Stream<Entity> stream = new XmlEntityReader().read(inputStream);
process(stream, 1, 1, 24951, 4015, 243);
}
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/RelationGeometryBuilderTest.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/RelationGeometryBuilderTest.java
index 5512d1cd..fb9fca78 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/RelationGeometryBuilderTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/RelationGeometryBuilderTest.java
@@ -36,7 +36,7 @@ class RelationGeometryBuilderTest {
Geometry handleRelation(String file) throws IOException {
InputStream input = new GZIPInputStream(this.getClass().getResourceAsStream(file));
- List<Entity> entities = new XmlEntityReader().stream(input).toList();
+ List<Entity> entities = new XmlEntityReader().read(input).toList();
LongDataMap<Coordinate> coordinateMap = new MockLongDataMap<>(
entities.stream().filter(e -> e instanceof Node).map(e -> (Node) e).collect(
Collectors.toMap(n -> n.getId(), n -> new Coordinate(n.getLon(), n.getLat()))));
diff --git a/baremaps-server/src/main/java/org/apache/baremaps/server/GeocoderResources.java b/baremaps-server/src/main/java/org/apache/baremaps/server/GeocoderResources.java
index 38376945..6da213c0 100644
--- a/baremaps-server/src/main/java/org/apache/baremaps/server/GeocoderResources.java
+++ b/baremaps-server/src/main/java/org/apache/baremaps/server/GeocoderResources.java
@@ -18,45 +18,77 @@ import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import javax.inject.Inject;
import javax.inject.Singleton;
-import javax.ws.rs.GET;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.*;
import javax.ws.rs.core.Response;
-import org.apache.baremaps.geocoder.Geocoder;
-import org.apache.baremaps.geocoder.request.Request;
-import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.baremaps.geocoder.geonames.GeonamesQueryBuilder;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.SearcherManager;
+
@Singleton
@javax.ws.rs.Path("/")
public class GeocoderResources {
- private final Geocoder geocoder;
+ record GeocoderResponse(List<GeocoderResult> results) {}
+
+
+ record GeocoderResult(float score, Map<String, Object> data) {}
+
+
+ private final SearcherManager searcherManager;
@Inject
- public GeocoderResources(Geocoder geocoder) {
- this.geocoder = geocoder;
+ public GeocoderResources(SearcherManager searcherManager) {
+ this.searcherManager = searcherManager;
}
@GET
@javax.ws.rs.Path("/api/geocoder")
- public Response getIpToLocation(@QueryParam("address") String address) {
- if (address == null) {
+ public Response getIpToLocation(@QueryParam("queryText") String queryText,
+ @QueryParam("countryCode") @DefaultValue("") String countryCode,
+ @QueryParam("limit") @DefaultValue("10") int limit) throws IOException {
+ if (queryText == null) {
throw new WebApplicationException(Response.status(Response.Status.BAD_REQUEST)
- .entity("address parameter is mandatory").build());
+ .entity("The queryText parameter is mandatory").build());
}
-
+ var query = new GeonamesQueryBuilder().queryText(queryText).countryCode(countryCode).build();
+ var searcher = searcherManager.acquire();
try {
- var request = new Request(address, 20);
- var response = geocoder.search(request);
+ var result = searcher.search(query, limit);
+ var results =
+ Arrays.stream(result.scoreDocs).map(scoreDoc -> asResult(searcher, scoreDoc)).toList();
return Response.status(200).header(ACCESS_CONTROL_ALLOW_ORIGIN, "*")
- .header(CONTENT_TYPE, APPLICATION_JSON).entity(response).build();
+ .header(CONTENT_TYPE, APPLICATION_JSON).entity(new GeocoderResponse(results)).build();
} catch (IllegalArgumentException e) {
return Response.status(400).entity(e.getMessage()).build();
- } catch (IOException | ParseException e) {
+ } catch (IOException e) {
return Response.status(500).entity(e.getMessage()).build();
+ } finally {
+ searcherManager.release(searcher);
+ }
+ }
+
+ private GeocoderResult asResult(IndexSearcher indexSearcher, ScoreDoc scoreDoc) {
+ try {
+ var document = indexSearcher.doc(scoreDoc.doc);
+ var data = new HashMap<String, Object>();
+ for (var field : document.getFields()) {
+ if (field.numericValue() != null) {
+ data.put(field.name(), field.numericValue());
+ } else if (field.stringValue() != null) {
+ data.put(field.name(), field.stringValue());
+ }
+ }
+ return new GeocoderResult(scoreDoc.score, data);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
}
diff --git a/baremaps-server/src/main/resources/geocoder/index.html b/baremaps-server/src/main/resources/geocoder/index.html
index 5058f08c..2662e311 100644
--- a/baremaps-server/src/main/resources/geocoder/index.html
+++ b/baremaps-server/src/main/resources/geocoder/index.html
@@ -21,116 +21,62 @@
</head>
<body>
<!-- A form to submit the IP address -->
-<form onsubmit="searchByAddress(event)">
- <label for="address">Address:</label>
- <input type="text" id="address" name="address" value=""/>
+<form id="searchForm">
+ <input type="text" id="queryText" name="queryText" placeholder="Query text" value=""/>
+ <input type="text" id="countryCode" name="countryCode" placeholder="Country code" value=""/>
<input type="submit" value="Submit"/>
</form>
<!-- The table of retrieved locations -->
<table id="results"></table>
+
<script>
- function searchByAddress(event) {
- event.preventDefault();
- // Get the IP address from the form
- var address = document.getElementById('address').value;
+ // Add a listener to submit the search form
+ const searchForm = document.getElementById('searchForm');
+ searchForm.addEventListener('submit', search);
- // Make a query to retrieve the location from the address
- const request = new XMLHttpRequest();
- request.open('GET', `http://localhost:9000/api/geocoder?address=${address}`, true);
+ // Get the table of results
+ const table = document.getElementById('results');
- // Set request Accept header to application/json
- request.setRequestHeader('Accept', 'application/json');
+ function search(event) {
+ event.preventDefault();
- // Display the request result in the map
- request.onload = function () {
- if (request.status >= 200 && request.status < 400) {
- // Success!
- const response = JSON.parse(request.responseText);
+ // Build the query string
+ const formData = new FormData(searchForm);
+ const queryString = new URLSearchParams(formData).toString()
- // Write the result address in a href
- // Fill the table of geo locations from the resulting geoLocations
- // Geo locations contain an address, an ipv4Range, a location, a network and a country
- const table = document.getElementById('results');
+ // Send the search request
+ fetch(`${window.location.origin}/api/geocoder?${queryString}`)
+ .then(response => response.json())
+ .then(response => {
+ // Extract the headers from the first row
+ const headers = response.results && response.results.length > 0
+ ? Object.keys(response.results[0].data).sort()
+ : [];
+
+ // Clear the results table
table.innerHTML = '';
- // Insert header row
- table.insertRow().innerHTML =
- `<th>#</th>
- <th>Score</th>
- <th>Url</th>
- <th>Name</th>
- <th>Ascii name</th>
- <th>Alternate names</th>
- <th>Latitude</th>
- <th>Longitude</th>
- <th>Feature Class</th>
- <th>feature Code</th>
- <th>Country Code</th>
- <th>Country Code 2</th>
- <th>Admin1Code</th>
- <th>Admin2Code</th>
- <th>Admin3Code</th>
- <th>Admin4Code</th>
- <th>Population</th>
- <th>Elevation</th>
- <th>Dem</th>
- <th>Timezone</th>
- <th>Modification Date</th>`;
- for (let i = 0; i < response.results.length; i++) {
- const result = response.results[i];
- const row = table.insertRow(i + 1);
- let pos = 0;
- let cell = row.insertCell(pos++);
- cell.innerHTML = i + 1;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.score;
- cell = row.insertCell(pos++);
- cell.innerHTML = `<a href="http://www.openstreetmap.org/?mlat=${result.data.latitude}&mlon=${result.data.longitude}&zoom=15" target="_blank">OSM</a>`;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.name;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.asciiname;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.alternatenames;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.latitude;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.longitude;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.featureClass;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.featureCode;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.countryCode;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.cc2;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.admin1Code;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.admin2Code;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.admin3Code;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.admin4Code;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.population;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.elevation;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.dem;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.timezone;
- cell = row.insertCell(pos++);
- cell.innerHTML = result.data.modificationDate;
- }
- } else {
- // We reached our target server, but it returned an error
- console.log('Error');
- }
- };
- request.send();
+ // Insert the headers in the table
+ const headerRow = table.insertRow();
+ headerRow.innerHTML = `<th>#</th><th>score</th>${headers.map(header => `<th>${header}</th>`).join('')}`;
+
+ // Insert the results in the table
+ response.results.forEach((result, index) => {
+ const row = table.insertRow();
+ row.insertCell().innerText = index;
+ row.insertCell().innerText = result.score;
+ headers.forEach(header => {
+ if (result.data[header] !== undefined) {
+ row.insertCell().innerText = result.data[header];
+ } else {
+ row.insertCell().innerText = '';
+ }
+ });
+ });
+ })
+ .catch(error => console.error(error));
}
</script>
</body>
-</html>
\ No newline at end of file
+</html>
diff --git a/examples/geocoding/workflow.js b/examples/geocoding/workflow.js
index 6bd3f837..972d4a24 100644
--- a/examples/geocoding/workflow.js
+++ b/examples/geocoding/workflow.js
@@ -8,7 +8,7 @@ const FetchAndUnzipGeonames = {id: "fetch-geonames-allcountries", needs: [], tas
// Create the Geocoder index
const createGeonamesIndex = {id: "geocoder-index", needs: [FetchAndUnzipGeonames.id], tasks: [
- {type: "CreateGeonamesIndex", geonamesDumpPath: "archives/allCountries.txt", targetGeonamesIndexPath: "geocoder-index"}
+ {type: "CreateGeonamesIndex", dataFile: "archives/allCountries.txt", indexDirectory: "geocoder-index"}
]};
-export default {"steps": [FetchAndUnzipGeonames, createGeonamesIndex]};
\ No newline at end of file
+export default {"steps": [FetchAndUnzipGeonames, createGeonamesIndex]};
diff --git a/examples/ip-to-location/workflow.js b/examples/ip-to-location/workflow.js
index cd0b58ff..2c0acecb 100644
--- a/examples/ip-to-location/workflow.js
+++ b/examples/ip-to-location/workflow.js
@@ -27,26 +27,67 @@ const geonamesUrl =
"https://download.geonames.org/export/dump/allCountries.zip";
// Iterate over nic urls to create a list of downloads and ungzip
-const fetchAndUnzipNic = nics.map((nic,index) =>
- ({id: `fetch-nic-${index}`, needs: [], tasks: [
- {type: "DownloadUrl", url: nic.url, path: `downloads/${nic.filename}.gz`},
- {type: "UngzipFile", file: `downloads/${nic.filename}.gz`, directory: "archives"}
- ]}));
+const fetchAndUnzipNic = nics.map((nic, index) => ({
+ id: `fetch-nic-${index}`,
+ needs: [],
+ tasks: [
+ {
+ type: "DownloadUrl",
+ url: nic.url,
+ path: `downloads/${nic.filename}.gz`
+ },
+ {
+ type: "UngzipFile",
+ file: `downloads/${nic.filename}.gz`,
+ directory: "archives"
+ }
+ ]
+}));
// Fetch and unzip Geonames
-const FetchAndUnzipGeonames = {id: "fetch-geonames-allcountries", needs: [], tasks: [
- {type: "DownloadUrl", url: geonamesUrl, path: "downloads/geonames-allcountries.zip", force: true},
- {type: "UnzipFile", file: "downloads/geonames-allcountries.zip", directory: "archives"}
-]};
+const FetchAndUnzipGeonames = {
+ id: "fetch-geonames-allcountries",
+ needs: [],
+ tasks: [
+ {
+ type: "DownloadUrl",
+ url: geonamesUrl,
+ path: "downloads/geonames-allcountries.zip",
+ force: true
+ },
+ {
+ type: "UnzipFile",
+ file: "downloads/geonames-allcountries.zip",
+ directory: "archives"
+ }
+ ]
+};
// Create the Geocoder index
-const createGeonamesIndex = {id: "geocoder-index", needs: [FetchAndUnzipGeonames.id], tasks: [
- {type: "CreateGeonamesIndex", geonamesDumpPath: "archives/allCountries.txt", targetGeonamesIndexPath: "geocoder-index"}
-]};
+const createGeonamesIndex = {
+ id: "geocoder-index",
+ needs: [FetchAndUnzipGeonames.id],
+ tasks: [
+ {
+ type: "CreateGeonamesIndex",
+ dataFile: "archives/allCountries.txt",
+ indexDirectory: "geocoder-index"
+ }
+ ]
+};
// Create the iploc database
-const createIplocIndex = {id: "iploc-index", needs: fetchAndUnzipNic.map(e => e.id).concat([createGeonamesIndex.id]), tasks: [
- {type: "CreateIplocIndex", geonamesIndexPath: "geocoder-index", nicPaths: nics.map(nic => `archives/${nic.filename}`), targetIplocIndexPath: "iploc.db"}
-]};
+const createIplocIndex = {
+ id: "iploc-index",
+ needs: fetchAndUnzipNic.map(e => e.id).concat([createGeonamesIndex.id]),
+ tasks: [
+ {
+ type: "CreateIplocIndex",
+ geonamesIndexPath: "geocoder-index",
+ nicPaths: nics.map(nic => `archives/${nic.filename}`),
+ targetIplocIndexPath: "iploc.db"
+ }
+ ]
+};
-export default {"steps": fetchAndUnzipNic.concat([FetchAndUnzipGeonames, createGeonamesIndex, createIplocIndex])};
\ No newline at end of file
+export default {"steps": fetchAndUnzipNic.concat([FetchAndUnzipGeonames, createGeonamesIndex, createIplocIndex])};
diff --git a/pom.xml b/pom.xml
index ca1e9894..372487cf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -102,8 +102,8 @@
<version.lib.swagger-parser>2.0.24</version.lib.swagger-parser>
<version.lib.testcontainers>1.17.3</version.lib.testcontainers>
<version.lib.validation>2.0.2</version.lib.validation>
- <version.lucene>8.10.1</version.lucene>
- <version.lucene-spatial>8.4.1</version.lucene-spatial>
+ <version.lucene>9.4.2</version.lucene>
+ <version.lucene-spatial>9.4.2</version.lucene-spatial>
<version.plugin.jacoco-maven-plugin>0.8.8</version.plugin.jacoco-maven-plugin>
<version.plugin.jib-maven-plugin>3.0.0</version.plugin.jib-maven-plugin>
<version.plugin.maven-compiler-plugin>3.10.1</version.plugin.maven-compiler-plugin>
@@ -346,12 +346,12 @@
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-replicator</artifactId>
- <version>${version.lucene-spatial}</version>
+ <version>${version.lucene}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
- <artifactId>lucene-spatial</artifactId>
- <version>${version.lucene-spatial}</version>
+ <artifactId>lucene-spatial-extras</artifactId>
+ <version>${version.lucene}</version>
</dependency>
<dependency>
<groupId>org.apache.sis.core</groupId>