You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2023/11/08 17:31:24 UTC
(incubator-baremaps) 01/01: Add wikidata processor
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch wikidata
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 7352fb55fb1dc02d45f3e6741378608e14bde144
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Wed Nov 8 18:31:12 2023 +0100
Add wikidata processor
---
.../java/org/apache/baremaps/workflow/Task.java | 3 +-
.../baremaps/workflow/tasks/ImportWikidata.java | 59 ++++++++++++++++++++++
examples/wikidata/workflow.json | 14 +++++
pom.xml | 26 ++++++++++
4 files changed, 101 insertions(+), 1 deletion(-)
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
index 06fdad9e..20cf880e 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
@@ -51,7 +51,8 @@ import org.apache.baremaps.workflow.tasks.*;
@JsonSubTypes.Type(value = CreateIplocIndex.class, name = "CreateIplocIndex"),
@JsonSubTypes.Type(value = ImportDaylightTranslations.class,
name = "ImportDaylightTranslations"),
- @JsonSubTypes.Type(value = ImportDaylightFeatures.class, name = "ImportDaylightFeatures")
+ @JsonSubTypes.Type(value = ImportDaylightFeatures.class, name = "ImportDaylightFeatures"),
+ @JsonSubTypes.Type(value = ImportWikidata.class, name = "ImportWikidata")
})
public interface Task {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportWikidata.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportWikidata.java
new file mode 100644
index 00000000..4b218b3e
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportWikidata.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import org.apache.baremaps.workflow.Task;
+import org.apache.baremaps.workflow.WorkflowContext;
+import org.wikidata.wdtk.datamodel.interfaces.*;
+import org.wikidata.wdtk.dumpfiles.DumpProcessingController;
+
+public record ImportWikidata(Object database) implements Task {
+
+ @Override
+ public void execute(WorkflowContext context) throws Exception {
+ DumpProcessingController controller = new DumpProcessingController("wikidatawiki");
+ controller.setOfflineMode(false);
+ controller.registerEntityDocumentProcessor(new CustomProcessor(), null, true);
+ controller.processMostRecentJsonDump();
+ }
+
+ public class CustomProcessor implements EntityDocumentDumpProcessor {
+
+ @Override
+ public void processItemDocument(ItemDocument itemDocument) {}
+
+ @Override
+ public void processPropertyDocument(PropertyDocument propertyDocument) {}
+
+ @Override
+ public void processLexemeDocument(LexemeDocument lexemeDocument) {}
+
+ @Override
+ public void processMediaInfoDocument(MediaInfoDocument mediaInfoDocument) {}
+
+ @Override
+ public void open() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+ }
+}
diff --git a/examples/wikidata/workflow.json b/examples/wikidata/workflow.json
new file mode 100644
index 00000000..77ed204c
--- /dev/null
+++ b/examples/wikidata/workflow.json
@@ -0,0 +1,14 @@
+{
+ "steps": [
+ {
+ "id": "natural_earth_vector",
+ "needs": [],
+ "tasks": [
+ {
+ "type": "ImportWikidata",
+ "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps"
+ }
+ ]
+ }
+ ]
+}
diff --git a/pom.xml b/pom.xml
index 7bb72aff..d87a6a75 100644
--- a/pom.xml
+++ b/pom.xml
@@ -110,6 +110,7 @@ limitations under the License.
<version.lib.slf4j>2.0.9</version.lib.slf4j>
<version.lib.sqlite>3.43.2.1</version.lib.sqlite>
<version.lib.testcontainers>1.19.1</version.lib.testcontainers>
+ <version.lib.wdtk>0.14.6</version.lib.wdtk>
<version.lucene>9.4.2</version.lucene>
<version.lucene-spatial>9.4.2</version.lucene-spatial>
<version.plugin.jacoco-maven-plugin>0.8.11</version.plugin.jacoco-maven-plugin>
@@ -530,6 +531,31 @@ limitations under the License.
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.wikidata.wdtk</groupId>
+ <artifactId>wdtk-datamodel</artifactId>
+ <version>${version.lib.wdtk}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.wikidata.wdtk</groupId>
+ <artifactId>wdtk-dumpfiles</artifactId>
+ <version>${version.lib.wdtk}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.wikidata.wdtk</groupId>
+ <artifactId>wdtk-rdf</artifactId>
+ <version>${version.lib.wdtk}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.wikidata.wdtk</groupId>
+ <artifactId>wdtk-util</artifactId>
+ <version>${version.lib.wdtk}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.wikidata.wdtk</groupId>
+ <artifactId>wdtk-wikibaseapi</artifactId>
+ <version>${version.lib.wdtk}</version>
+ </dependency>
<dependency>
<groupId>org.wololo</groupId>
<artifactId>flatgeobuf</artifactId>