You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2020/05/02 13:03:15 UTC
[incubator-streampipes-extensions] 02/02: [STREAMPIPES-116]: Modify
reverse geocoder
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch rel/0.66.0
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes-extensions.git
commit f1103d574929b1585d10d0ca5727966404e2331d
Author: Dominik Riemer <ri...@fzi.de>
AuthorDate: Sat May 2 15:02:58 2020 +0200
[STREAMPIPES-116]: Modify reverse geocoder
---
LICENSE | 34 ++++++++++++++++++++-
pom.xml | 1 +
.../processor/revgeocoder/ReverseGeocoding.java | 28 +++++++++++------
.../jvm/processor/revgeocoder/geocode/GeoName.java | 18 -----------
.../revgeocoder/geocode/ReverseGeoCode.java | 17 -----------
.../revgeocoder/geocode/kdtree/KDNode.java | 18 -----------
.../geocode/kdtree/KDNodeComparator.java | 18 -----------
.../revgeocoder/geocode/kdtree/KDTree.java | 18 -----------
.../src/main/resources/cities1000.zip | Bin 7841510 -> 0 bytes
.../documentation.md | 2 ++
10 files changed, 55 insertions(+), 99 deletions(-)
diff --git a/LICENSE b/LICENSE
index ac3c823..331fb0b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -205,4 +205,36 @@
This project includes some sources that are also licensed under the Apache
License Version 2.0:
-All files in streampipes-processors-transformation-flink/src/main/java/com.kohlschutter.boilerpipe (https://github.com/kohlschutter/boilerpipe)
\ No newline at end of file
+All files in streampipes-processors-transformation-flink/src/main/java/src/main/java/com/kohlschutter/boilerpipe (https://github.com/kohlschutter/boilerpipe)
+
+================================================================
+
+This project includes some sources that are licensed under the MIT License:
+
+All files in streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor
+/revgeocoder/geocode (https://github.com/AReallyGoodName/OfflineReverseGeocode)
+
+The MIT License (MIT)
+[OSI Approved License]
+The MIT License (MIT)
+
+Copyright (c) 2014 Daniel Glasson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
diff --git a/pom.xml b/pom.xml
index 26e63d6..efde269 100644
--- a/pom.xml
+++ b/pom.xml
@@ -590,6 +590,7 @@
<!-- External files -->
<exclude>streampipes-processors-transformation-flink/src/main/java/com/kohlschutter/boilerpipe/**</exclude>
+ <exclude>streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/**</exclude>
</excludes>
</configuration>
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/ReverseGeocoding.java b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/ReverseGeocoding.java
index 59d42bd..14cb8d4 100644
--- a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/ReverseGeocoding.java
+++ b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/ReverseGeocoding.java
@@ -17,6 +17,7 @@
*/
package org.apache.streampipes.processors.geo.jvm.processor.revgeocoder;
+import org.apache.http.client.fluent.Request;
import org.apache.streampipes.commons.exceptions.SpRuntimeException;
import org.apache.streampipes.model.runtime.Event;
import org.apache.streampipes.processors.geo.jvm.processor.revgeocoder.geocode.GeoName;
@@ -25,12 +26,16 @@ import org.apache.streampipes.wrapper.context.EventProcessorRuntimeContext;
import org.apache.streampipes.wrapper.routing.SpOutputCollector;
import org.apache.streampipes.wrapper.runtime.EventProcessor;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.ZipInputStream;
public class ReverseGeocoding implements EventProcessor<ReverseGeocodingParameters> {
+ private static final String CITIES_DATASET_URL = "http://download.geonames" +
+ ".org/export/dump/cities1000.zip";
+
private String latitudeField;
private String longitudeField;
@@ -41,16 +46,15 @@ public class ReverseGeocoding implements EventProcessor<ReverseGeocodingParamete
this.latitudeField = parameters.getLatitudeField();
this.longitudeField = parameters.getLongitudeField();
- InputStream stream = getClass()
- .getClassLoader().getResourceAsStream("cities1000.zip");
- if (stream != null) {
- ZipInputStream zipInputStream = null;
- zipInputStream = new ZipInputStream(stream);
- try {
- this.reverseGeoCode = new ReverseGeoCode(zipInputStream, false);
- } catch (IOException e) {
- throw new SpRuntimeException("Could not load cities file...");
+ try {
+ InputStream stream = downloadCitiesDataSet();
+ if (stream != null) {
+ ZipInputStream zipInputStream = null;
+ zipInputStream = new ZipInputStream(stream);
+ this.reverseGeoCode = new ReverseGeoCode(zipInputStream, false);
}
+ } catch (IOException e) {
+ throw new SpRuntimeException("Could not download cities file...");
}
}
@@ -69,4 +73,10 @@ public class ReverseGeocoding implements EventProcessor<ReverseGeocodingParamete
public void onDetach() throws SpRuntimeException {
}
+
+ private InputStream downloadCitiesDataSet() throws IOException {
+ byte[] citiesDataset = Request.Get(CITIES_DATASET_URL).execute().returnContent().asBytes();
+ return new ByteArrayInputStream(citiesDataset);
+ }
+
}
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/GeoName.java b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/GeoName.java
index ed17127..170d424 100644
--- a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/GeoName.java
+++ b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/GeoName.java
@@ -1,22 +1,4 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/*
The MIT License (MIT)
[OSI Approved License]
The MIT License (MIT)
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/ReverseGeoCode.java b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/ReverseGeoCode.java
index 7329c50..a8bde42 100644
--- a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/ReverseGeoCode.java
+++ b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/ReverseGeoCode.java
@@ -1,20 +1,3 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
/*
The MIT License (MIT)
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDNode.java b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDNode.java
index cc833ce..f7f0d49 100644
--- a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDNode.java
+++ b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDNode.java
@@ -1,22 +1,4 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/*
The MIT License (MIT)
[OSI Approved License]
The MIT License (MIT)
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDNodeComparator.java b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDNodeComparator.java
index b186e60..dfe5cae 100644
--- a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDNodeComparator.java
+++ b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDNodeComparator.java
@@ -1,22 +1,4 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/*
The MIT License (MIT)
[OSI Approved License]
The MIT License (MIT)
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDTree.java b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDTree.java
index b989a50..7d9b063 100644
--- a/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDTree.java
+++ b/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/kdtree/KDTree.java
@@ -1,22 +1,4 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/*
The MIT License (MIT)
[OSI Approved License]
The MIT License (MIT)
diff --git a/streampipes-processors-geo-jvm/src/main/resources/cities1000.zip b/streampipes-processors-geo-jvm/src/main/resources/cities1000.zip
deleted file mode 100644
index 3150fea..0000000
Binary files a/streampipes-processors-geo-jvm/src/main/resources/cities1000.zip and /dev/null differ
diff --git a/streampipes-processors-geo-jvm/src/main/resources/org.apache.streampipes.processor.geo.jvm.reversegeocoding/documentation.md b/streampipes-processors-geo-jvm/src/main/resources/org.apache.streampipes.processor.geo.jvm.reversegeocoding/documentation.md
index b1211f7..cb0f348 100644
--- a/streampipes-processors-geo-jvm/src/main/resources/org.apache.streampipes.processor.geo.jvm.reversegeocoding/documentation.md
+++ b/streampipes-processors-geo-jvm/src/main/resources/org.apache.streampipes.processor.geo.jvm.reversegeocoding/documentation.md
@@ -27,6 +27,8 @@
This processor computes place name based on given lat/lng coordinates that are transmitted as fields from an event.
+This processor automatically downloads the file cities1000.zip from Geonames (http://download.geonames.org/export/dump/cities1000.zip). This file is provided under the CC BY 4.0 license, see https://creativecommons.org/licenses/by/4.0/ for license details.
+
***
## Required input