You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by cs...@apache.org on 2026/08/04 14:04:50 UTC
(impala) 01/02: IMPALA-15160: Add WKB_EXPERIMENTAL geospatial serialization mode
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ae81daed38448fe9973910a49b46239a424b7dc5
Author: Csaba Ringhofer <cs...@cloudera.com>
AuthorDate: Sat Jul 4 14:03:08 2026 +0200
IMPALA-15160: Add WKB_EXPERIMENTAL geospatial serialization mode
Add WKB_EXPERIMENTAL as a new value for the --geospatial_library flag.
When enabled, the copied ESRI UDFs serialize geometries as OGC
Well-Known Binary instead of the ESRI Shape format. The format is
chosen at startup via a static member in GeometryUtils.
The flag must be consistently set up in coordinators and executors.
Key behavioral differences in WKB mode:
1. Native C++ functions are disabled, all functions run in Java.
The plan is to implement most functions in c++ in the future
with WKB serialiation.
2. WKB doesn't store SRID, a serialization roundtrip sets it to 0.
3. Higher dimension geometry support is dropped. WKB could store them,
but not supporting these rare cases for now makes implementing and
testing c++ functions easier.
4. Relational functions (ST_Intersects, ST_Contains, ...) accept only
(BINARY, BINARY) arguments; the STRING overloads are removed
Other changes:
- Deserialized geometry caching logic is removed - this was never
working in Impala environment which always uses the same ByteBuffer.
Change-Id: Ifbb1b91171bf9a8834a55443d4a19518b38b759e
Assisted-by: Claude Opus 4.8
Reviewed-on: http://gerrit.cloudera.org:8080/24549
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/common/global-flags.cc | 3 +-
be/src/util/backend-gflag-util.cc | 3 +
common/thrift/BackendGflags.thrift | 3 +-
.../impala/compat/HiveEsriGeospatialBuiltins.java | 33 ++++---
.../java/org/apache/impala/catalog/BuiltinsDb.java | 4 +-
.../org/apache/impala/service/JniFrontend.java | 14 +++
.../impala/hive/geospatial/esri/GeometryUtils.java | 105 +++++++++++++++------
.../hive/geospatial/esri/HiveGeometryOIHelper.java | 17 +---
.../apache/impala/hive/geospatial/esri/README.md | 36 +++++++
.../queries/QueryTest/geospatial-esri-srid.test | 8 +-
.../queries/QueryTest/geospatial-esri.test | 11 +++
tests/custom_cluster/test_geospatial_library.py | 14 +++
12 files changed, 182 insertions(+), 69 deletions(-)
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index 0a7b66c56..ee60505dd 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -485,11 +485,12 @@ DEFINE_int32(state_store_ha_port, 24020,
// TGeospatialLibrary's values are mapped here as constants
static const string geo_lib_none = "NONE";
static const string geo_lib_hive_esri = "HIVE_ESRI";
+static const string geo_lib_wkb = "WKB_EXPERIMENTAL";
static const string geo_lib_help_msg =
"Specifies which implementation of "
"geospatial functions should be included as builtins. Possible values: [\""
- + geo_lib_none + "\", \"" + geo_lib_hive_esri + "\"]";
+ + geo_lib_none + "\", \"" + geo_lib_hive_esri + "\", \"" + geo_lib_wkb + "\"]";
DEFINE_string(geospatial_library, geo_lib_none, geo_lib_help_msg.c_str());
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
index 2e8e51c9b..757f04844 100644
--- a/be/src/util/backend-gflag-util.cc
+++ b/be/src/util/backend-gflag-util.cc
@@ -532,6 +532,9 @@ Status PopulateThriftBackendGflags(TBackendGflags& cfg) {
cfg.__set_enable_reload_events(FLAGS_enable_reload_events);
if (FLAGS_geospatial_library == to_string(TGeospatialLibrary::NONE)) {
cfg.__set_geospatial_library(TGeospatialLibrary::NONE);
+ } else if (
+ FLAGS_geospatial_library == to_string(TGeospatialLibrary::WKB_EXPERIMENTAL)) {
+ cfg.__set_geospatial_library(TGeospatialLibrary::WKB_EXPERIMENTAL);
} else {
DCHECK_EQ(FLAGS_geospatial_library, to_string(TGeospatialLibrary::HIVE_ESRI));
cfg.__set_geospatial_library(TGeospatialLibrary::HIVE_ESRI);
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
index 55918a43c..96455649c 100644
--- a/common/thrift/BackendGflags.thrift
+++ b/common/thrift/BackendGflags.thrift
@@ -28,7 +28,8 @@ enum TReservedWordsVersion {
// Options for Geospatial function library support
enum TGeospatialLibrary{
NONE,
- HIVE_ESRI
+ HIVE_ESRI,
+ WKB_EXPERIMENTAL
}
// Used to pass gflags from backend to frontend, JniCatalog and JniFrontend
diff --git a/fe/src/compat-hive-3/java/org/apache/impala/compat/HiveEsriGeospatialBuiltins.java b/fe/src/compat-hive-3/java/org/apache/impala/compat/HiveEsriGeospatialBuiltins.java
index 91ba73ba9..35be568de 100644
--- a/fe/src/compat-hive-3/java/org/apache/impala/compat/HiveEsriGeospatialBuiltins.java
+++ b/fe/src/compat-hive-3/java/org/apache/impala/compat/HiveEsriGeospatialBuiltins.java
@@ -64,13 +64,18 @@ public class HiveEsriGeospatialBuiltins {
*/
public static void initBuiltins(Db db) {
TGeospatialLibrary lib = BackendConfig.INSTANCE.getGeospatialLibrary();
- // Currently all native functions are expected to be 100% compatible with the Java
- // version. This is not true for the full function set of
- // https://gerrit.cloudera.org/#/c/20602/ , which sets based on a flag to allow
- // full compatibility with Hive.
- boolean addNatives = true;
+
+ // Set the serialization format on both branches so it is self-correcting
+ // even if initBuiltins were ever re-run with a different library.
+ GeometryUtils.setFormat(lib.equals(TGeospatialLibrary.WKB_EXPERIMENTAL)
+ ? GeometryUtils.SerializationFormat.WKB
+ : GeometryUtils.SerializationFormat.ESRI_SHAPE);
+
+ // Native C++ functions only work with ESRI Shape format, not WKB.
+ boolean addNatives = lib.equals(TGeospatialLibrary.HIVE_ESRI);
+ boolean isWkb = lib.equals(TGeospatialLibrary.WKB_EXPERIMENTAL);
addLegacyUDFs(db, addNatives);
- addGenericUDFs(db);
+ addGenericUDFs(db, isWkb);
addVarargsUDFs(db);
if(addNatives) {
addNatives(db);
@@ -78,7 +83,7 @@ public class HiveEsriGeospatialBuiltins {
}
private static void addLegacyUDFs(Db db, boolean addNatives) {
- List<UDF> legacyUDFs = Arrays.asList(new ST_Area(), new ST_AsBinary(),
+ List<UDF> legacyUDFs = new ArrayList<>(Arrays.asList(new ST_Area(), new ST_AsBinary(),
new ST_AsGeoJson(), new ST_AsJson(), new ST_AsShape(), new ST_AsText(),
new ST_Boundary(), new ST_Buffer(), new ST_Centroid(), new ST_CoordDim(),
new ST_Difference(), new ST_Dimension(), new ST_Distance(),
@@ -95,7 +100,7 @@ public class HiveEsriGeospatialBuiltins {
new ST_NumInteriorRing(), new ST_NumPoints(),
new ST_PointFromWKB(), new ST_PointN(), new ST_PolyFromWKB(),
new ST_Relate(), new ST_StartPoint(), new ST_SymmetricDiff(),
- new ST_Z());
+ new ST_Z()));
List<UDF> legacyUDFsWithNativeImplementation = Arrays.asList(
new ST_EnvIntersects(), new ST_GeometryType(),
@@ -117,7 +122,7 @@ public class HiveEsriGeospatialBuiltins {
addJavaStPoint(db, addNatives);
}
- private static void addGenericUDFs(Db db) {
+ private static void addGenericUDFs(Db db, boolean isWkb) {
List<ScalarFunction> genericUDFs = new ArrayList<>();
List<Set<Type>> stBinArguments =
@@ -140,20 +145,22 @@ public class HiveEsriGeospatialBuiltins {
genericUDFs.add(createScalarFunction(
ST_MultiLineString.class, Type.BINARY, new Type[] {Type.STRING}));
- createRelationalGenericUDFs(genericUDFs);
+ createRelationalGenericUDFs(genericUDFs, isWkb);
for (ScalarFunction function : genericUDFs) {
db.addBuiltin(function);
}
}
- private static void createRelationalGenericUDFs(List<ScalarFunction> genericUDFs) {
+ private static void createRelationalGenericUDFs(
+ List<ScalarFunction> genericUDFs, boolean isWkb) {
List<GenericUDF> relationalUDFs = Arrays.asList(new ST_Contains(), new ST_Crosses(),
new ST_Disjoint(), new ST_Equals(), new ST_Intersects(), new ST_Overlaps(),
new ST_Touches(), new ST_Within());
- List<Set<Type>> relationalUDFArguments =
- ImmutableList.of(ImmutableSet.of(Type.STRING, Type.BINARY),
+ List<Set<Type>> relationalUDFArguments = isWkb
+ ? ImmutableList.of(ImmutableSet.of(Type.BINARY), ImmutableSet.of(Type.BINARY))
+ : ImmutableList.of(ImmutableSet.of(Type.STRING, Type.BINARY),
ImmutableSet.of(Type.STRING, Type.BINARY));
for (GenericUDF relationalUDF : relationalUDFs) {
diff --git a/fe/src/main/java/org/apache/impala/catalog/BuiltinsDb.java b/fe/src/main/java/org/apache/impala/catalog/BuiltinsDb.java
index 5c67820cb..1da272d90 100644
--- a/fe/src/main/java/org/apache/impala/catalog/BuiltinsDb.java
+++ b/fe/src/main/java/org/apache/impala/catalog/BuiltinsDb.java
@@ -114,8 +114,8 @@ public class BuiltinsDb extends Db {
LikePredicate.initBuiltins(this);
ScalarBuiltins.initBuiltins(this);
- if (BackendConfig.INSTANCE.getGeospatialLibrary().equals(
- TGeospatialLibrary.HIVE_ESRI)) {
+ TGeospatialLibrary geoLib = BackendConfig.INSTANCE.getGeospatialLibrary();
+ if (!geoLib.equals(TGeospatialLibrary.NONE)) {
HiveEsriGeospatialBuiltins.initBuiltins(this);
}
}
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index 81f7327fe..78cf8959e 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -42,6 +42,8 @@ import org.apache.impala.authorization.ImpalaInternalAdminUser;
import org.apache.impala.authorization.ranger.RangerUtil;
import org.apache.impala.authorization.User;
import org.apache.impala.catalog.FeDataSource;
+import org.apache.impala.hive.geospatial.esri.GeometryUtils;
+import org.apache.impala.thrift.TGeospatialLibrary;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.Function;
import org.apache.impala.common.UserCancelledException;
@@ -153,6 +155,18 @@ public class JniFrontend {
BackendConfig.create(cfg);
+ // Initialize the geo serialization format on all impalads (coordinator and
+ // executor). On executor-only impalads the Java Frontend is not instantiated
+ // so BuiltinsDb/HiveEsriGeospatialBuiltins.initBuiltins() never runs, but
+ // Hive geo UDFs still execute via HiveUdfCall and therefore require the
+ // correct GeometryUtils.format to deserialize geometry values.
+ TGeospatialLibrary geoLib = BackendConfig.INSTANCE.getGeospatialLibrary();
+ if (!geoLib.equals(TGeospatialLibrary.NONE)) {
+ GeometryUtils.setFormat(geoLib.equals(TGeospatialLibrary.WKB_EXPERIMENTAL)
+ ? GeometryUtils.SerializationFormat.WKB
+ : GeometryUtils.SerializationFormat.ESRI_SHAPE);
+ }
+
GlogAppender.Install(TLogLevel.values()[cfg.impala_log_lvl],
TLogLevel.values()[cfg.non_impala_java_vlog]);
LOG.info(JniUtil.getJavaVersion());
diff --git a/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/GeometryUtils.java b/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/GeometryUtils.java
index 6c8efa0a3..795d58ed2 100644
--- a/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/GeometryUtils.java
+++ b/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/GeometryUtils.java
@@ -24,8 +24,6 @@ import com.esri.core.geometry.OperatorImportFromESRIShape;
import com.esri.core.geometry.Polygon;
import com.esri.core.geometry.SpatialReference;
import com.esri.core.geometry.ogc.OGCGeometry;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector;
import org.apache.hadoop.io.BytesWritable;
@@ -35,6 +33,14 @@ import java.nio.ByteOrder;
public class GeometryUtils {
+ // Selects the serialization format for all ESRI UDFs in this JVM. Assumed to
+ // be set once at startup by HiveEsriGeospatialBuiltins.initBuiltins (which
+ // runs per frontend JVM before any UDF executes) and not changed afterwards.
+ public enum SerializationFormat { ESRI_SHAPE, WKB }
+ private static volatile SerializationFormat format = SerializationFormat.ESRI_SHAPE;
+ public static void setFormat(SerializationFormat f) { format = f; }
+ public static SerializationFormat getFormat() { return format; }
+
private static final int SIZE_WKID = 4;
private static final int SIZE_TYPE = 1;
@@ -67,22 +73,31 @@ public class GeometryUtils {
public static final WritableBinaryObjectInspector geometryTransportObjectInspector =
PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
- private static final Cache<BytesWritable, OGCGeometry> geometryCache = CacheBuilder.newBuilder().weakKeys().build();
-
/**
* @param geomref1
* @param geomref2
* @return return true if both geometries are in the same spatial reference
*/
public static boolean compareSpatialReferences(BytesWritable geomref1, BytesWritable geomref2) {
+ if (format == SerializationFormat.WKB) return true;
return getWKID(geomref1) == getWKID(geomref2);
}
public static BytesWritable geometryToEsriShapeBytesWritable(MapGeometry mapGeometry) {
+ if (format == SerializationFormat.WKB) {
+ OGCGeometry ogc = OGCGeometry.createFromEsriGeometry(
+ mapGeometry.getGeometry(), mapGeometry.getSpatialReference());
+ return serializeWkb(ogc);
+ }
return serialize(mapGeometry);
}
public static BytesWritable geometryToEsriShapeBytesWritable(Geometry geometry, int wkid, OGCType type) {
+ if (format == SerializationFormat.WKB) {
+ SpatialReference sr = (wkid != WKID_UNKNOWN) ? SpatialReference.create(wkid) : null;
+ OGCGeometry ogc = OGCGeometry.createFromEsriGeometry(geometry, sr);
+ return serializeWkb(ogc);
+ }
return serialize(geometry, wkid, type);
}
@@ -91,36 +106,21 @@ public class GeometryUtils {
}
public static OGCGeometry geometryFromEsriShape(BytesWritable geomref) {
- // always assume bytes are recycled and can't be cached by using
- // geomref.getBytes() as the key
- return geometryFromEsriShape(geomref, true);
- }
-
- public static OGCGeometry geometryFromEsriShape(BytesWritable geomref, boolean bytesRecycled) {
if (geomref == null) {
return null;
}
- // this geomref might actually be a LazyGeometryBytesWritable which
+ // this geomref might actually be a CachedGeometryBytesWritable which
// means we don't need to deserialize from bytes
if (geomref instanceof CachedGeometryBytesWritable) {
return ((CachedGeometryBytesWritable) geomref).getGeometry();
}
- // if geomref bytes are recycled, we can't use the cache because every
- // key in the cache will be the same byte array
- if (!bytesRecycled) {
- // check for a cache hit to previously created geometries
- OGCGeometry cachedGeom = geometryCache.getIfPresent(geomref);
-
- if (cachedGeom != null) {
- return cachedGeom;
- }
+ if (format == SerializationFormat.WKB) {
+ return deserializeWkb(geomref);
}
- // not in cache or instance of CachedGeometryBytesWritable. now
- // need to create the geometry from its bytes
int wkid = getWKID(geomref);
ByteBuffer shapeBuffer = getShapeByteBuffer(geomref);
@@ -137,14 +137,7 @@ public class GeometryUtils {
}
Geometry esriGeom = OperatorImportFromESRIShape.local().execute(0, Geometry.Type.Unknown, shapeBuffer);
- OGCGeometry createdGeom = OGCGeometry.createFromEsriGeometry(esriGeom, spatialReference);
-
- if (!bytesRecycled) {
- // only add bytes to cache if we know they aren't being recycled
- geometryCache.put(geomref, createdGeom);
- }
-
- return createdGeom;
+ return OGCGeometry.createFromEsriGeometry(esriGeom, spatialReference);
}
}
}
@@ -156,6 +149,9 @@ public class GeometryUtils {
* @return OGCType set in the 5th byte of the hive geometry bytes
*/
public static OGCType getType(BytesWritable geomref) {
+ if (format == SerializationFormat.WKB) {
+ return getTypeFromWkb(geomref);
+ }
// SIZE_WKID is the offset to the byte that stores the type information
return OGCTypeLookup[geomref.getBytes()[SIZE_WKID]];
}
@@ -176,6 +172,7 @@ public class GeometryUtils {
* @return WKID set in the first 4 bytes of the hive geometry bytes
*/
public static int getWKID(BytesWritable geomref) {
+ if (format == SerializationFormat.WKB) return WKID_UNKNOWN;
ByteBuffer bb = ByteBuffer.wrap(geomref.getBytes());
return bb.getInt(0);
}
@@ -187,6 +184,7 @@ public class GeometryUtils {
* @param wkid
*/
public static void setWKID(BytesWritable geomref, int wkid) {
+ if (format == SerializationFormat.WKB) return;
ByteBuffer bb = ByteBuffer.allocate(4);
bb.putInt(wkid);
System.arraycopy(bb.array(), 0, geomref.getBytes(), 0, SIZE_WKID);
@@ -310,12 +308,57 @@ public class GeometryUtils {
return ret;
}
+ // --- WKB support methods ---
+
+ private static OGCGeometry deserializeWkb(BytesWritable geomref) {
+ byte[] bytes = geomref.getBytes();
+ int len = geomref.getLength();
+ if (len < 5) return null;
+
+ ByteBuffer wkbBuffer = ByteBuffer.wrap(bytes, 0, len);
+ return OGCGeometry.fromBinary(wkbBuffer);
+ }
+
+ private static BytesWritable serializeWkb(OGCGeometry ogcGeometry) {
+ if (ogcGeometry == null) return null;
+ ByteBuffer wkb = ogcGeometry.asBinary();
+ byte[] bytes = new byte[wkb.remaining()];
+ wkb.get(bytes);
+ return new BytesWritable(bytes);
+ }
+
+ private static OGCType getTypeFromWkb(BytesWritable geomref) {
+ byte[] bytes = geomref.getBytes();
+ int len = geomref.getLength();
+ if (len < 5) return OGCType.UNKNOWN;
+ ByteOrder order = (bytes[0] == 1) ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN;
+ int wkbType = ByteBuffer.wrap(bytes, 1, 4).order(order).getInt();
+ // 2D only: ISO-WKB would mark with Z/M/ZM offsets (1000/2000/3000). This does
+ // not handle PostGIS-style EWKB high-bit flags; revisit if >2D lands (IMPALA-15168).
+ // Currently WKB mode is designed to use Parquet/Iceberg geometries without
+ // modifications, but in the future it may be useful to alter the internal format
+ // e.g using EWKB.
+ switch (wkbType) {
+ case 1: return OGCType.ST_POINT;
+ case 2: return OGCType.ST_LINESTRING;
+ case 3: return OGCType.ST_POLYGON;
+ case 4: return OGCType.ST_MULTIPOINT;
+ case 5: return OGCType.ST_MULTILINESTRING;
+ case 6: return OGCType.ST_MULTIPOLYGON;
+ default: return OGCType.UNKNOWN;
+ }
+ }
+
public static class CachedGeometryBytesWritable extends BytesWritable {
OGCGeometry cachedGeom;
public CachedGeometryBytesWritable(OGCGeometry geom) {
cachedGeom = geom;
- super.set(serialize(cachedGeom));
+ if (format == SerializationFormat.WKB) {
+ super.set(serializeWkb(cachedGeom));
+ } else {
+ super.set(serialize(cachedGeom));
+ }
}
public OGCGeometry getGeometry() {
diff --git a/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/HiveGeometryOIHelper.java b/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/HiveGeometryOIHelper.java
index d575130f4..5e0f8cf25 100644
--- a/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/HiveGeometryOIHelper.java
+++ b/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/HiveGeometryOIHelper.java
@@ -145,23 +145,8 @@ public class HiveGeometryOIHelper {
}
}
- private BytesWritable last = null;
-
- // always assume bytes are reused until we determine they aren't
- private boolean bytesReused = true;
-
private OGCGeometry getGeometryFromBytes(BytesWritable writable) {
-
- if (bytesReused) {
- if (last != null && last != writable) {
- // this assumes that the source of these bytes will either always
- // reuse the bytes or never reuse the bytes.
- bytesReused = false;
- }
- last = writable;
- }
-
- return GeometryUtils.geometryFromEsriShape(writable, bytesReused);
+ return GeometryUtils.geometryFromEsriShape(writable);
}
@Override
diff --git a/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/README.md b/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/README.md
index 93e972bc0..b6ad776e7 100644
--- a/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/README.md
+++ b/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/README.md
@@ -14,3 +14,39 @@ originally ported from the [Esri Spatial Framework for Hadoop](https://github.co
- Package renamed from `org.apache.hadoop.hive.ql.udf.esri`
- `serde/` and `shims/` subdirectories not copied (Hive-specific table SerDes, not
needed for ST_ UDF execution)
+- Deserialized geometry caching removed from `GeometryUtils` and `HiveGeometryOIHelper`.
+ The original Hive caching (`geometryCache` / `bytesRecycled`) never worked in Impala
+ due to the way Hive UDFs parameters are passed.
+- `GeometryUtils.SerializationFormat` enum added; the active format is set once at
+ startup via `HiveEsriGeospatialBuiltins.initBuiltins` and must be consistent across
+ coordinators and executors.
+
+## Serialization Modes
+
+The active mode is controlled by the `--geospatial_library` impalad startup flag.
+
+### HIVE_ESRI (default)
+
+Geometries are serialized as ESRI Shape format: a 4-byte WKID (spatial reference ID)
+followed by a 1-byte OGC type tag, followed by the ESRI binary shape payload. Native
+C++ implementations of selected ST_ functions are registered alongside the Java UDFs.
+Relational functions (e.g. `ST_Intersects`) accept both `STRING` and `BINARY` arguments.
+
+### WKB_EXPERIMENTAL
+
+Geometries are serialized as OGC Well-Known Binary (WKB). This mode is intended as
+the foundation for future native C++ implementations using the WKB wire format.
+
+Key behavioral differences from HIVE_ESRI:
+
+1. **No native C++ functions** — all ST_ functions execute in Java.
+2. **SRID is not stored** — WKB has no SRID field, so a serialize/deserialize roundtrip
+ always yields SRID 0. `ST_SRID` and functions that compare spatial references ignore
+ the SRID silently.
+3. **Higher-dimension geometry dropped** — geometries with Z/M coordinates are not
+ supported. This simplifies future C++ implementation and testing.
+4. **Relational functions are BINARY-only** — `ST_Intersects`, `ST_Contains`, etc.
+ are registered only for `(BINARY, BINARY)` argument types; the `STRING` overloads
+ present in HIVE_ESRI mode are non-standard and are intentionally dropped.
+ An alternative can be to allow implicit casting from STRING to GEOMETRY in the
+ future that parses the STRING as WKT.
diff --git a/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri-srid.test b/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri-srid.test
index 80cf6aa64..72088a098 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri-srid.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri-srid.test
@@ -78,12 +78,10 @@ select ST_AsGeoJson(ST_GeomFromJson('{"x":1.2,"y":2.4,"spatialReference":{"wkid"
'{"type":"Point","coordinates":[1.2,2.4],"crs":{"type":"name","properties":{"name":"EPSG:4326"}}}'
====
---- QUERY
+# TODO: WKB deserialization via OGCGeometry.fromBinary() assigns EPSG:4326 as default SR,
+# while ESRI Shape deserialization with WKID=0 assigns null SR. Both produce different
+# crs output. SRID handling is deferred to IMPALA-15169.
select ST_AsGeoJson(ST_GeomFromJson('{"x":1.2,"y":2.4,"spatialReference":{"wkid": 0}}'));
---- RESULTS
'{"type":"Point","coordinates":[1.2,2.4],"crs":null}'
====
----- QUERY
-select ST_AsText(ST_Intersection(ST_Polygon(2,0, 2,3, 3,0), ST_Polygon(1,1, 4,1, 4,4, 1,4)));
----- RESULTS
-'POLYGON ((2 1, 2.6666666666666665 1, 2 3, 2 1))'
-====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri.test b/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri.test
index a133ed008..a81676b9a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri.test
@@ -2064,6 +2064,17 @@ select ST_AsText(ST_Intersection(ST_Polygon(1,1, 1,4, 4,4, 4,1), ST_LineString(1
'LINESTRING (1 3, 1 4)'
====
---- QUERY
+# WKB roundtrip changes internal polygon representation, causing intersection to differ slightly.
+# TODO: The input polygon ST_Polygon(2,0, 2,3, 3,0) has clockwise winding (incorrect per OGC).
+# ESRI silently corrects to CCW, but the CW input may contribute to format-specific differences.
+# Can be probably fixed when all these functions are ported to c++ (IMPALA-15164).
+select ST_AsText(ST_Intersection(ST_Polygon(2,0, 2,3, 3,0), ST_Polygon(1,1, 4,1, 4,4, 1,4)))
+ in ('POLYGON ((2 1, 2.6666666666666665 1, 2 3, 2 1))',
+ 'POLYGON ((2 1, 2.666666666666667 1, 2 3, 2 1))');
+---- RESULTS
+true
+====
+---- QUERY
select ST_AsText(ST_Intersection(ST_Polygon(1,0, 3,0, 1,2), ST_Polygon(0,1, 2,1, 0,3)));
---- RESULTS
'POLYGON ((1 1, 2 1, 1 2, 1 1))'
diff --git a/tests/custom_cluster/test_geospatial_library.py b/tests/custom_cluster/test_geospatial_library.py
index c44dee15e..6ab0350ac 100644
--- a/tests/custom_cluster/test_geospatial_library.py
+++ b/tests/custom_cluster/test_geospatial_library.py
@@ -42,3 +42,17 @@ class TestGeospatialLibrary(CustomClusterTestSuite):
result = self.execute_query(SHOW_FUNCTIONS)
assert ST_POINT_SIGNATURE in result.data
assert ST_X_SIGNATURE_BUILTIN in result.data
+
+ # Only geospatial-esri and geospatial-esri-extra run under WKB mode. The other
+ # ESRI test files are intentionally excluded.
+ @CustomClusterTestSuite.with_args(
+ start_args='--geospatial_library=WKB_EXPERIMENTAL')
+ @SkipIfApacheHive.feature_not_supported
+ def test_wkb_experimental_esri(self, vector):
+ self.run_test_case('QueryTest/geospatial-esri', vector)
+
+ @CustomClusterTestSuite.with_args(
+ start_args='--geospatial_library=WKB_EXPERIMENTAL')
+ @SkipIfApacheHive.feature_not_supported
+ def test_wkb_experimental_extra(self, vector):
+ self.run_test_case('QueryTest/geospatial-esri-extra', vector)