You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by dw...@apache.org on 2019/08/19 23:52:40 UTC
[incubator-iceberg] branch master updated: Bringing expression
implementations into a consistent state (#381)
This is an automated email from the ASF dual-hosted git repository.
dweeks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 36d2bf0 Bringing expression implementations into a consistent state (#381)
36d2bf0 is described below
commit 36d2bf0af586549be4e4d4893685183f05a15edb
Author: TGooch44 <te...@gmail.com>
AuthorDate: Mon Aug 19 16:52:36 2019 -0700
Bringing expression implementations into a consistent state (#381)
---
python/iceberg/api/expressions/evaluator.py | 17 ++++++++---------
.../api/expressions/inclusive_manifest_evaluator.py | 16 ++++++++--------
.../api/expressions/inclusive_metrics_evaluator.py | 14 +++++++-------
python/iceberg/api/expressions/literals.py | 10 ++--------
python/iceberg/api/expressions/residual_evaluator.py | 16 ++++++++--------
.../iceberg/api/expressions/strict_metrics_evaluator.py | 16 ++++++++--------
6 files changed, 41 insertions(+), 48 deletions(-)
diff --git a/python/iceberg/api/expressions/evaluator.py b/python/iceberg/api/expressions/evaluator.py
index 3f2a29f..2371624 100644
--- a/python/iceberg/api/expressions/evaluator.py
+++ b/python/iceberg/api/expressions/evaluator.py
@@ -22,20 +22,19 @@ from .expressions import ExpressionVisitors
class Evaluator(object):
- THREAD_LOCAL_DATA = threading.local()
-
- def visitor(self):
- if not hasattr(Evaluator.THREAD_LOCAL_DATA, "visitors"):
- Evaluator.THREAD_LOCAL_DATA.visitors = Evaluator.EvalVisitor()
-
- return Evaluator.THREAD_LOCAL_DATA.visitors
def __init__(self, struct, unbound, case_sensitive=True):
self.expr = Binder.bind(struct, unbound, case_sensitive)
- self.visitors = None
+ self.thread_local_data = threading.local()
+
+ def _visitor(self):
+ if not hasattr(self.thread_local_data, "visitors"):
+ self.thread_local_data.visitors = Evaluator.EvalVisitor()
+
+ return self.thread_local_data.visitors
def eval(self, data):
- return self.visitor().eval(data, self.expr)
+ return self._visitor().eval(data, self.expr)
class EvalVisitor(ExpressionVisitors.BoundExpressionVisitor):
diff --git a/python/iceberg/api/expressions/inclusive_manifest_evaluator.py b/python/iceberg/api/expressions/inclusive_manifest_evaluator.py
index e1c55fb..953ee99 100644
--- a/python/iceberg/api/expressions/inclusive_manifest_evaluator.py
+++ b/python/iceberg/api/expressions/inclusive_manifest_evaluator.py
@@ -28,12 +28,6 @@ ROWS_CANNOT_MATCH = False
class InclusiveManifestEvaluator(object):
- def visitor(self):
- if not hasattr(self.thread_local_data, "visitors"):
- self.thread_local_data.visitors = ManifestEvalVistor(self.expr)
-
- return self.thread_local_data.visitors
-
def __init__(self, spec, row_filter, case_sensitive=True):
self.struct = spec.partition_type()
self.expr = Binder.bind(self.struct,
@@ -42,11 +36,17 @@ class InclusiveManifestEvaluator(object):
case_sensitive=case_sensitive)
self.thread_local_data = threading.local()
+ def _visitor(self):
+ if not hasattr(self.thread_local_data, "visitors"):
+ self.thread_local_data.visitors = ManifestEvalVisitor(self.expr)
+
+ return self.thread_local_data.visitors
+
def eval(self, manifest):
- return self.visitor().eval(manifest)
+ return self._visitor().eval(manifest)
-class ManifestEvalVistor(ExpressionVisitors.BoundExpressionVisitor):
+class ManifestEvalVisitor(ExpressionVisitors.BoundExpressionVisitor):
def __init__(self, expr):
self.expr = expr
diff --git a/python/iceberg/api/expressions/inclusive_metrics_evaluator.py b/python/iceberg/api/expressions/inclusive_metrics_evaluator.py
index ac54935..e9ab1a0 100644
--- a/python/iceberg/api/expressions/inclusive_metrics_evaluator.py
+++ b/python/iceberg/api/expressions/inclusive_metrics_evaluator.py
@@ -24,12 +24,6 @@ from ..types import Conversions
class InclusiveMetricsEvaluator(object):
- def visitor(self):
- if not hasattr(self.thread_local_data, "visitors"):
- self.thread_local_data.visitors = MetricsEvalVisitor(self.expr, self.schema, self.struct)
-
- return self.thread_local_data.visitors
-
def __init__(self, schema, unbound, case_sensitive=True):
self.schema = schema
self.struct = schema.as_struct()
@@ -37,8 +31,14 @@ class InclusiveMetricsEvaluator(object):
self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound), case_sensitive)
self.thread_local_data = threading.local()
+ def _visitor(self):
+ if not hasattr(self.thread_local_data, "visitors"):
+ self.thread_local_data.visitors = MetricsEvalVisitor(self.expr, self.schema, self.struct)
+
+ return self.thread_local_data.visitors
+
def eval(self, file):
- return self.visitor().eval(file)
+ return self._visitor().eval(file)
class MetricsEvalVisitor(ExpressionVisitors.BoundExpressionVisitor):
diff --git a/python/iceberg/api/expressions/literals.py b/python/iceberg/api/expressions/literals.py
index d340172..e8a9fa5 100644
--- a/python/iceberg/api/expressions/literals.py
+++ b/python/iceberg/api/expressions/literals.py
@@ -18,7 +18,6 @@
import datetime
from decimal import (Decimal,
ROUND_HALF_UP)
-import sys
import uuid
import pytz
@@ -30,11 +29,6 @@ from .java_variables import (JAVA_MAX_FLOAT,
JAVA_MIN_FLOAT)
from ..types.type import TypeID
-# to-do un-wind python 2 switches
-if sys.version_info >= (3, 0):
- unicode = str
- long = int
-
class Literals(object):
@@ -55,7 +49,7 @@ class Literals(object):
if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT:
return FloatLiteral(value)
return DoubleLiteral(value)
- elif isinstance(value, (str, unicode)):
+ elif isinstance(value, str):
return StringLiteral(value)
elif isinstance(value, uuid.UUID):
return UUIDLiteral(value)
@@ -96,7 +90,7 @@ class Literal(object):
if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT:
return DoubleLiteral(value)
return FloatLiteral(value)
- elif isinstance(value, (str, unicode)):
+ elif isinstance(value, str):
return StringLiteral(value)
elif isinstance(value, uuid.UUID):
return UUIDLiteral(value)
diff --git a/python/iceberg/api/expressions/residual_evaluator.py b/python/iceberg/api/expressions/residual_evaluator.py
index 917c49d..d8d506f 100644
--- a/python/iceberg/api/expressions/residual_evaluator.py
+++ b/python/iceberg/api/expressions/residual_evaluator.py
@@ -23,19 +23,19 @@ from .predicate import BoundPredicate, Predicate, UnboundPredicate
class ResidualEvaluator(object):
- def visitor(self):
- if not hasattr(self.thread_local_data, "visitors"):
- self.thread_local_data.visitors = ResidualVisitor()
-
- return self.thread_local_data.visitors
-
def __init__(self, spec, expr):
self._spec = spec
self._expr = expr
- self.thread_local_data = threading.local()
+ self.__visitor = None
+
+ def _visitor(self):
+ if self.__visitor is None:
+ self.__visitor = ResidualVisitor()
+
+ return self.__visitor
def residual_for(self, partition_data):
- return self.visitor().eval(partition_data)
+ return self._visitor().eval(partition_data)
class ResidualVisitor(ExpressionVisitors.BoundExpressionVisitor):
diff --git a/python/iceberg/api/expressions/strict_metrics_evaluator.py b/python/iceberg/api/expressions/strict_metrics_evaluator.py
index a1752f9..6d44df3 100644
--- a/python/iceberg/api/expressions/strict_metrics_evaluator.py
+++ b/python/iceberg/api/expressions/strict_metrics_evaluator.py
@@ -24,7 +24,13 @@ from ..types import Conversions
class StrictMetricsEvaluator(object):
- def visitor(self):
+ def __init__(self, schema, unbound):
+ self.schema = schema
+ self.struct = schema.as_struct()
+ self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound))
+ self.thread_local_data = threading.local()
+
+ def _visitor(self):
if not hasattr(self.thread_local_data, "visitors"):
self.thread_local_data.visitors = StrictMetricsEvaluator.MetricsEvalVisitor(
self.expr,
@@ -34,14 +40,8 @@ class StrictMetricsEvaluator(object):
return self.thread_local_data.visitors
- def __init__(self, schema, unbound):
- self.schema = schema
- self.struct = schema.as_struct()
- self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound))
- self.thread_local_data = threading.local()
-
def eval(self, file):
- return self.visitor().eval(file)
+ return self._visitor().eval(file)
class MetricsEvalVisitor(ExpressionVisitors.BoundExpressionVisitor):
ROWS_MUST_MATCH = True