You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by uw...@apache.org on 2017/03/24 10:54:23 UTC
arrow git commit: ARROW-682: [Integration] Check implementations
against themselves
Repository: arrow
Updated Branches:
refs/heads/master dcaa8e5d7 -> 13c12c6ea
ARROW-682: [Integration] Check implementations against themselves
This adds an additional layer of internal consistency checks
Author: Wes McKinney <we...@twosigma.com>
Closes #433 from wesm/ARROW-682 and squashes the following commits:
b33ac7a [Wes McKinney] Run integration tests with same implementation producing and consuming to validate internal consistency
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/13c12c6e
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/13c12c6e
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/13c12c6e
Branch: refs/heads/master
Commit: 13c12c6ea5e23928268b5c2c7b962d223cca7bd4
Parents: dcaa8e5
Author: Wes McKinney <we...@twosigma.com>
Authored: Fri Mar 24 11:54:18 2017 +0100
Committer: Uwe L. Korn <uw...@xhochy.com>
Committed: Fri Mar 24 11:54:18 2017 +0100
----------------------------------------------------------------------
integration/integration_test.py | 56 +++++++++++++++++++-----------------
1 file changed, 29 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/13c12c6e/integration/integration_test.py
----------------------------------------------------------------------
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 5cd63c5..ec2a38d 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -34,10 +34,12 @@ ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
# Control for flakiness
np.random.seed(12345)
+
def load_version_from_pom():
import xml.etree.ElementTree as ET
tree = ET.parse(os.path.join(ARROW_HOME, 'java', 'pom.xml'))
- version_tag = list(tree.getroot().findall('{http://maven.apache.org/POM/4.0.0}version'))[0]
+ tag_pattern = '{http://maven.apache.org/POM/4.0.0}version'
+ version_tag = list(tree.getroot().findall(tag_pattern))[0]
return version_tag.text
@@ -596,32 +598,32 @@ class IntegrationRunner(object):
def run(self):
for producer, consumer in itertools.product(self.testers,
self.testers):
- if producer is consumer:
- continue
-
- print('-- {0} producing, {1} consuming'.format(producer.name,
- consumer.name))
-
- for json_path in self.json_files:
- print('Testing file {0}'.format(json_path))
-
- # Make the random access file
- print('-- Creating binary inputs')
- producer_file_path = os.path.join(self.temp_dir, guid())
- producer.json_to_file(json_path, producer_file_path)
-
- # Validate the file
- print('-- Validating file')
- consumer.validate(json_path, producer_file_path)
-
- print('-- Validating stream')
- producer_stream_path = os.path.join(self.temp_dir, guid())
- consumer_file_path = os.path.join(self.temp_dir, guid())
- producer.file_to_stream(producer_file_path,
- producer_stream_path)
- consumer.stream_to_file(producer_stream_path,
- consumer_file_path)
- consumer.validate(json_path, consumer_file_path)
+ self._compare_implementations(producer, consumer)
+
+ def _compare_implementations(self, producer, consumer):
+ print('-- {0} producing, {1} consuming'.format(producer.name,
+ consumer.name))
+
+ for json_path in self.json_files:
+ print('Testing file {0}'.format(json_path))
+
+ # Make the random access file
+ print('-- Creating binary inputs')
+ producer_file_path = os.path.join(self.temp_dir, guid())
+ producer.json_to_file(json_path, producer_file_path)
+
+ # Validate the file
+ print('-- Validating file')
+ consumer.validate(json_path, producer_file_path)
+
+ print('-- Validating stream')
+ producer_stream_path = os.path.join(self.temp_dir, guid())
+ consumer_file_path = os.path.join(self.temp_dir, guid())
+ producer.file_to_stream(producer_file_path,
+ producer_stream_path)
+ consumer.stream_to_file(producer_stream_path,
+ consumer_file_path)
+ consumer.validate(json_path, consumer_file_path)
class Tester(object):