You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/12/30 11:16:09 UTC
spark git commit: [SPARK-18922][TESTS] Fix more path-related test
failures on Windows
Repository: spark
Updated Branches:
refs/heads/master 56d3a7eb8 -> 852782b83
[SPARK-18922][TESTS] Fix more path-related test failures on Windows
## What changes were proposed in this pull request?
This PR proposes to fix the test failures due to different format of paths on Windows.
Failed tests are as below:
```
ColumnExpressionSuite:
- input_file_name, input_file_block_start, input_file_block_length - FileScanRDD *** FAILED *** (187 milliseconds)
"file:///C:/projects/spark/target/tmp/spark-0b21b963-6cfa-411c-8d6f-e6a5e1e73bce/part-00001-c083a03a-e55e-4b05-9073-451de352d006.snappy.parquet" did not contain "C:\projects\spark\target\tmp\spark-0b21b963-6cfa-411c-8d6f-e6a5e1e73bce" (ColumnExpressionSuite.scala:545)
- input_file_name, input_file_block_start, input_file_block_length - HadoopRDD *** FAILED *** (172 milliseconds)
"file:/C:/projects/spark/target/tmp/spark-5d0afa94-7c2f-463b-9db9-2e8403e2bc5f/part-00000-f6530138-9ad3-466d-ab46-0eeb6f85ed0b.txt" did not contain "C:\projects\spark\target\tmp\spark-5d0afa94-7c2f-463b-9db9-2e8403e2bc5f" (ColumnExpressionSuite.scala:569)
- input_file_name, input_file_block_start, input_file_block_length - NewHadoopRDD *** FAILED *** (156 milliseconds)
"file:/C:/projects/spark/target/tmp/spark-a894c7df-c74d-4d19-82a2-a04744cb3766/part-00000-29674e3f-3fcf-4327-9b04-4dab1d46338d.txt" did not contain "C:\projects\spark\target\tmp\spark-a894c7df-c74d-4d19-82a2-a04744cb3766" (ColumnExpressionSuite.scala:598)
```
```
DataStreamReaderWriterSuite:
- source metadataPath *** FAILED *** (62 milliseconds)
org.mockito.exceptions.verification.junit.ArgumentsAreDifferent: Argument(s) are different! Wanted:
streamSourceProvider.createSource(
org.apache.spark.sql.SQLContext3b04133b,
"C:\projects\spark\target\tmp\streaming.metadata-b05db6ae-c8dc-4ce4-b0d9-1eb8c84876c0/sources/0",
None,
"org.apache.spark.sql.streaming.test",
Map()
);
-> at org.apache.spark.sql.streaming.test.DataStreamReaderWriterSuite$$anonfun$12.apply$mcV$sp(DataStreamReaderWriterSuite.scala:374)
Actual invocation has different arguments:
streamSourceProvider.createSource(
org.apache.spark.sql.SQLContext3b04133b,
"/C:/projects/spark/target/tmp/streaming.metadata-b05db6ae-c8dc-4ce4-b0d9-1eb8c84876c0/sources/0",
None,
"org.apache.spark.sql.streaming.test",
Map()
);
```
```
GlobalTempViewSuite:
- CREATE GLOBAL TEMP VIEW USING *** FAILED *** (110 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-960398ba-a0a1-45f6-a59a-d98533f9f519;
```
```
CreateTableAsSelectSuite:
- CREATE TABLE USING AS SELECT *** FAILED *** (0 milliseconds)
java.lang.IllegalArgumentException: Can not create a Path from an empty string
- create a table, drop it and create another one with the same name *** FAILED *** (16 milliseconds)
java.lang.IllegalArgumentException: Can not create a Path from an empty string
- create table using as select - with partitioned by *** FAILED *** (0 milliseconds)
java.lang.IllegalArgumentException: Can not create a Path from an empty string
- create table using as select - with non-zero buckets *** FAILED *** (0 milliseconds)
java.lang.IllegalArgumentException: Can not create a Path from an empty string
```
```
HiveMetadataCacheSuite:
- partitioned table is cached when partition pruning is true *** FAILED *** (532 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- partitioned table is cached when partition pruning is false *** FAILED *** (297 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
```
```
MultiDatabaseSuite:
- createExternalTable() to non-default database - with USE *** FAILED *** (954 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-0839d9a7-5e29-467a-9e3e-3e4cd618ee09;
- createExternalTable() to non-default database - without USE *** FAILED *** (500 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-c7e24d73-1d8f-45e8-ab7d-53a83087aec3;
- invalid database name and table names *** FAILED *** (31 milliseconds)
"Path does not exist: file:/C:projectsspark arget mpspark-15a2a494-3483-4876-80e5-ec396e704b77;" did not contain "`t:a` is not a valid name for tables/databases. Valid names only contain alphabet characters, numbers and _." (MultiDatabaseSuite.scala:296)
```
```
OrcQuerySuite:
- SPARK-8501: Avoids discovery schema from empty ORC files *** FAILED *** (15 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- Verify the ORC conversion parameter: CONVERT_METASTORE_ORC *** FAILED *** (78 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- converted ORC table supports resolving mixed case field *** FAILED *** (297 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
```
```
HadoopFsRelationTest - JsonHadoopFsRelationSuite, OrcHadoopFsRelationSuite, ParquetHadoopFsRelationSuite, SimpleTextHadoopFsRelationSuite:
- Locality support for FileScanRDD *** FAILED *** (15 milliseconds)
java.lang.IllegalArgumentException: Wrong FS: file://C:\projects\spark\target\tmp\spark-383d1f13-8783-47fd-964d-9c75e5eec50f, expected: file:///
```
```
HiveQuerySuite:
- CREATE TEMPORARY FUNCTION *** FAILED *** (0 milliseconds)
java.net.MalformedURLException: For input string: "%5Cprojects%5Cspark%5Csql%5Chive%5Ctarget%5Cscala-2.11%5Ctest-classes%5CTestUDTF.jar"
- ADD FILE command *** FAILED *** (500 milliseconds)
java.net.URISyntaxException: Illegal character in opaque part at index 2: C:\projects\spark\sql\hive\target\scala-2.11\test-classes\data\files\v1.txt
- ADD JAR command 2 *** FAILED *** (110 milliseconds)
org.apache.spark.sql.AnalysisException: LOAD DATA input path does not exist: C:projectssparksqlhive argetscala-2.11 est-classesdatafilessample.json;
```
```
PruneFileSourcePartitionsSuite:
- PruneFileSourcePartitions should not change the output of LogicalRelation *** FAILED *** (15 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
```
```
HiveCommandSuite:
- LOAD DATA LOCAL *** FAILED *** (109 milliseconds)
org.apache.spark.sql.AnalysisException: LOAD DATA input path does not exist: C:projectssparksqlhive argetscala-2.11 est-classesdatafilesemployee.dat;
- LOAD DATA *** FAILED *** (93 milliseconds)
java.net.URISyntaxException: Illegal character in opaque part at index 15: C:projectsspark arget mpemployee.dat7496657117354281006.tmp
- Truncate Table *** FAILED *** (78 milliseconds)
org.apache.spark.sql.AnalysisException: LOAD DATA input path does not exist: C:projectssparksqlhive argetscala-2.11 est-classesdatafilesemployee.dat;
```
```
HiveExternalCatalogBackwardCompatibilitySuite:
- make sure we can read table created by old version of Spark *** FAILED *** (0 milliseconds)
"[/C:/projects/spark/target/tmp/]spark-0554d859-74e1-..." did not equal "[C:\projects\spark\target\tmp\]spark-0554d859-74e1-..." (HiveExternalCatalogBackwardCompatibilitySuite.scala:213)
org.scalatest.exceptions.TestFailedException
- make sure we can alter table location created by old version of Spark *** FAILED *** (110 milliseconds)
java.net.URISyntaxException: Illegal character in opaque part at index 15: C:projectsspark arget mpspark-0e9b2c5f-49a1-4e38-a32a-c0ab1813a79f
```
```
ExternalCatalogSuite:
- create/drop/rename partitions should create/delete/rename the directory *** FAILED *** (610 milliseconds)
java.net.URISyntaxException: Illegal character in opaque part at index 2: C:\projects\spark\target\tmp\spark-4c24f010-18df-437b-9fed-990c6f9adece
```
```
SQLQuerySuite:
- describe functions - temporary user defined functions *** FAILED *** (16 milliseconds)
java.net.URISyntaxException: Illegal character in opaque part at index 22: C:projectssparksqlhive argetscala-2.11 est-classesTestUDTF.jar
- specifying database name for a temporary table is not allowed *** FAILED *** (125 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-a34c9814-a483-43f2-be29-37f616b6df91;
```
```
PartitionProviderCompatibilitySuite:
- convert partition provider to hive with repair table *** FAILED *** (281 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-ee5fc96d-8c7d-4ebf-8571-a1d62736473e;
- when partition management is enabled, new tables have partition provider hive *** FAILED *** (187 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-803ad4d6-3e8c-498d-9ca5-5cda5d9b2a48;
- when partition management is disabled, new tables have no partition provider *** FAILED *** (172 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-c9fda9e2-4020-465f-8678-52cd72d0a58f;
- when partition management is disabled, we preserve the old behavior even for new tables *** FAILED *** (203 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget
mpspark-f4a518a6-c49d-43d3-b407-0ddd76948e13;
- insert overwrite partition of legacy datasource table *** FAILED *** (188 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-f4a518a6-c49d-43d3-b407-0ddd76948e79;
- insert overwrite partition of new datasource table overwrites just partition *** FAILED *** (219 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-6ba3a88d-6f6c-42c5-a9f4-6d924a0616ff;
- SPARK-18544 append with saveAsTable - partition management true *** FAILED *** (173 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-cd234a6d-9cb4-4d1d-9e51-854ae9543bbd;
- SPARK-18635 special chars in partition values - partition management true *** FAILED *** (2 seconds, 967 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- SPARK-18635 special chars in partition values - partition management false *** FAILED *** (62 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- SPARK-18659 insert overwrite table with lowercase - partition management true *** FAILED *** (63 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- SPARK-18544 append with saveAsTable - partition management false *** FAILED *** (266 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- SPARK-18659 insert overwrite table files - partition management false *** FAILED *** (63 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- SPARK-18659 insert overwrite table with lowercase - partition management false *** FAILED *** (78 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- sanity check table setup *** FAILED *** (31 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- insert into partial dynamic partitions *** FAILED *** (47 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- insert into fully dynamic partitions *** FAILED *** (62 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- insert into static partition *** FAILED *** (78 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- overwrite partial dynamic partitions *** FAILED *** (63 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- overwrite fully dynamic partitions *** FAILED *** (47 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- overwrite static partition *** FAILED *** (63 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
```
```
MetastoreDataSourcesSuite:
- check change without refresh *** FAILED *** (203 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-00713fe4-ca04-448c-bfc7-6c5e9a2ad2a1;
- drop, change, recreate *** FAILED *** (78 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-2030a21b-7d67-4385-a65b-bb5e2bed4861;
- SPARK-15269 external data source table creation *** FAILED *** (78 milliseconds)
org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-4d50fd4a-14bc-41d6-9232-9554dd233f86;
- CTAS *** FAILED *** (109 milliseconds)
java.lang.IllegalArgumentException: Can not create a Path from an empty string
- CTAS with IF NOT EXISTS *** FAILED *** (109 milliseconds)
java.lang.IllegalArgumentException: Can not create a Path from an empty string
- CTAS: persisted partitioned bucketed data source table *** FAILED *** (0 milliseconds)
java.lang.IllegalArgumentException: Can not create a Path from an empty string
- SPARK-15025: create datasource table with path with select *** FAILED *** (16 milliseconds)
java.lang.IllegalArgumentException: Can not create a Path from an empty string
- CTAS: persisted partitioned data source table *** FAILED *** (47 milliseconds)
java.lang.IllegalArgumentException: Can not create a Path from an empty string
```
```
HiveMetastoreCatalogSuite:
- Persist non-partitioned parquet relation into metastore as managed table using CTAS *** FAILED *** (16 milliseconds)
java.lang.IllegalArgumentException: Can not create a Path from an empty string
- Persist non-partitioned orc relation into metastore as managed table using CTAS *** FAILED *** (16 milliseconds)
java.lang.IllegalArgumentException: Can not create a Path from an empty string
```
```
HiveUDFSuite:
- SPARK-11522 select input_file_name from non-parquet table *** FAILED *** (16 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
```
```
QueryPartitionSuite:
- SPARK-13709: reading partitioned Avro table with nested schema *** FAILED *** (250 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
```
```
ParquetHiveCompatibilitySuite:
- simple primitives *** FAILED *** (16 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- SPARK-10177 timestamp *** FAILED *** (0 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- array *** FAILED *** (16 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- map *** FAILED *** (16 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- struct *** FAILED *** (0 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
- SPARK-16344: array of struct with a single field named 'array_element' *** FAILED *** (15 milliseconds)
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string);
```
## How was this patch tested?
Manually tested via AppVeyor.
```
ColumnExpressionSuite:
- input_file_name, input_file_block_start, input_file_block_length - FileScanRDD (234 milliseconds)
- input_file_name, input_file_block_start, input_file_block_length - HadoopRDD (235 milliseconds)
- input_file_name, input_file_block_start, input_file_block_length - NewHadoopRDD (203 milliseconds)
```
```
DataStreamReaderWriterSuite:
- source metadataPath (63 milliseconds)
```
```
GlobalTempViewSuite:
- CREATE GLOBAL TEMP VIEW USING (436 milliseconds)
```
```
CreateTableAsSelectSuite:
- CREATE TABLE USING AS SELECT (171 milliseconds)
- create a table, drop it and create another one with the same name (422 milliseconds)
- create table using as select - with partitioned by (141 milliseconds)
- create table using as select - with non-zero buckets (125 milliseconds)
```
```
HiveMetadataCacheSuite:
- partitioned table is cached when partition pruning is true (3 seconds, 211 milliseconds)
- partitioned table is cached when partition pruning is false (1 second, 781 milliseconds)
```
```
MultiDatabaseSuite:
- createExternalTable() to non-default database - with USE (797 milliseconds)
- createExternalTable() to non-default database - without USE (640 milliseconds)
- invalid database name and table names (62 milliseconds)
```
```
OrcQuerySuite:
- SPARK-8501: Avoids discovery schema from empty ORC files (703 milliseconds)
- Verify the ORC conversion parameter: CONVERT_METASTORE_ORC (750 milliseconds)
- converted ORC table supports resolving mixed case field (625 milliseconds)
```
```
HadoopFsRelationTest - JsonHadoopFsRelationSuite, OrcHadoopFsRelationSuite, ParquetHadoopFsRelationSuite, SimpleTextHadoopFsRelationSuite:
- Locality support for FileScanRDD (296 milliseconds)
```
```
HiveQuerySuite:
- CREATE TEMPORARY FUNCTION (125 milliseconds)
- ADD FILE command (250 milliseconds)
- ADD JAR command 2 (609 milliseconds)
```
```
PruneFileSourcePartitionsSuite:
- PruneFileSourcePartitions should not change the output of LogicalRelation (359 milliseconds)
```
```
HiveCommandSuite:
- LOAD DATA LOCAL (1 second, 829 milliseconds)
- LOAD DATA (1 second, 735 milliseconds)
- Truncate Table (1 second, 641 milliseconds)
```
```
HiveExternalCatalogBackwardCompatibilitySuite:
- make sure we can read table created by old version of Spark (32 milliseconds)
- make sure we can alter table location created by old version of Spark (125 milliseconds)
- make sure we can rename table created by old version of Spark (281 milliseconds)
```
```
ExternalCatalogSuite:
- create/drop/rename partitions should create/delete/rename the directory (625 milliseconds)
```
```
SQLQuerySuite:
- describe functions - temporary user defined functions (31 milliseconds)
- specifying database name for a temporary table is not allowed (390 milliseconds)
```
```
PartitionProviderCompatibilitySuite:
- convert partition provider to hive with repair table (813 milliseconds)
- when partition management is enabled, new tables have partition provider hive (562 milliseconds)
- when partition management is disabled, new tables have no partition provider (344 milliseconds)
- when partition management is disabled, we preserve the old behavior even for new tables (422 milliseconds)
- insert overwrite partition of legacy datasource table (750 milliseconds)
- SPARK-18544 append with saveAsTable - partition management true (985 milliseconds)
- SPARK-18635 special chars in partition values - partition management true (3 seconds, 328 milliseconds)
- SPARK-18635 special chars in partition values - partition management false (2 seconds, 891 milliseconds)
- SPARK-18659 insert overwrite table with lowercase - partition management true (750 milliseconds)
- SPARK-18544 append with saveAsTable - partition management false (656 milliseconds)
- SPARK-18659 insert overwrite table files - partition management false (922 milliseconds)
- SPARK-18659 insert overwrite table with lowercase - partition management false (469 milliseconds)
- sanity check table setup (937 milliseconds)
- insert into partial dynamic partitions (2 seconds, 985 milliseconds)
- insert into fully dynamic partitions (1 second, 937 milliseconds)
- insert into static partition (1 second, 578 milliseconds)
- overwrite partial dynamic partitions (7 seconds, 561 milliseconds)
- overwrite fully dynamic partitions (1 second, 766 milliseconds)
- overwrite static partition (1 second, 797 milliseconds)
```
```
MetastoreDataSourcesSuite:
- check change without refresh (610 milliseconds)
- drop, change, recreate (437 milliseconds)
- SPARK-15269 external data source table creation (297 milliseconds)
- CTAS with IF NOT EXISTS (437 milliseconds)
- CTAS: persisted partitioned bucketed data source table (422 milliseconds)
- SPARK-15025: create datasource table with path with select (265 milliseconds)
- CTAS (438 milliseconds)
- CTAS with IF NOT EXISTS (469 milliseconds)
- CTAS: persisted partitioned bucketed data source table (406 milliseconds)
```
```
HiveMetastoreCatalogSuite:
- Persist non-partitioned parquet relation into metastore as managed table using CTAS (406 milliseconds)
- Persist non-partitioned orc relation into metastore as managed table using CTAS (313 milliseconds)
```
```
HiveUDFSuite:
- SPARK-11522 select input_file_name from non-parquet table (3 seconds, 144 milliseconds)
```
```
QueryPartitionSuite:
- SPARK-13709: reading partitioned Avro table with nested schema (1 second, 67 milliseconds)
```
```
ParquetHiveCompatibilitySuite:
- simple primitives (745 milliseconds)
- SPARK-10177 timestamp (375 milliseconds)
- array (407 milliseconds)
- map (409 milliseconds)
- struct (437 milliseconds)
- SPARK-16344: array of struct with a single field named 'array_element' (391 milliseconds)
```
Author: hyukjinkwon <gu...@gmail.com>
Closes #16397 from HyukjinKwon/SPARK-18922-paths.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/852782b8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/852782b8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/852782b8
Branch: refs/heads/master
Commit: 852782b83c89c358cf429e3913b71a1a6c44f27a
Parents: 56d3a7e
Author: hyukjinkwon <gu...@gmail.com>
Authored: Fri Dec 30 11:16:03 2016 +0000
Committer: Sean Owen <so...@cloudera.com>
Committed: Fri Dec 30 11:16:03 2016 +0000
----------------------------------------------------------------------
.../catalyst/catalog/ExternalCatalogSuite.scala | 4 +-
.../spark/sql/ColumnExpressionSuite.scala | 6 +--
.../sql/execution/GlobalTempViewSuite.scala | 2 +-
.../sql/sources/CreateTableAsSelectSuite.scala | 20 ++++-----
.../test/DataStreamReaderWriterSuite.scala | 9 ++--
...ernalCatalogBackwardCompatibilitySuite.scala | 45 ++++++++++++--------
.../spark/sql/hive/HiveMetadataCacheSuite.scala | 2 +-
.../sql/hive/HiveMetastoreCatalogSuite.scala | 4 +-
.../sql/hive/MetastoreDataSourcesSuite.scala | 24 ++++-------
.../spark/sql/hive/MultiDatabaseSuite.scala | 14 +++---
.../hive/ParquetHiveCompatibilitySuite.scala | 2 +-
.../PartitionProviderCompatibilitySuite.scala | 12 +++---
.../spark/sql/hive/QueryPartitionSuite.scala | 8 ++--
.../sql/hive/execution/HiveCommandSuite.scala | 25 +++++------
.../sql/hive/execution/HiveQuerySuite.scala | 10 ++---
.../spark/sql/hive/execution/HiveUDFSuite.scala | 6 +--
.../PruneFileSourcePartitionsSuite.scala | 2 +-
.../sql/hive/execution/SQLQuerySuite.scala | 8 ++--
.../spark/sql/hive/orc/OrcQuerySuite.scala | 6 +--
.../sql/sources/HadoopFsRelationTest.scala | 2 +-
20 files changed, 106 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
index 00e663c..742f900 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
@@ -751,7 +751,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac
val partWithExistingDir = CatalogTablePartition(
Map("partCol1" -> "7", "partCol2" -> "8"),
CatalogStorageFormat(
- Some(tempPath.getAbsolutePath),
+ Some(tempPath.toURI.toString),
None, None, None, false, Map.empty))
catalog.createPartitions("db1", "tbl", Seq(partWithExistingDir), ignoreIfExists = false)
@@ -760,7 +760,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac
val partWithNonExistingDir = CatalogTablePartition(
Map("partCol1" -> "9", "partCol2" -> "10"),
CatalogStorageFormat(
- Some(tempPath.getAbsolutePath),
+ Some(tempPath.toURI.toString),
None, None, None, false, Map.empty))
catalog.createPartitions("db1", "tbl", Seq(partWithNonExistingDir), ignoreIfExists = false)
assert(tempPath.exists())
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index b0339a8..ee280a3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -542,7 +542,7 @@ class ColumnExpressionSuite extends QueryTest with SharedSQLContext {
val q = spark.read.parquet(dir.getCanonicalPath).select(
input_file_name(), expr("input_file_block_start()"), expr("input_file_block_length()"))
val firstRow = q.head()
- assert(firstRow.getString(0).contains(dir.getCanonicalPath))
+ assert(firstRow.getString(0).contains(dir.toURI.getPath))
assert(firstRow.getLong(1) == 0)
assert(firstRow.getLong(2) > 0)
@@ -566,7 +566,7 @@ class ColumnExpressionSuite extends QueryTest with SharedSQLContext {
val q = df.select(
input_file_name(), expr("input_file_block_start()"), expr("input_file_block_length()"))
val firstRow = q.head()
- assert(firstRow.getString(0).contains(dir.getCanonicalPath))
+ assert(firstRow.getString(0).contains(dir.toURI.getPath))
assert(firstRow.getLong(1) == 0)
assert(firstRow.getLong(2) > 0)
@@ -595,7 +595,7 @@ class ColumnExpressionSuite extends QueryTest with SharedSQLContext {
val q = df.select(
input_file_name(), expr("input_file_block_start()"), expr("input_file_block_length()"))
val firstRow = q.head()
- assert(firstRow.getString(0).contains(dir.getCanonicalPath))
+ assert(firstRow.getString(0).contains(dir.toURI.getPath))
assert(firstRow.getLong(1) == 0)
assert(firstRow.getLong(2) > 0)
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala
index 391bcb8..5c63c6a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala
@@ -93,7 +93,7 @@ class GlobalTempViewSuite extends QueryTest with SharedSQLContext {
withTempPath { path =>
try {
Seq(1 -> "a").toDF("i", "j").write.parquet(path.getAbsolutePath)
- sql(s"CREATE GLOBAL TEMP VIEW src USING parquet OPTIONS (PATH '${path.getAbsolutePath}')")
+ sql(s"CREATE GLOBAL TEMP VIEW src USING parquet OPTIONS (PATH '${path.toURI}')")
checkAnswer(spark.table(s"$globalTempDB.src"), Row(1, "a"))
sql(s"INSERT INTO $globalTempDB.src SELECT 2, 'b'")
checkAnswer(spark.table(s"$globalTempDB.src"), Row(1, "a") :: Row(2, "b") :: Nil)
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
index 61939fe..99da196 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
@@ -70,7 +70,7 @@ class CreateTableAsSelectSuite
|CREATE TABLE jsonTable
|USING json
|OPTIONS (
- | path '${path.toString}'
+ | path '${path.toURI}'
|) AS
|SELECT a, b FROM jt
""".stripMargin)
@@ -94,7 +94,7 @@ class CreateTableAsSelectSuite
|CREATE TABLE jsonTable
|USING json
|OPTIONS (
- | path '${childPath.toString}'
+ | path '${childPath.toURI}'
|) AS
|SELECT a, b FROM jt
""".stripMargin)
@@ -112,7 +112,7 @@ class CreateTableAsSelectSuite
|CREATE TABLE jsonTable
|USING json
|OPTIONS (
- | path '${path.toString}'
+ | path '${path.toURI}'
|) AS
|SELECT a, b FROM jt
""".stripMargin)
@@ -127,7 +127,7 @@ class CreateTableAsSelectSuite
|CREATE TABLE IF NOT EXISTS jsonTable
|USING json
|OPTIONS (
- | path '${path.toString}'
+ | path '${path.toURI}'
|) AS
|SELECT a * 4 FROM jt
""".stripMargin)
@@ -145,7 +145,7 @@ class CreateTableAsSelectSuite
|CREATE TABLE jsonTable
|USING json
|OPTIONS (
- | path '${path.toString}'
+ | path '${path.toURI}'
|) AS
|SELECT b FROM jt
""".stripMargin)
@@ -162,7 +162,7 @@ class CreateTableAsSelectSuite
sql(
s"""
|CREATE TEMPORARY TABLE t USING PARQUET
- |OPTIONS (PATH '${path.toString}')
+ |OPTIONS (PATH '${path.toURI}')
|PARTITIONED BY (a)
|AS SELECT 1 AS a, 2 AS b
""".stripMargin
@@ -179,7 +179,7 @@ class CreateTableAsSelectSuite
sql(
s"""
|CREATE EXTERNAL TABLE t USING PARQUET
- |OPTIONS (PATH '${path.toString}')
+ |OPTIONS (PATH '${path.toURI}')
|AS SELECT 1 AS a, 2 AS b
""".stripMargin
)
@@ -196,7 +196,7 @@ class CreateTableAsSelectSuite
sql(
s"""
|CREATE TABLE t USING PARQUET
- |OPTIONS (PATH '${path.toString}')
+ |OPTIONS (PATH '${path.toURI}')
|PARTITIONED BY (a)
|AS SELECT 1 AS a, 2 AS b
""".stripMargin
@@ -212,7 +212,7 @@ class CreateTableAsSelectSuite
sql(
s"""
|CREATE TABLE t USING PARQUET
- |OPTIONS (PATH '${path.toString}')
+ |OPTIONS (PATH '${path.toURI}')
|CLUSTERED BY (a) SORTED BY (b) INTO 5 BUCKETS
|AS SELECT 1 AS a, 2 AS b
""".stripMargin
@@ -228,7 +228,7 @@ class CreateTableAsSelectSuite
sql(
s"""
|CREATE TABLE t USING PARQUET
- |OPTIONS (PATH '${path.toString}')
+ |OPTIONS (PATH '${path.toURI}')
|CLUSTERED BY (a) SORTED BY (b) INTO 0 BUCKETS
|AS SELECT 1 AS a, 2 AS b
""".stripMargin
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
index 097dd6e..30a957e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit
import scala.concurrent.duration._
+import org.apache.hadoop.fs.Path
import org.mockito.Mockito._
import org.scalatest.{BeforeAndAfter, PrivateMethodTester}
import org.scalatest.PrivateMethodTester.PrivateMethod
@@ -355,7 +356,7 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter with Pr
test("source metadataPath") {
LastOptions.clear()
- val checkpointLocation = newMetadataDir
+ val checkpointLocationURI = new Path(newMetadataDir).toUri
val df1 = spark.readStream
.format("org.apache.spark.sql.streaming.test")
@@ -367,21 +368,21 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter with Pr
val q = df1.union(df2).writeStream
.format("org.apache.spark.sql.streaming.test")
- .option("checkpointLocation", checkpointLocation)
+ .option("checkpointLocation", checkpointLocationURI.toString)
.trigger(ProcessingTime(10.seconds))
.start()
q.stop()
verify(LastOptions.mockStreamSourceProvider).createSource(
spark.sqlContext,
- checkpointLocation + "/sources/0",
+ s"$checkpointLocationURI/sources/0",
None,
"org.apache.spark.sql.streaming.test",
Map.empty)
verify(LastOptions.mockStreamSourceProvider).createSource(
spark.sqlContext,
- checkpointLocation + "/sources/1",
+ s"$checkpointLocationURI/sources/1",
None,
"org.apache.spark.sql.streaming.test",
Map.empty)
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala
index c5753ce..00fdfbc 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.hive
import java.net.URI
+import org.apache.hadoop.fs.Path
import org.scalatest.BeforeAndAfterEach
import org.apache.spark.sql.QueryTest
@@ -39,6 +40,7 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
val tempDir = Utils.createTempDir().getCanonicalFile
+ val tempDirUri = tempDir.toURI.toString.stripSuffix("/")
override def beforeEach(): Unit = {
sql("CREATE DATABASE test_db")
@@ -56,11 +58,12 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
spark.sharedState.externalCatalog.getTable("test_db", tableName)
}
- private def defaultTablePath(tableName: String): String = {
- spark.sessionState.catalog.defaultTablePath(TableIdentifier(tableName, Some("test_db")))
+ private def defaultTableURI(tableName: String): URI = {
+ val defaultPath =
+ spark.sessionState.catalog.defaultTablePath(TableIdentifier(tableName, Some("test_db")))
+ new Path(defaultPath).toUri
}
-
// Raw table metadata that are dumped from tables created by Spark 2.0. Note that, all spark
// versions prior to 2.1 would generate almost same raw table metadata for a specific table.
val simpleSchema = new StructType().add("i", "int")
@@ -78,7 +81,7 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
identifier = TableIdentifier("tbl2", Some("test_db")),
tableType = CatalogTableType.EXTERNAL,
storage = CatalogStorageFormat.empty.copy(
- locationUri = Some(tempDir.getAbsolutePath),
+ locationUri = Some(tempDirUri),
inputFormat = Some("org.apache.hadoop.mapred.TextInputFormat"),
outputFormat = Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat")),
schema = simpleSchema)
@@ -128,7 +131,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
lazy val dataSourceTable = CatalogTable(
identifier = TableIdentifier("tbl4", Some("test_db")),
tableType = CatalogTableType.MANAGED,
- storage = CatalogStorageFormat.empty.copy(properties = Map("path" -> defaultTablePath("tbl4"))),
+ storage = CatalogStorageFormat.empty.copy(
+ properties = Map("path" -> defaultTableURI("tbl4").toString)),
schema = new StructType(),
properties = Map(
"spark.sql.sources.provider" -> "json",
@@ -138,7 +142,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
lazy val hiveCompatibleDataSourceTable = CatalogTable(
identifier = TableIdentifier("tbl5", Some("test_db")),
tableType = CatalogTableType.MANAGED,
- storage = CatalogStorageFormat.empty.copy(properties = Map("path" -> defaultTablePath("tbl5"))),
+ storage = CatalogStorageFormat.empty.copy(
+ properties = Map("path" -> defaultTableURI("tbl5").toString)),
schema = simpleSchema,
properties = Map(
"spark.sql.sources.provider" -> "parquet",
@@ -148,7 +153,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
lazy val partitionedDataSourceTable = CatalogTable(
identifier = TableIdentifier("tbl6", Some("test_db")),
tableType = CatalogTableType.MANAGED,
- storage = CatalogStorageFormat.empty.copy(properties = Map("path" -> defaultTablePath("tbl6"))),
+ storage = CatalogStorageFormat.empty.copy(
+ properties = Map("path" -> defaultTableURI("tbl6").toString)),
schema = new StructType(),
properties = Map(
"spark.sql.sources.provider" -> "json",
@@ -161,8 +167,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
identifier = TableIdentifier("tbl7", Some("test_db")),
tableType = CatalogTableType.EXTERNAL,
storage = CatalogStorageFormat.empty.copy(
- locationUri = Some(defaultTablePath("tbl7") + "-__PLACEHOLDER__"),
- properties = Map("path" -> tempDir.getAbsolutePath)),
+ locationUri = Some(defaultTableURI("tbl7").toString + "-__PLACEHOLDER__"),
+ properties = Map("path" -> tempDirUri)),
schema = new StructType(),
properties = Map(
"spark.sql.sources.provider" -> "json",
@@ -173,8 +179,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
identifier = TableIdentifier("tbl8", Some("test_db")),
tableType = CatalogTableType.EXTERNAL,
storage = CatalogStorageFormat.empty.copy(
- locationUri = Some(tempDir.getAbsolutePath),
- properties = Map("path" -> tempDir.getAbsolutePath)),
+ locationUri = Some(tempDirUri),
+ properties = Map("path" -> tempDirUri)),
schema = simpleSchema,
properties = Map(
"spark.sql.sources.provider" -> "parquet",
@@ -185,8 +191,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
identifier = TableIdentifier("tbl9", Some("test_db")),
tableType = CatalogTableType.EXTERNAL,
storage = CatalogStorageFormat.empty.copy(
- locationUri = Some(defaultTablePath("tbl9") + "-__PLACEHOLDER__"),
- properties = Map("path" -> tempDir.getAbsolutePath)),
+ locationUri = Some(defaultTableURI("tbl9").toString + "-__PLACEHOLDER__"),
+ properties = Map("path" -> tempDirUri)),
schema = new StructType(),
properties = Map("spark.sql.sources.provider" -> "json"))
@@ -210,7 +216,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
if (tbl.tableType == CatalogTableType.EXTERNAL) {
// trim the URI prefix
val tableLocation = new URI(readBack.storage.locationUri.get).getPath
- assert(tableLocation == tempDir.getAbsolutePath)
+ val expectedLocation = tempDir.toURI.getPath.stripSuffix("/")
+ assert(tableLocation == expectedLocation)
}
}
}
@@ -218,13 +225,15 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
test("make sure we can alter table location created by old version of Spark") {
withTempDir { dir =>
for ((tbl, _) <- rawTablesAndExpectations if tbl.tableType == CatalogTableType.EXTERNAL) {
- sql(s"ALTER TABLE ${tbl.identifier} SET LOCATION '${dir.getAbsolutePath}'")
+ val path = dir.toURI.toString.stripSuffix("/")
+ sql(s"ALTER TABLE ${tbl.identifier} SET LOCATION '$path'")
val readBack = getTableMetadata(tbl.identifier.table)
// trim the URI prefix
val actualTableLocation = new URI(readBack.storage.locationUri.get).getPath
- assert(actualTableLocation == dir.getAbsolutePath)
+ val expected = dir.toURI.getPath.stripSuffix("/")
+ assert(actualTableLocation == expected)
}
}
}
@@ -240,10 +249,10 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
// trim the URI prefix
val actualTableLocation = new URI(readBack.storage.locationUri.get).getPath
val expectedLocation = if (tbl.tableType == CatalogTableType.EXTERNAL) {
- tempDir.getAbsolutePath
+ tempDir.toURI.getPath.stripSuffix("/")
} else {
// trim the URI prefix
- new URI(defaultTablePath(newName)).getPath
+ defaultTableURI(newName).getPath
}
assert(actualTableLocation == expectedLocation)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
index 6e887d9..0c28a1b 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
@@ -75,7 +75,7 @@ class HiveMetadataCacheSuite extends QueryTest with SQLTestUtils with TestHiveSi
|create external table test (id long)
|partitioned by (f1 int, f2 int)
|stored as parquet
- |location "${dir.getAbsolutePath}"""".stripMargin)
+ |location "${dir.toURI}"""".stripMargin)
spark.sql("msck repair table test")
val df = spark.sql("select * from test")
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index 0a280b4..16cf4d7 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -156,11 +156,9 @@ class DataSourceWithHiveMetastoreCatalogSuite
test(s"Persist non-partitioned $provider relation into metastore as managed table using CTAS") {
withTempPath { dir =>
withTable("t") {
- val path = dir.getCanonicalPath
-
sql(
s"""CREATE TABLE t USING $provider
- |OPTIONS (path '$path')
+ |OPTIONS (path '${dir.toURI}')
|AS SELECT 1 AS d1, "val_1" AS d2
""".stripMargin)
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
index 0f787be..2b8d4e2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
@@ -179,7 +179,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
s"""CREATE TABLE jsonTable
|USING org.apache.spark.sql.json
|OPTIONS (
- | path '${tempDir.getCanonicalPath}'
+ | path '${tempDir.toURI}'
|)
""".stripMargin)
@@ -215,7 +215,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
s"""CREATE TABLE jsonTable
|USING org.apache.spark.sql.json
|OPTIONS (
- | path '${tempDir.getCanonicalPath}'
+ | path '${tempDir.toURI}'
|)
""".stripMargin)
@@ -232,7 +232,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
s"""CREATE TABLE jsonTable
|USING org.apache.spark.sql.json
|OPTIONS (
- | path '${tempDir.getCanonicalPath}'
+ | path '${tempDir.toURI}'
|)
""".stripMargin)
@@ -291,7 +291,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
s"""CREATE TABLE ctasJsonTable
|USING org.apache.spark.sql.json.DefaultSource
|OPTIONS (
- | path '$tempPath'
+ | path '${tempPath.toURI}'
|) AS
|SELECT * FROM jsonTable
""".stripMargin)
@@ -307,7 +307,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
test("CTAS with IF NOT EXISTS") {
withTempPath { path =>
- val tempPath = path.getCanonicalPath
+ val tempPath = path.toURI
withTable("jsonTable", "ctasJsonTable") {
sql(
@@ -1049,11 +1049,9 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
test("CTAS: persisted partitioned data source table") {
withTempPath { dir =>
withTable("t") {
- val path = dir.getCanonicalPath
-
sql(
s"""CREATE TABLE t USING PARQUET
- |OPTIONS (PATH '$path')
+ |OPTIONS (PATH '${dir.toURI}')
|PARTITIONED BY (a)
|AS SELECT 1 AS a, 2 AS b
""".stripMargin
@@ -1119,11 +1117,9 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
test("CTAS: persisted partitioned bucketed data source table") {
withTempPath { dir =>
withTable("t") {
- val path = dir.getCanonicalPath
-
sql(
s"""CREATE TABLE t USING PARQUET
- |OPTIONS (PATH '$path')
+ |OPTIONS (PATH '${dir.toURI}')
|PARTITIONED BY (a)
|CLUSTERED BY (b) SORTED BY (c) INTO 2 BUCKETS
|AS SELECT 1 AS a, 2 AS b, 3 AS c
@@ -1289,11 +1285,9 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
test("SPARK-15025: create datasource table with path with select") {
withTempPath { dir =>
withTable("t") {
- val path = dir.getCanonicalPath
-
sql(
s"""CREATE TABLE t USING PARQUET
- |OPTIONS (PATH '$path')
+ |OPTIONS (PATH '${dir.toURI}')
|AS SELECT 1 AS a, 2 AS b, 3 AS c
""".stripMargin
)
@@ -1307,7 +1301,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
test("SPARK-15269 external data source table creation") {
withTempPath { dir =>
- val path = dir.getCanonicalPath
+ val path = dir.toURI.toString
spark.range(1).write.json(path)
withTable("t") {
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala
index 7322465..47ee4dd 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala
@@ -80,7 +80,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle
|CREATE TABLE t1
|USING parquet
|OPTIONS (
- | path '$path'
+ | path '${dir.toURI}'
|)
""".stripMargin)
assert(getTableNames(Option(db)).contains("t1"))
@@ -105,7 +105,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle
|CREATE TABLE $db.t1
|USING parquet
|OPTIONS (
- | path '$path'
+ | path '${dir.toURI}'
|)
""".stripMargin)
assert(getTableNames(Option(db)).contains("t1"))
@@ -212,7 +212,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle
s"""CREATE EXTERNAL TABLE t (id BIGINT)
|PARTITIONED BY (p INT)
|STORED AS PARQUET
- |LOCATION '$path'
+ |LOCATION '${dir.toURI}'
""".stripMargin)
checkAnswer(spark.table("t"), spark.emptyDataFrame)
@@ -244,7 +244,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle
s"""CREATE EXTERNAL TABLE $db.t (id BIGINT)
|PARTITIONED BY (p INT)
|STORED AS PARQUET
- |LOCATION '$path'
+ |LOCATION '${dir.toURI}'
""".stripMargin)
checkAnswer(spark.table(s"$db.t"), spark.emptyDataFrame)
@@ -280,8 +280,6 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle
}
withTempDir { dir =>
- val path = dir.getCanonicalPath
-
{
val message = intercept[AnalysisException] {
sql(
@@ -289,7 +287,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle
|CREATE TABLE `d:b`.`t:a` (a int)
|USING parquet
|OPTIONS (
- | path '$path'
+ | path '${dir.toURI}'
|)
""".stripMargin)
}.getMessage
@@ -304,7 +302,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle
|CREATE TABLE `d:b`.`table` (a int)
|USING parquet
|OPTIONS (
- | path '$path'
+ | path '${dir.toURI}'
|)
""".stripMargin)
}.getMessage
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetHiveCompatibilitySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetHiveCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetHiveCompatibilitySuite.scala
index 14266e6..05b6059 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetHiveCompatibilitySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetHiveCompatibilitySuite.scala
@@ -45,7 +45,7 @@ class ParquetHiveCompatibilitySuite extends ParquetCompatibilityTest with TestHi
private def testParquetHiveCompatibility(row: Row, hiveTypes: String*): Unit = {
withTable("parquet_compat") {
withTempPath { dir =>
- val path = dir.getCanonicalPath
+ val path = dir.toURI.toString
// Hive columns are always nullable, so here we append a all-null row.
val rows = row :: Row(Seq.fill(row.length)(null): _*) :: Nil
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
index 3f84cbd..f88fc4a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
@@ -38,7 +38,7 @@ class PartitionProviderCompatibilitySuite
spark.sql(s"""
|create table $tableName (fieldOne long, partCol int)
|using parquet
- |options (path "${dir.getAbsolutePath}")
+ |options (path "${dir.toURI}")
|partitioned by (partCol)""".stripMargin)
}
@@ -239,7 +239,7 @@ class PartitionProviderCompatibilitySuite
// custom locations sanity check
spark.sql(s"""
|alter table test partition (A=0, B='%')
- |set location '${dir.getAbsolutePath}'""".stripMargin)
+ |set location '${dir.toURI}'""".stripMargin)
assert(spark.sql("select * from test").count() == 28) // moved to empty dir
// rename partition sanity check
@@ -315,11 +315,11 @@ class PartitionProviderCompatibilitySuite
spark.sql(s"""
|create table test (id long, P1 int, P2 int)
|using parquet
- |options (path "${base.getAbsolutePath}")
+ |options (path "${base.toURI}")
|partitioned by (P1, P2)""".stripMargin)
- spark.sql(s"alter table test add partition (P1=0, P2=0) location '${a.getAbsolutePath}'")
- spark.sql(s"alter table test add partition (P1=0, P2=1) location '${b.getAbsolutePath}'")
- spark.sql(s"alter table test add partition (P1=1, P2=0) location '${c.getAbsolutePath}'")
+ spark.sql(s"alter table test add partition (P1=0, P2=0) location '${a.toURI}'")
+ spark.sql(s"alter table test add partition (P1=0, P2=1) location '${b.toURI}'")
+ spark.sql(s"alter table test add partition (P1=1, P2=0) location '${c.toURI}'")
spark.sql(s"alter table test add partition (P1=1, P2=1)")
testFn
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala
index feeaade..b20c10c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala
@@ -40,7 +40,7 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl
val tmpDir = Files.createTempDir()
// create the table for test
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
- s"PARTITIONED by (ds string) location '${tmpDir.toURI.toString}' ")
+ s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
@@ -71,12 +71,12 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl
test("SPARK-13709: reading partitioned Avro table with nested schema") {
withTempDir { dir =>
- val path = dir.getCanonicalPath
+ val path = dir.toURI.toString
val tableName = "spark_13709"
val tempTableName = "spark_13709_temp"
- new File(path, tableName).mkdir()
- new File(path, tempTableName).mkdir()
+ new File(dir.getAbsolutePath, tableName).mkdir()
+ new File(dir.getAbsolutePath, tempTableName).mkdir()
val avroSchema =
"""{
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
index 1680f6c..8fda1c5 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
@@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.StructType
@@ -202,11 +201,12 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
// LOAD DATA INTO non-partitioned table can't specify partition
intercept[AnalysisException] {
- sql(s"""$loadQuery INPATH "$testData" INTO TABLE non_part_table PARTITION(ds="1")""")
+ sql(
+ s"""$loadQuery INPATH "${testData.toURI}" INTO TABLE non_part_table PARTITION(ds="1")""")
}
withInputFile { path =>
- sql(s"""$loadQuery INPATH "$path" INTO TABLE non_part_table""")
+ sql(s"""$loadQuery INPATH "${path.toURI}" INTO TABLE non_part_table""")
// Non-local mode is expected to move the file, while local mode is expected to copy it.
// Check once here that the behavior is the expected.
@@ -222,7 +222,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
//
// TODO: need a similar test for non-local mode.
if (local) {
- val incorrectUri = "file:/" + testData.getAbsolutePath()
+ val incorrectUri = "file://path/to/data/files/employee.dat"
intercept[AnalysisException] {
sql(s"""LOAD DATA LOCAL INPATH "$incorrectUri" INTO TABLE non_part_table""")
}
@@ -231,7 +231,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
// Use URI as inpath:
// file:/path/to/data/files/employee.dat
withInputFile { path =>
- sql(s"""$loadQuery INPATH "${path.toURI()}" INTO TABLE non_part_table""")
+ sql(s"""$loadQuery INPATH "${path.toURI}" INTO TABLE non_part_table""")
}
checkAnswer(
@@ -240,7 +240,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
// Overwrite existing data.
withInputFile { path =>
- sql(s"""$loadQuery INPATH "${path.toURI()}" OVERWRITE INTO TABLE non_part_table""")
+ sql(s"""$loadQuery INPATH "${path.toURI}" OVERWRITE INTO TABLE non_part_table""")
}
checkAnswer(
@@ -257,7 +257,8 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
""".stripMargin)
// LOAD DATA INTO partitioned table must specify partition
- withInputFile { path =>
+ withInputFile { f =>
+ val path = f.toURI
intercept[AnalysisException] {
sql(s"""$loadQuery INPATH "$path" INTO TABLE part_table""")
}
@@ -273,16 +274,16 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
}
}
- withInputFile { path =>
- sql(s"""$loadQuery INPATH "$path" INTO TABLE part_table PARTITION(c="1", d="2")""")
+ withInputFile { f =>
+ sql(s"""$loadQuery INPATH "${f.toURI}" INTO TABLE part_table PARTITION(c="1", d="2")""")
}
checkAnswer(
sql("SELECT employeeID, employeeName FROM part_table WHERE c = '1' AND d = '2'"),
sql("SELECT * FROM non_part_table").collect())
// Different order of partition columns.
- withInputFile { path =>
- sql(s"""$loadQuery INPATH "$path" INTO TABLE part_table PARTITION(d="1", c="2")""")
+ withInputFile { f =>
+ sql(s"""$loadQuery INPATH "${f.toURI}" INTO TABLE part_table PARTITION(d="1", c="2")""")
}
checkAnswer(
sql("SELECT employeeID, employeeName FROM part_table WHERE c = '2' AND d = '1'"),
@@ -300,7 +301,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
|LINES TERMINATED BY '\n'
""".stripMargin)
- val testData = hiveContext.getHiveFile("data/files/employee.dat").getCanonicalPath
+ val testData = hiveContext.getHiveFile("data/files/employee.dat").toURI
sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE non_part_table""")
checkAnswer(
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 6fbbed1..e5b23da 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -858,8 +858,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
test("ADD JAR command 2") {
// this is a test case from mapjoin_addjar.q
- val testJar = TestHive.getHiveFile("hive-hcatalog-core-0.13.1.jar").getCanonicalPath
- val testData = TestHive.getHiveFile("data/files/sample.json").getCanonicalPath
+ val testJar = TestHive.getHiveFile("hive-hcatalog-core-0.13.1.jar").toURI
+ val testData = TestHive.getHiveFile("data/files/sample.json").toURI
sql(s"ADD JAR $testJar")
sql(
"""CREATE TABLE t1(a string, b string)
@@ -877,8 +877,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
}
test("CREATE TEMPORARY FUNCTION") {
- val funcJar = TestHive.getHiveFile("TestUDTF.jar").getCanonicalPath
- val jarURL = s"file://$funcJar"
+ val funcJar = TestHive.getHiveFile("TestUDTF.jar")
+ val jarURL = funcJar.toURI.toURL
sql(s"ADD JAR $jarURL")
sql(
"""CREATE TEMPORARY FUNCTION udtf_count2 AS
@@ -889,7 +889,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
}
test("ADD FILE command") {
- val testFile = TestHive.getHiveFile("data/files/v1.txt").getCanonicalFile
+ val testFile = TestHive.getHiveFile("data/files/v1.txt").toURI
sql(s"ADD FILE $testFile")
val checkAddFileRDD = sparkContext.parallelize(1 to 2, 1).mapPartitions { _ =>
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 4098bb5..58909ab 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -428,7 +428,7 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
\"separatorChar\" = \",\",
\"quoteChar\" = \"\\\"\",
\"escapeChar\" = \"\\\\\")
- LOCATION '$tempDir'
+ LOCATION '${tempDir.toURI}'
""")
val answer1 =
@@ -444,7 +444,7 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
sql(
s"""CREATE EXTERNAL TABLE external_t5 (c1 int, c2 int)
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
- LOCATION '$tempDir'
+ LOCATION '${tempDir.toURI}'
""")
val answer2 =
@@ -460,7 +460,7 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
// External parquet pointing to LOCATION
- val parquetLocation = tempDir + "/external_parquet"
+ val parquetLocation = s"${tempDir.toURI}/external_parquet"
sql("SELECT 1, 2").write.parquet(parquetLocation)
sql(
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
index cdbc26c..cd8f94b 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
@@ -42,7 +42,7 @@ class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with Te
|CREATE EXTERNAL TABLE test(i int)
|PARTITIONED BY (p int)
|STORED AS parquet
- |LOCATION '${dir.getAbsolutePath}'""".stripMargin)
+ |LOCATION '${dir.toURI}'""".stripMargin)
val tableMeta = spark.sharedState.externalCatalog.getTable("default", "test")
val catalogFileIndex = new CatalogFileIndex(spark, tableMeta, 0)
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index e607af6..f65b5f4 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -346,7 +346,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
s"""
|CREATE TEMPORARY FUNCTION udtf_count_temp
|AS 'org.apache.spark.sql.hive.execution.GenericUDTFCount2'
- |USING JAR '${hiveContext.getHiveFile("TestUDTF.jar").getCanonicalPath()}'
+ |USING JAR '${hiveContext.getHiveFile("TestUDTF.jar").toURI}'
""".stripMargin)
checkKeywordsExist(sql("describe function udtf_count_temp"),
@@ -1290,7 +1290,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
test("specifying database name for a temporary table is not allowed") {
withTempPath { dir =>
- val path = dir.getCanonicalPath
+ val path = dir.toURI.toString
val df = sparkContext.parallelize(1 to 10).map(i => (i, i.toString)).toDF("num", "str")
df
.write
@@ -1298,7 +1298,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
.save(path)
// We don't support creating a temporary table while specifying a database
- val message = intercept[AnalysisException] {
+ intercept[AnalysisException] {
spark.sql(
s"""
|CREATE TEMPORARY TABLE db.t
@@ -1307,7 +1307,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
| path '$path'
|)
""".stripMargin)
- }.getMessage
+ }
// If you use backticks to quote the name then it's OK.
spark.sql(
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala
index b8761e9..9fa1fb9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala
@@ -350,7 +350,7 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest {
spark.sql(
s"""CREATE TABLE empty_orc(key INT, value STRING)
|STORED AS ORC
- |LOCATION '$path'
+ |LOCATION '${dir.toURI}'
""".stripMargin)
val emptyDF = Seq.empty[(Int, String)].toDF("key", "value").coalesce(1)
@@ -451,7 +451,7 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest {
s"""
|CREATE TABLE dummy_orc(key INT, value STRING)
|STORED AS ORC
- |LOCATION '$path'
+ |LOCATION '${dir.toURI}'
""".stripMargin)
spark.sql(
@@ -500,7 +500,7 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest {
|create external table dummy_orc (id long, valueField long)
|partitioned by (partitionValue int)
|stored as orc
- |location "${dir.getAbsolutePath}"""".stripMargin)
+ |location "${dir.toURI}"""".stripMargin)
spark.sql(s"msck repair table dummy_orc")
checkAnswer(spark.sql("select * from dummy_orc"), df)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/852782b8/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
index 06566a9..2446bed 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
@@ -847,7 +847,7 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes
"fs.file.impl.disable.cache" -> "true"
)
withTempPath { dir =>
- val path = "file://" + dir.getCanonicalPath
+ val path = dir.toURI.toString
val df1 = spark.range(4)
df1.coalesce(1).write.mode("overwrite").options(options).format(dataSourceName).save(path)
df1.coalesce(1).write.mode("append").options(options).format(dataSourceName).save(path)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org