You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2020/10/22 07:11:13 UTC
[GitHub] [spark] dongjoon-hyun commented on a change in pull request #30101: [SPARK-33193][SQL][TEST] Hive ThriftServer JDBC Database MetaData API Behavior Auditing
dongjoon-hyun commented on a change in pull request #30101:
URL: https://github.com/apache/spark/pull/30101#discussion_r509928449
##########
File path: sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala
##########
@@ -396,4 +400,203 @@ class SparkMetadataOperationSuite extends HiveThriftJdbcTest {
}
}
}
+
+ test("Hive ThriftServer JDBC Database MetaData API Auditing - Method not supported") {
+ // These APIs belong to the upstream Apache Hive's hive-jdbc artifact where defines the hive
+ // behavior. Users can also use it to interact with Spark ThriftServer directly. Some behaviors
+ // are not fully consistent with Spark e.g. we support correlated subqueries but the hive-jdbc
+ // now fail directly at client side. There is nothing we can do but accept the current
+ // condition and highlight the difference and make it perspective in future changes both from
+ // upstream and inside Spark.
+ withJdbcStatement() { statement =>
+ val metaData = statement.getConnection.getMetaData
+ Seq(
+ () => metaData.allProceduresAreCallable,
+ () => metaData.getURL,
+ () => metaData.getUserName,
+ () => metaData.isReadOnly,
+ () => metaData.nullsAreSortedHigh,
+ () => metaData.nullsAreSortedLow,
+ () => metaData.nullsAreSortedAtStart,
+ () => metaData.nullsAreSortedAtEnd,
+ () => metaData.usesLocalFiles,
+ () => metaData.usesLocalFilePerTable,
+ () => metaData.supportsMixedCaseIdentifiers,
+ () => metaData.supportsMixedCaseQuotedIdentifiers,
+ () => metaData.storesUpperCaseIdentifiers,
+ () => metaData.storesUpperCaseQuotedIdentifiers,
+ () => metaData.storesLowerCaseIdentifiers,
+ () => metaData.storesLowerCaseQuotedIdentifiers,
+ () => metaData.storesMixedCaseIdentifiers,
+ () => metaData.storesMixedCaseQuotedIdentifiers,
+ () => metaData.getSQLKeywords,
+ () => metaData.nullPlusNonNullIsNull,
+ () => metaData.supportsConvert,
+ () => metaData.supportsTableCorrelationNames,
+ () => metaData.supportsDifferentTableCorrelationNames,
+ () => metaData.supportsExpressionsInOrderBy,
+ () => metaData.supportsOrderByUnrelated,
+ () => metaData.supportsGroupByUnrelated,
+ () => metaData.supportsGroupByBeyondSelect,
+ () => metaData.supportsLikeEscapeClause,
+ () => metaData.supportsMultipleTransactions,
+ () => metaData.supportsMinimumSQLGrammar,
+ () => metaData.supportsCoreSQLGrammar,
+ () => metaData.supportsExtendedSQLGrammar,
+ () => metaData.supportsANSI92EntryLevelSQL,
+ () => metaData.supportsANSI92IntermediateSQL,
+ () => metaData.supportsANSI92FullSQL,
+ () => metaData.supportsIntegrityEnhancementFacility,
+ () => metaData.isCatalogAtStart,
+ () => metaData.supportsSubqueriesInComparisons,
+ () => metaData.supportsSubqueriesInExists,
+ () => metaData.supportsSubqueriesInIns,
+ () => metaData.supportsSubqueriesInQuantifieds,
+ // Spark support this, see https://issues.apache.org/jira/browse/SPARK-18455
+ () => metaData.supportsCorrelatedSubqueries,
+ () => metaData.supportsOpenCursorsAcrossCommit,
+ () => metaData.supportsOpenCursorsAcrossRollback,
+ () => metaData.supportsOpenStatementsAcrossCommit,
+ () => metaData.supportsOpenStatementsAcrossRollback,
+ () => metaData.getMaxBinaryLiteralLength,
+ () => metaData.getMaxCharLiteralLength,
+ () => metaData.getMaxColumnsInGroupBy,
+ () => metaData.getMaxColumnsInIndex,
+ () => metaData.getMaxColumnsInOrderBy,
+ () => metaData.getMaxColumnsInSelect,
+ () => metaData.getMaxColumnsInTable,
+ () => metaData.getMaxConnections,
+ () => metaData.getMaxCursorNameLength,
+ () => metaData.getMaxIndexLength,
+ () => metaData.getMaxSchemaNameLength,
+ () => metaData.getMaxProcedureNameLength,
+ () => metaData.getMaxCatalogNameLength,
+ () => metaData.getMaxRowSize,
+ () => metaData.doesMaxRowSizeIncludeBlobs,
+ () => metaData.getMaxStatementLength,
+ () => metaData.getMaxStatements,
+ () => metaData.getMaxTableNameLength,
+ () => metaData.getMaxTablesInSelect,
+ () => metaData.getMaxUserNameLength,
+ () => metaData.supportsTransactionIsolationLevel(1),
+ () => metaData.supportsDataDefinitionAndDataManipulationTransactions,
+ () => metaData.supportsDataManipulationTransactionsOnly,
+ () => metaData.dataDefinitionCausesTransactionCommit,
+ () => metaData.dataDefinitionIgnoredInTransactions,
+ () => metaData.getColumnPrivileges("", "%", "%", "%"),
+ () => metaData.getTablePrivileges("", "%", "%"),
+ () => metaData.getBestRowIdentifier("", "%", "%", 0, true),
+ () => metaData.getVersionColumns("", "%", "%"),
+ () => metaData.getExportedKeys("", "default", ""),
+ () => metaData.supportsResultSetConcurrency(ResultSet.TYPE_FORWARD_ONLY, 2),
+ () => metaData.ownUpdatesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+ () => metaData.ownDeletesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+ () => metaData.ownInsertsAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+ () => metaData.othersUpdatesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+ () => metaData.othersDeletesAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+ () => metaData.othersInsertsAreVisible(ResultSet.TYPE_FORWARD_ONLY),
+ () => metaData.updatesAreDetected(ResultSet.TYPE_FORWARD_ONLY),
+ () => metaData.deletesAreDetected(ResultSet.TYPE_FORWARD_ONLY),
+ () => metaData.insertsAreDetected(ResultSet.TYPE_FORWARD_ONLY),
+ () => metaData.supportsNamedParameters,
+ () => metaData.supportsMultipleOpenResults,
+ () => metaData.supportsGetGeneratedKeys,
+ () => metaData.getSuperTypes("", "%", "%"),
+ () => metaData.getSuperTables("", "%", "%"),
+ () => metaData.getAttributes("", "%", "%", "%"),
+ () => metaData.getResultSetHoldability,
+ () => metaData.locatorsUpdateCopy,
+ () => metaData.supportsStatementPooling,
+ () => metaData.getRowIdLifetime,
+ () => metaData.supportsStoredFunctionsUsingCallSyntax,
+ () => metaData.autoCommitFailureClosesAllResultSets,
+ () => metaData.getClientInfoProperties,
+ () => metaData.getFunctionColumns("", "%", "%", "%"),
+ () => metaData.getPseudoColumns("", "%", "%", "%"),
+ () => metaData.generatedKeyAlwaysReturned).foreach { func =>
+ val e = intercept[SQLFeatureNotSupportedException](func())
+ assert(e.getMessage === "Method not supported")
+ }
+ }
+ }
+
+ test("Hive ThriftServer JDBC Database MetaData API Auditing - Method supported") {
+ // These APIs belong to the upstream Apache Hive's hive-jdbc artifact where defines the hive
+ // behavior. Users can also use it to interact with Spark ThriftServer directly. Some behaviors
+ // are not fully consistent with Spark e.g. we can work with multiple catalogs.
+ // There is nothing we can do but accept the current condition and highlight the difference
+ // and make it perspective in future changes both from upstream and inside Spark.
+ withJdbcStatement() { statement =>
+ val metaData = statement.getConnection.getMetaData
+ import org.apache.spark.SPARK_VERSION
+ assert(metaData.allTablesAreSelectable)
+ assert(metaData.getDatabaseProductName === "Spark SQL")
+ assert(metaData.getDatabaseProductVersion === SPARK_VERSION)
+ assert(metaData.getDriverName === "Hive JDBC")
+ assert(metaData.getDriverVersion === HiveVersionInfo.getVersion)
+ assert(metaData.getDatabaseMajorVersion === VersionUtils.majorVersion(SPARK_VERSION))
+ assert(metaData.getDatabaseMinorVersion === VersionUtils.minorVersion(SPARK_VERSION))
+ assert(metaData.getIdentifierQuoteString === " ",
+ "This method returns a space \" \" if identifier quoting is not supported")
+ assert(metaData.getNumericFunctions === "")
+ assert(metaData.getStringFunctions === "")
+ assert(metaData.getSystemFunctions === "")
+ assert(metaData.getTimeDateFunctions === "")
+ assert(metaData.getSearchStringEscape === "\\")
+ assert(metaData.getExtraNameCharacters === "")
+ assert(metaData.supportsAlterTableWithAddColumn())
+ assert(!metaData.supportsAlterTableWithDropColumn())
+ assert(metaData.supportsColumnAliasing())
+ assert(metaData.supportsGroupBy)
+ assert(!metaData.supportsMultipleResultSets)
+ assert(!metaData.supportsNonNullableColumns)
+ assert(metaData.supportsOuterJoins)
+ assert(metaData.supportsFullOuterJoins)
+ assert(metaData.supportsLimitedOuterJoins)
+ assert(metaData.getSchemaTerm === "database")
+ assert(metaData.getProcedureTerm === "UDF")
+ assert(metaData.getCatalogTerm === "instance")
+ assert(metaData.getCatalogSeparator === ".")
+ assert(metaData.supportsSchemasInDataManipulation)
+ assert(!metaData.supportsSchemasInProcedureCalls)
+ assert(metaData.supportsSchemasInTableDefinitions)
+ assert(!metaData.supportsSchemasInIndexDefinitions)
+ assert(!metaData.supportsSchemasInPrivilegeDefinitions)
+ // This is actually supported, but hive jdbc package return false
+ assert(!metaData.supportsCatalogsInDataManipulation)
+ assert(!metaData.supportsCatalogsInProcedureCalls)
+ // This is actually supported, but hive jdbc package return false
+ assert(!metaData.supportsCatalogsInTableDefinitions)
+ assert(!metaData.supportsCatalogsInIndexDefinitions)
+ assert(!metaData.supportsCatalogsInPrivilegeDefinitions)
+ assert(!metaData.supportsPositionedDelete)
+ assert(!metaData.supportsPositionedUpdate)
+ assert(!metaData.supportsSelectForUpdate)
+ assert(!metaData.supportsStoredProcedures)
+ // This is actually supported, but hive jdbc package return false
+ assert(!metaData.supportsUnion)
+ assert(metaData.supportsUnionAll)
+ assert(metaData.getMaxColumnNameLength === 128)
+ assert(metaData.getDefaultTransactionIsolation === java.sql.Connection.TRANSACTION_NONE)
+ assert(!metaData.supportsTransactions)
+ assert(!metaData.getProcedureColumns("", "%", "%", "%").next())
+ assert(!metaData.getImportedKeys("", "default", "").next())
+
+ // We should disable these two APIs
Review comment:
Could you file JIRAs for these two?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org