You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2023/02/07 02:52:00 UTC
[kyuubi] branch master updated: [KYUUBI #3968][FOLLOWUP] Get schema url with correct version ordering
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new f62a7ac58 [KYUUBI #3968][FOLLOWUP] Get schema url with correct version ordering
f62a7ac58 is described below
commit f62a7ac5870de622c3b7c41713527a6dd9375e2d
Author: fwang12 <fw...@ebay.com>
AuthorDate: Tue Feb 7 10:51:51 2023 +0800
[KYUUBI #3968][FOLLOWUP] Get schema url with correct version ordering
### _Why are the changes needed?_
Get schema url with correct version ordering.
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4254 from turboFei/version_order.
Closes #3968
57474e65c [fwang12] refactor
561a1cde9 [fwang12] refactor
0a3275aa7 [fwang12] save
64236740d [fwang12] save
a723a4023 [fwang12] save
c31564865 [fwang12] save
13b3d29e0 [fwang12] save
347081d77 [fwang12] save
Authored-by: fwang12 <fw...@ebay.com>
Signed-off-by: fwang12 <fw...@ebay.com>
---
.../server/metadata/jdbc/JDBCMetadataStore.scala | 43 ++++++++++++++--------
.../metadata/jdbc/JDBCMetadataStoreSuite.scala | 16 ++++++++
2 files changed, 44 insertions(+), 15 deletions(-)
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala
index 151d846d8..f6caa9c1a 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala
@@ -96,37 +96,49 @@ class JDBCMetadataStore(conf: KyuubiConf) extends MetadataStore with Logging {
private[jdbc] def getInitSchema(dbType: DatabaseType): Option[String] = {
val classLoader = Utils.getContextOrKyuubiClassLoader
val schemaPackage = s"sql/${dbType.toString.toLowerCase}"
- val schemaUrlPattern = """^metadata-store-schema-(\d+)\.(\d+)\.(\d+)\.(.*)\.sql$""".r
- val schemaUrls = ListBuffer[String]()
- Option(classLoader.getResource(schemaPackage)).map(_.toURI).foreach { uri =>
+ Option(classLoader.getResource(schemaPackage)).map(_.toURI).flatMap { uri =>
val pathNames = if (uri.getScheme == "jar") {
val fs = FileSystems.newFileSystem(uri, Map.empty[String, AnyRef].asJava)
try {
Files.walk(fs.getPath(schemaPackage), 1).iterator().asScala.map(
_.getFileName.toString).filter { name =>
- schemaUrlPattern.findFirstMatchIn(name).isDefined
+ SCHEMA_URL_PATTERN.findFirstMatchIn(name).isDefined
}.toArray
} finally {
fs.close()
}
} else {
Paths.get(uri).toFile.listFiles((_, name) => {
- schemaUrlPattern.findFirstMatchIn(name).isDefined
+ SCHEMA_URL_PATTERN.findFirstMatchIn(name).isDefined
}).map(_.getName)
}
- pathNames.foreach(name => schemaUrls += s"$schemaPackage/$name")
- }
-
- schemaUrls.sorted.lastOption.map { schemaUrl =>
- val inputStream = classLoader.getResourceAsStream(schemaUrl)
- try {
- new BufferedReader(new InputStreamReader(inputStream)).lines()
- .collect(Collectors.joining("\n"))
- } finally {
- inputStream.close()
+ getLatestSchemaUrl(pathNames).map(name => s"$schemaPackage/$name").map { schemaUrl =>
+ val inputStream = classLoader.getResourceAsStream(schemaUrl)
+ try {
+ new BufferedReader(new InputStreamReader(inputStream)).lines()
+ .collect(Collectors.joining("\n"))
+ } finally {
+ inputStream.close()
+ }
}
+ }.headOption
+ }
+
+ def getSchemaVersion(schemaUrl: String): (Int, Int, Int) =
+ SCHEMA_URL_PATTERN.findFirstMatchIn(schemaUrl) match {
+ case Some(m) => (m.group(1).toInt, m.group(2).toInt, m.group(3).toInt)
+ case _ => throw new KyuubiException(s"Invalid schema url: $schemaUrl")
}
+
+ def getLatestSchemaUrl(schemaUrls: Seq[String]): Option[String] = {
+ schemaUrls.sortWith { (u1, u2) =>
+ val v1 = getSchemaVersion(u1)
+ val v2 = getSchemaVersion(u2)
+ v1._1 > v2._1 ||
+ (v1._1 == v2._1 && v1._2 > v2._2) ||
+ (v1._1 == v2._1 && v1._2 == v2._2 && v1._3 > v2._3)
+ }.headOption
}
override def close(): Unit = {
@@ -505,6 +517,7 @@ class JDBCMetadataStore(conf: KyuubiConf) extends MetadataStore with Logging {
}
object JDBCMetadataStore {
+ private val SCHEMA_URL_PATTERN = """^metadata-store-schema-(\d+)\.(\d+)\.(\d+)\.(.*)\.sql$""".r
private val METADATA_TABLE = "metadata"
private val METADATA_STATE_ONLY_COLUMNS = Seq(
"identifier",
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStoreSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStoreSuite.scala
index 73dc105c3..aa53af3a9 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStoreSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStoreSuite.scala
@@ -279,4 +279,20 @@ class JDBCMetadataStoreSuite extends KyuubiFunSuite {
jdbcMetadataStore.updateMetadata(metadata)
}
}
+
+ test("get schema urls with correct version ordering") {
+ val url1 = "metadata-store-schema-1.7.0.mysql.sql"
+ val url2 = "metadata-store-schema-1.7.1.mysql.sql"
+ val url3 = "metadata-store-schema-1.8.0.mysql.sql"
+ val url4 = "metadata-store-schema-1.10.0.mysql.sql"
+ val url5 = "metadata-store-schema-2.1.0.mysql.sql"
+ assert(jdbcMetadataStore.getSchemaVersion(url1) === ((1, 7, 0)))
+ assert(jdbcMetadataStore.getSchemaVersion(url2) === ((1, 7, 1)))
+ assert(jdbcMetadataStore.getSchemaVersion(url3) === ((1, 8, 0)))
+ assert(jdbcMetadataStore.getSchemaVersion(url4) === ((1, 10, 0)))
+ assert(jdbcMetadataStore.getSchemaVersion(url5) === ((2, 1, 0)))
+ assert(jdbcMetadataStore.getLatestSchemaUrl(Seq(url1, url2, url3, url4)).get === url4)
+ assert(jdbcMetadataStore.getLatestSchemaUrl(Seq(url1, url3, url4, url2)).get === url4)
+ assert(jdbcMetadataStore.getLatestSchemaUrl(Seq(url1, url2, url3, url4, url5)).get === url5)
+ }
}