You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by "anishshri-db (via GitHub)" <gi...@apache.org> on 2024/12/27 00:39:23 UTC
[PR] [SPARK-50655][SS] Move virt col family related mapping into db layer instead of encoder [spark]
anishshri-db opened a new pull request, #49304:
URL: https://github.com/apache/spark/pull/49304
### What changes were proposed in this pull request?
Move virt col family related mapping into db layer instead of encoder
### Why are the changes needed?
Keep abstraction clear around ownership and also expose internal/non-internal key metrics correctly
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Unit tests
### Was this patch authored or co-authored using generative AI tooling?
No
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1946049732
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -656,31 +803,75 @@ class RocksDB(
*
* @note This update is not committed to disk until commit() is called.
*/
- def merge(key: Array[Byte], value: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val oldValue = db.get(readOptions, key)
- if (oldValue == null) {
- numKeysOnWritingVersion += 1
+ def merge(
+ key: Array[Byte],
+ value: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion += 1
+ } else {
+ numKeysOnWritingVersion += 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ numKeysOnWritingVersion += 1
+ }
}
}
- db.merge(writeOptions, key, value)
- changelogWriter.foreach(_.merge(key, value))
+ db.merge(writeOptions, keyWithPrefix, value)
+ changelogWriter.foreach(_.merge(keyWithPrefix, value))
}
/**
* Remove the key if present.
* @note This update is not committed to disk until commit() is called.
*/
- def remove(key: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val value = db.get(readOptions, key)
- if (value != null) {
- numKeysOnWritingVersion -= 1
+ def remove(key: Array[Byte], cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue != null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion -= 1
+ } else {
+ numKeysOnWritingVersion -= 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val value = db.get(readOptions, keyWithPrefix)
+ if (value != null) {
+ numKeysOnWritingVersion -= 1
+ }
}
}
- db.delete(writeOptions, key)
- changelogWriter.foreach(_.delete(key))
+
+ db.delete(writeOptions, keyWithPrefix)
+ changelogWriter.foreach(_.delete(keyWithPrefix))
}
/**
Review Comment:
That is at the provider layer right ? at this layer we expect colFamilyName to be passed if multiple column families are being used
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1960628663
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala:
##########
@@ -1834,7 +1834,6 @@ class RocksDBStateStoreSuite extends StateStoreSuiteBase[RocksDBStateStoreProvid
store = provider.getRocksDBStateStore(2)
store.createColFamilyIfAbsent(colFamily3, keySchema, valueSchema,
NoPrefixKeyStateEncoderSpec(keySchema))
- assert(store.getColumnFamilyId(colFamily3) == 3)
Review Comment:
I removed this API actually - so removed this part of the test also
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "ericm-db (via GitHub)" <gi...@apache.org>.
ericm-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1901289015
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -473,23 +487,35 @@ class RocksDB(
private def openLocalRocksDB(metadata: RocksDBCheckpointMetadata): Unit = {
setInitialCFInfo()
metadata.columnFamilyMapping.foreach { mapping =>
- colFamilyNameToIdMap.putAll(mapping.asJava)
+ mapping.foreach { case (colFamilyName, colFamilyInfo) =>
+ colFamilyNameToInfoMap.putIfAbsent(colFamilyName, colFamilyInfo)
Review Comment:
I think we should make a private helper function that updates both of these maps at once
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1941761881
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -237,22 +272,22 @@ class RocksDB(
* @return - true if the column family exists, false otherwise
*/
def checkColFamilyExists(colFamilyName: String): Boolean = {
- colFamilyNameToIdMap.containsKey(colFamilyName)
+ db != null && colFamilyNameToInfoMap.containsKey(colFamilyName)
}
// This method sets the internal column family metadata to
// the default values it should be set to on load
private def setInitialCFInfo(): Unit = {
- colFamilyNameToIdMap.clear()
+ clearColFamilyMaps()
shouldForceSnapshot.set(false)
maxColumnFamilyId.set(0)
}
def getColFamilyCount(isInternal: Boolean): Long = {
if (isInternal) {
- colFamilyNameToIdMap.asScala.keys.toSeq.count(checkInternalColumnFamilies)
+ colFamilyNameToInfoMap.asScala.keys.toSeq.count(checkInternalColumnFamilies)
} else {
- colFamilyNameToIdMap.asScala.keys.toSeq.count(!checkInternalColumnFamilies(_))
+ colFamilyNameToInfoMap.asScala.keys.toSeq.count(!checkInternalColumnFamilies(_))
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1946037921
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -621,28 +690,106 @@ class RocksDB(
}
}
+ /**
+ * Function to encode state row with virtual col family id prefix
+ * @param data - passed byte array to be stored in state store
+ * @param cfName - name of column family
+ * @return - encoded byte array with virtual column family id prefix
+ */
+ private def encodeStateRowWithPrefix(
Review Comment:
Yea I did not change this
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -621,28 +690,106 @@ class RocksDB(
}
}
+ /**
+ * Function to encode state row with virtual col family id prefix
+ * @param data - passed byte array to be stored in state store
+ * @param cfName - name of column family
+ * @return - encoded byte array with virtual column family id prefix
+ */
+ private def encodeStateRowWithPrefix(
+ data: Array[Byte],
+ cfName: String): Array[Byte] = {
+ // Create result array big enough for all prefixes plus data
+ val result = new Array[Byte](StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES + data.length)
+ val offset = Platform.BYTE_ARRAY_OFFSET + StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES
+
+ val cfInfo = getColumnFamilyInfo(cfName)
+ Platform.putShort(result, Platform.BYTE_ARRAY_OFFSET, cfInfo.cfId)
+
+ // Write the actual data
+ Platform.copyMemory(
+ data, Platform.BYTE_ARRAY_OFFSET,
+ result, offset,
+ data.length
+ )
+
+ result
+ }
+
+ /**
+ * Function to decode state row with virtual col family id prefix
+ * @param data - passed byte array retrieved from state store
+ * @return - pair of decoded byte array without virtual column family id prefix
+ * and name of column family
+ */
+ private def decodeStateRowWithPrefix(data: Array[Byte]): (Array[Byte], String) = {
Review Comment:
Same here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1902268795
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -578,7 +578,11 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
if (isChangelogCheckpointingEnabled) {
assert(changelogVersionsPresent(remoteDir) === (1 to 50))
- assert(snapshotVersionsPresent(remoteDir) === Range.inclusive(5, 50, 5))
+ if (colFamiliesEnabled) {
Review Comment:
Yea the only difference is that when we create a new col family - we force the snapshot creation by setting a flag. In the tests - we have to create this each time if there is no commit involved - hence we see the difference
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1946060254
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBFileManager.scala:
##########
@@ -931,7 +941,8 @@ case class RocksDBCheckpointMetadata(
sstFiles: Seq[RocksDBSstFile],
logFiles: Seq[RocksDBLogFile],
numKeys: Long,
- columnFamilyMapping: Option[Map[String, Short]] = None,
+ numInternalKeys: Long,
+ columnFamilyMapping: Option[Map[String, ColumnFamilyInfo]] = None,
Review Comment:
I don't think we had this in 3.5 - do we still need a compatibility test then ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1959234122
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -656,31 +803,75 @@ class RocksDB(
*
* @note This update is not committed to disk until commit() is called.
*/
- def merge(key: Array[Byte], value: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val oldValue = db.get(readOptions, key)
- if (oldValue == null) {
- numKeysOnWritingVersion += 1
+ def merge(
+ key: Array[Byte],
+ value: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion += 1
+ } else {
+ numKeysOnWritingVersion += 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ numKeysOnWritingVersion += 1
+ }
}
}
- db.merge(writeOptions, key, value)
- changelogWriter.foreach(_.merge(key, value))
+ db.merge(writeOptions, keyWithPrefix, value)
+ changelogWriter.foreach(_.merge(keyWithPrefix, value))
}
/**
* Remove the key if present.
* @note This update is not committed to disk until commit() is called.
*/
- def remove(key: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val value = db.get(readOptions, key)
- if (value != null) {
- numKeysOnWritingVersion -= 1
+ def remove(key: Array[Byte], cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue != null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion -= 1
+ } else {
+ numKeysOnWritingVersion -= 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val value = db.get(readOptions, keyWithPrefix)
+ if (value != null) {
+ numKeysOnWritingVersion -= 1
+ }
}
}
- db.delete(writeOptions, key)
- changelogWriter.foreach(_.delete(key))
+
+ db.delete(writeOptions, keyWithPrefix)
+ changelogWriter.foreach(_.delete(keyWithPrefix))
}
/**
Review Comment:
Done - left both interfaces for now
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1960633610
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -918,18 +968,34 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
}
// Check that snapshots and changelogs get purged correctly.
db.doMaintenance()
- assert(snapshotVersionsPresent(remoteDir) === Seq(30, 60))
+
+ if (enableStateStoreCheckpointIds && colFamiliesEnabled) {
Review Comment:
No - one is checking for snapshot version and the other for changelog version
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1961021758
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -1074,14 +1140,25 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
db.load(version, versionToUniqueId.get(version))
assert(db.iterator().map(toStr).toSet === Set((version.toString, version.toString)))
}
+
for (version <- 31 to 60) {
db.load(version - 1, versionToUniqueId.get(version - 1))
db.put(version.toString, version.toString)
db.remove((version - 1).toString)
db.commit()
}
assert(changelogVersionsPresent(remoteDir) === (1 to 30))
- assert(snapshotVersionsPresent(remoteDir) === (31 to 60))
+
+ var result: Seq[Long] = if (colFamiliesEnabled) {
+ Seq(1)
+ } else {
+ Seq.empty
+ }
+
+ (31 to 60).foreach { i =>
Review Comment:
Actually have to revert to original for this to work
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virt col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on PR #49304:
URL: https://github.com/apache/spark/pull/49304#issuecomment-2564235720
cc - @ericm-db @jingz-db - PTAL, thx !
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1947392942
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateEncoder.scala:
##########
@@ -724,20 +723,21 @@ class AvroStateEncoder(
keyStateEncoderSpec: KeyStateEncoderSpec,
valueSchema: StructType,
stateSchemaProvider: Option[StateSchemaProvider],
- columnFamilyInfo: Option[ColumnFamilyInfo]
+ columnFamilyName: Option[String]
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "jingz-db (via GitHub)" <gi...@apache.org>.
jingz-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1902110017
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -578,7 +578,11 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
if (isChangelogCheckpointingEnabled) {
assert(changelogVersionsPresent(remoteDir) === (1 to 50))
- assert(snapshotVersionsPresent(remoteDir) === Range.inclusive(5, 50, 5))
+ if (colFamiliesEnabled) {
Review Comment:
I might be missing something, but why will we get different results for snapshot with colFamiliesEnabled and disabled? I thought we are writing to the changelog the same way so this should be the same?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "HeartSaVioR (via GitHub)" <gi...@apache.org>.
HeartSaVioR commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1926469488
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateEncoder.scala:
##########
@@ -724,20 +723,21 @@ class AvroStateEncoder(
keyStateEncoderSpec: KeyStateEncoderSpec,
valueSchema: StructType,
stateSchemaProvider: Option[StateSchemaProvider],
- columnFamilyInfo: Option[ColumnFamilyInfo]
+ columnFamilyName: Option[String]
Review Comment:
Why not just have `columnFamilyName: String` rather than asserting? Do we create the instance of this class dynamically?
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala:
##########
@@ -702,6 +696,8 @@ object StateStore extends Logging {
val DEFAULT_COL_FAMILY_NAME = "default"
+ val VIRTUAL_COL_FAMILY_PREFIX_BYTES = 2
Review Comment:
I guess this is bound to RocksDB and don't need to be coupled with StateStore.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -237,22 +272,22 @@ class RocksDB(
* @return - true if the column family exists, false otherwise
*/
def checkColFamilyExists(colFamilyName: String): Boolean = {
- colFamilyNameToIdMap.containsKey(colFamilyName)
+ db != null && colFamilyNameToInfoMap.containsKey(colFamilyName)
}
// This method sets the internal column family metadata to
// the default values it should be set to on load
private def setInitialCFInfo(): Unit = {
- colFamilyNameToIdMap.clear()
+ clearColFamilyMaps()
shouldForceSnapshot.set(false)
maxColumnFamilyId.set(0)
}
def getColFamilyCount(isInternal: Boolean): Long = {
if (isInternal) {
- colFamilyNameToIdMap.asScala.keys.toSeq.count(checkInternalColumnFamilies)
+ colFamilyNameToInfoMap.asScala.keys.toSeq.count(checkInternalColumnFamilies)
} else {
- colFamilyNameToIdMap.asScala.keys.toSeq.count(!checkInternalColumnFamilies(_))
+ colFamilyNameToInfoMap.asScala.keys.toSeq.count(!checkInternalColumnFamilies(_))
Review Comment:
Would this work to remove if statement?
```
colFamilyNameToInfoMap.asScala.keys.toSeq.count(checkInternalColumnFamilies(_) == isInternal)
```
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -621,28 +690,106 @@ class RocksDB(
}
}
+ /**
+ * Function to encode state row with virtual col family id prefix
+ * @param data - passed byte array to be stored in state store
+ * @param cfName - name of column family
+ * @return - encoded byte array with virtual column family id prefix
+ */
+ private def encodeStateRowWithPrefix(
+ data: Array[Byte],
+ cfName: String): Array[Byte] = {
+ // Create result array big enough for all prefixes plus data
+ val result = new Array[Byte](StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES + data.length)
+ val offset = Platform.BYTE_ARRAY_OFFSET + StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES
+
+ val cfInfo = getColumnFamilyInfo(cfName)
+ Platform.putShort(result, Platform.BYTE_ARRAY_OFFSET, cfInfo.cfId)
+
+ // Write the actual data
+ Platform.copyMemory(
+ data, Platform.BYTE_ARRAY_OFFSET,
+ result, offset,
+ data.length
+ )
+
+ result
+ }
+
+ /**
+ * Function to decode state row with virtual col family id prefix
+ * @param data - passed byte array retrieved from state store
+ * @return - pair of decoded byte array without virtual column family id prefix
+ * and name of column family
+ */
+ private def decodeStateRowWithPrefix(data: Array[Byte]): (Array[Byte], String) = {
Review Comment:
ditto
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBFileManager.scala:
##########
@@ -931,7 +941,8 @@ case class RocksDBCheckpointMetadata(
sstFiles: Seq[RocksDBSstFile],
logFiles: Seq[RocksDBLogFile],
numKeys: Long,
- columnFamilyMapping: Option[Map[String, Short]] = None,
+ numInternalKeys: Long,
+ columnFamilyMapping: Option[Map[String, ColumnFamilyInfo]] = None,
Review Comment:
This is a breaking change as the existing field gets different representation for json. Shall we check whether we ever release the version before the change, to ensure we are not breaking the checkpoint? Let's also check with the latest bugfix version of Spark 3.5.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -237,22 +272,22 @@ class RocksDB(
* @return - true if the column family exists, false otherwise
*/
def checkColFamilyExists(colFamilyName: String): Boolean = {
- colFamilyNameToIdMap.containsKey(colFamilyName)
+ db != null && colFamilyNameToInfoMap.containsKey(colFamilyName)
Review Comment:
Mind explaining why we check for db instance?
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -621,28 +690,106 @@ class RocksDB(
}
}
+ /**
+ * Function to encode state row with virtual col family id prefix
+ * @param data - passed byte array to be stored in state store
+ * @param cfName - name of column family
+ * @return - encoded byte array with virtual column family id prefix
+ */
+ private def encodeStateRowWithPrefix(
Review Comment:
Shall we revisit the method name? It's not very clear we will **prefix vcf ID** to the **key row** from the method name.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBFileManager.scala:
##########
@@ -923,6 +924,15 @@ object RocksDBFileManagerMetrics {
val EMPTY_METRICS = RocksDBFileManagerMetrics(0L, 0L, 0L, None)
}
+/**
+ * Case class to keep track of column family info within checkpoint metadata.
+ * @param cfId - virtual column family id
+ * @param isInternal - whether the column family is internal or not
+ */
+case class ColumnFamilyInfo(
+ cfId: Short,
+ isInternal: Boolean)
Review Comment:
It'd be ideal to make clear which is the source of truth - we seem to use the naming convention (starting with `_`) to determine the internal column family. Is it always reliable or we should look at the flag? If we decide the source of truth for this, I'd say the other should not be used to determine whether the cf is internal or not.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -656,31 +803,75 @@ class RocksDB(
*
* @note This update is not committed to disk until commit() is called.
*/
- def merge(key: Array[Byte], value: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val oldValue = db.get(readOptions, key)
- if (oldValue == null) {
- numKeysOnWritingVersion += 1
+ def merge(
+ key: Array[Byte],
+ value: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion += 1
+ } else {
+ numKeysOnWritingVersion += 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ numKeysOnWritingVersion += 1
+ }
}
}
- db.merge(writeOptions, key, value)
- changelogWriter.foreach(_.merge(key, value))
+ db.merge(writeOptions, keyWithPrefix, value)
+ changelogWriter.foreach(_.merge(keyWithPrefix, value))
}
/**
* Remove the key if present.
* @note This update is not committed to disk until commit() is called.
*/
- def remove(key: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val value = db.get(readOptions, key)
- if (value != null) {
- numKeysOnWritingVersion -= 1
+ def remove(key: Array[Byte], cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue != null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion -= 1
+ } else {
+ numKeysOnWritingVersion -= 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val value = db.get(readOptions, keyWithPrefix)
+ if (value != null) {
+ numKeysOnWritingVersion -= 1
+ }
}
}
- db.delete(writeOptions, key)
- changelogWriter.foreach(_.delete(key))
+
+ db.delete(writeOptions, keyWithPrefix)
+ changelogWriter.foreach(_.delete(keyWithPrefix))
}
/**
Review Comment:
I'm not sure how we missed this (or we were doing prefix scan to cover this indirectly)...
I don't think iterator() makes sense for multiple column families use case. The resulting iterator won't give any information about column family, so it's not even useful for global scan on entire space as well. (Different CFs can have the same key.)
I think we have to make change on the iterator() to receive cfName to scan only for certain cf. Please let me know if you think we can't.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBFileManager.scala:
##########
@@ -931,7 +941,8 @@ case class RocksDBCheckpointMetadata(
sstFiles: Seq[RocksDBSstFile],
logFiles: Seq[RocksDBLogFile],
numKeys: Long,
- columnFamilyMapping: Option[Map[String, Short]] = None,
+ numInternalKeys: Long,
+ columnFamilyMapping: Option[Map[String, ColumnFamilyInfo]] = None,
Review Comment:
If we didn't have the test for compatibility then we probably should have it.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -723,20 +920,43 @@ class RocksDB(
iter.seekToFirst()
var keys = 0L
- while (iter.isValid) {
- keys += 1
- iter.next()
+ var internalKeys = 0L
+
+ if (!useColumnFamilies) {
+ while (iter.isValid) {
+ keys += 1
+ iter.next()
+ }
+ } else {
+ while (iter.isValid) {
+ val (_, cfName) = decodeStateRowWithPrefix(iter.key)
+ val cfInfo = getColumnFamilyInfo(cfName)
Review Comment:
nit: We could do some micro optimization here - all keys in the same CF will (and must) appear sequentially, so it is a waste of time to perform lookup from the map for every key. We can have current cf with internal flag (or just ColumnFamilyInfo) as variables, and leverage them.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -621,28 +690,106 @@ class RocksDB(
}
}
+ /**
+ * Function to encode state row with virtual col family id prefix
+ * @param data - passed byte array to be stored in state store
+ * @param cfName - name of column family
+ * @return - encoded byte array with virtual column family id prefix
+ */
+ private def encodeStateRowWithPrefix(
Review Comment:
Nvm if you just moved the existing method.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1960642397
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -629,7 +627,12 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
if (isChangelogCheckpointingEnabled) {
assert(changelogVersionsPresent(remoteDir) === (1 to 50))
- assert(snapshotVersionsPresent(remoteDir) === Range.inclusive(5, 50, 5))
+ if (colFamiliesEnabled) {
+ assert(snapshotVersionsPresent(remoteDir) ===
+ Seq(1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50))
Review Comment:
No - this just additionally sets the flag for the first batch as well. so just added 1 essentially
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1946059793
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -723,20 +920,43 @@ class RocksDB(
iter.seekToFirst()
var keys = 0L
- while (iter.isValid) {
- keys += 1
- iter.next()
+ var internalKeys = 0L
+
+ if (!useColumnFamilies) {
+ while (iter.isValid) {
+ keys += 1
+ iter.next()
+ }
+ } else {
+ while (iter.isValid) {
+ val (_, cfName) = decodeStateRowWithPrefix(iter.key)
+ val cfInfo = getColumnFamilyInfo(cfName)
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1901324278
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -723,20 +888,42 @@ class RocksDB(
iter.seekToFirst()
var keys = 0L
- while (iter.isValid) {
- keys += 1
- iter.next()
+ var internalKeys = 0L
+
+ if (!useColumnFamilies) {
+ while (iter.isValid) {
+ keys += 1
+ iter.next()
+ }
+ } else {
+ while (iter.isValid) {
+ val (_, cfName) = decodeStateRowWithPrefix(iter.key)
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ internalKeys += 1
+ } else {
+ keys += 1
+ }
+ iter.next()
+ }
}
- keys
+ (keys, internalKeys)
} finally {
iter.close()
}
}
- def prefixScan(prefix: Array[Byte]): Iterator[ByteArrayPair] = {
+ def prefixScan(
+ prefix: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Iterator[ByteArrayPair] = {
val iter = db.newIterator()
- iter.seek(prefix)
+ var updatedPrefix = prefix
Review Comment:
Done
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -473,23 +487,35 @@ class RocksDB(
private def openLocalRocksDB(metadata: RocksDBCheckpointMetadata): Unit = {
setInitialCFInfo()
metadata.columnFamilyMapping.foreach { mapping =>
- colFamilyNameToIdMap.putAll(mapping.asJava)
+ mapping.foreach { case (colFamilyName, colFamilyInfo) =>
+ colFamilyNameToInfoMap.putIfAbsent(colFamilyName, colFamilyInfo)
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "HeartSaVioR (via GitHub)" <gi...@apache.org>.
HeartSaVioR commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1926516850
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -656,31 +803,75 @@ class RocksDB(
*
* @note This update is not committed to disk until commit() is called.
*/
- def merge(key: Array[Byte], value: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val oldValue = db.get(readOptions, key)
- if (oldValue == null) {
- numKeysOnWritingVersion += 1
+ def merge(
+ key: Array[Byte],
+ value: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion += 1
+ } else {
+ numKeysOnWritingVersion += 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ numKeysOnWritingVersion += 1
+ }
}
}
- db.merge(writeOptions, key, value)
- changelogWriter.foreach(_.merge(key, value))
+ db.merge(writeOptions, keyWithPrefix, value)
+ changelogWriter.foreach(_.merge(keyWithPrefix, value))
}
/**
* Remove the key if present.
* @note This update is not committed to disk until commit() is called.
*/
- def remove(key: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val value = db.get(readOptions, key)
- if (value != null) {
- numKeysOnWritingVersion -= 1
+ def remove(key: Array[Byte], cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue != null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion -= 1
+ } else {
+ numKeysOnWritingVersion -= 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val value = db.get(readOptions, keyWithPrefix)
+ if (value != null) {
+ numKeysOnWritingVersion -= 1
+ }
}
}
- db.delete(writeOptions, key)
- changelogWriter.foreach(_.delete(key))
+
+ db.delete(writeOptions, keyWithPrefix)
+ changelogWriter.foreach(_.delete(keyWithPrefix))
}
/**
Review Comment:
I'm not sure how we missed this (or we were doing prefix scan to cover this indirectly)...
I don't think iterator() makes sense for multiple column families use case. The resulting iterator won't give any information about column family, so it's not even useful for global scan on entire space as well. (Different CFs can have the same key.)
I think we have to make change on the iterator() to receive cfName to scan only for certain cf. Please let me know if you think we can't (or you see use cases of performing global scan).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "HeartSaVioR (via GitHub)" <gi...@apache.org>.
HeartSaVioR commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1957541951
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala:
##########
@@ -678,7 +654,7 @@ object RocksDBStateStoreProvider {
// Version as a single byte that specifies the encoding of the row data in RocksDB
val STATE_ENCODING_NUM_VERSION_BYTES = 1
val STATE_ENCODING_VERSION: Byte = 0
- val VIRTUAL_COL_FAMILY_PREFIX_BYTES = 2
Review Comment:
This should be here, rather than StateStore. I think it was put to the right place.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala:
##########
@@ -847,35 +828,47 @@ class RocksDBStateStoreChangeDataReader(
endVersion: Long,
compressionCodec: CompressionCodec,
keyValueEncoderMap:
- ConcurrentHashMap[String, (RocksDBKeyStateEncoder, RocksDBValueStateEncoder)],
+ ConcurrentHashMap[String, (RocksDBKeyStateEncoder, RocksDBValueStateEncoder, Short)],
colFamilyNameOpt: Option[String] = None)
extends StateStoreChangeDataReader(
fm, stateLocation, startVersion, endVersion, compressionCodec, colFamilyNameOpt) {
override protected var changelogSuffix: String = "changelog"
- private def getColFamilyIdBytes: Option[Array[Byte]] = {
- if (colFamilyNameOpt.isDefined) {
- val colFamilyName = colFamilyNameOpt.get
- if (!keyValueEncoderMap.containsKey(colFamilyName)) {
- throw new IllegalStateException(
- s"Column family $colFamilyName not found in the key value encoder map")
- }
- Some(keyValueEncoderMap.get(colFamilyName)._1.getColumnFamilyIdBytes())
- } else {
- None
- }
+ /**
+ * Encodes a virtual column family ID into a byte array suitable for RocksDB.
+ *
+ * This method creates a fixed-size byte array prefixed with the virtual column family ID,
+ * which is used to partition data within RocksDB.
+ *
+ * @param virtualColFamilyId The column family identifier to encode
+ * @return A byte array containing the encoded column family ID
+ */
+ private def getColumnFamilyIdBytes(virtualColFamilyId: Short): Array[Byte] = {
+ val encodedBytes = new Array[Byte](StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES)
+ Platform.putShort(encodedBytes, Platform.BYTE_ARRAY_OFFSET, virtualColFamilyId)
+ encodedBytes
}
- private val colFamilyIdBytesOpt: Option[Array[Byte]] = getColFamilyIdBytes
+ private def getExtractedKey(data: Array[Byte]): Array[Byte] = {
Review Comment:
ditto
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateEncoder.scala:
##########
@@ -1814,18 +1663,17 @@ class NoPrefixKeyStateEncoder(
rowBytes.length
)
- encodeStateRowWithPrefix(dataWithVersion)
+ dataWithVersion
}
}
override def decodeKey(keyBytes: Array[Byte]): UnsafeRow = {
if (!useColumnFamilies) {
- dataEncoder.decodeKey(decodeStateRowData(keyBytes))
+ dataEncoder.decodeKey(keyBytes)
Review Comment:
ditto if the above is turned out to be a bug.
If the above is a bug, this cannot read from existing checkpoint from prior Spark versions.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -656,31 +813,75 @@ class RocksDB(
*
* @note This update is not committed to disk until commit() is called.
*/
- def merge(key: Array[Byte], value: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val oldValue = db.get(readOptions, key)
- if (oldValue == null) {
- numKeysOnWritingVersion += 1
+ def merge(
+ key: Array[Byte],
+ value: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion += 1
+ } else {
+ numKeysOnWritingVersion += 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ numKeysOnWritingVersion += 1
+ }
}
}
- db.merge(writeOptions, key, value)
- changelogWriter.foreach(_.merge(key, value))
+ db.merge(writeOptions, keyWithPrefix, value)
+ changelogWriter.foreach(_.merge(keyWithPrefix, value))
}
/**
* Remove the key if present.
* @note This update is not committed to disk until commit() is called.
*/
- def remove(key: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val value = db.get(readOptions, key)
- if (value != null) {
- numKeysOnWritingVersion -= 1
+ def remove(key: Array[Byte], cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
Review Comment:
nit: the extracted method in above can handle both directions (when putting/removing) with a flag. I'd be OK to have two different extracted methods if it is clearer.
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -918,18 +968,34 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
}
// Check that snapshots and changelogs get purged correctly.
db.doMaintenance()
- assert(snapshotVersionsPresent(remoteDir) === Seq(30, 60))
+
+ if (enableStateStoreCheckpointIds && colFamiliesEnabled) {
+ assert(snapshotVersionsPresent(remoteDir) === Seq(31, 60, 60))
+ } else {
+ assert(snapshotVersionsPresent(remoteDir) === Seq(30, 60))
+ }
if (enableStateStoreCheckpointIds) {
// recommit version 60 creates another changelog file with different unique id
- assert(changelogVersionsPresent(remoteDir) === (30 to 60) :+ 60)
+ if (colFamiliesEnabled) {
+ assert(changelogVersionsPresent(remoteDir) === (31 to 60) :+ 60)
+ } else {
+ assert(changelogVersionsPresent(remoteDir) === (30 to 60) :+ 60)
+ }
} else {
assert(changelogVersionsPresent(remoteDir) === (30 to 60))
}
// Verify the content of retained versions.
- for (version <- 30 to 60) {
- db.load(version, versionToUniqueId.get(version), readOnly = true)
- assert(db.iterator().map(toStr).toSet === Set((version.toString, version.toString)))
+ if (enableStateStoreCheckpointIds && colFamiliesEnabled) {
+ for (version <- 31 to 60) {
Review Comment:
nit: please leave a comment why version 30 does not work, and what is the state value in version 30 in this case. I'm not sure why there is a difference, from reading the test code.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala:
##########
@@ -847,35 +828,47 @@ class RocksDBStateStoreChangeDataReader(
endVersion: Long,
compressionCodec: CompressionCodec,
keyValueEncoderMap:
- ConcurrentHashMap[String, (RocksDBKeyStateEncoder, RocksDBValueStateEncoder)],
+ ConcurrentHashMap[String, (RocksDBKeyStateEncoder, RocksDBValueStateEncoder, Short)],
colFamilyNameOpt: Option[String] = None)
extends StateStoreChangeDataReader(
fm, stateLocation, startVersion, endVersion, compressionCodec, colFamilyNameOpt) {
override protected var changelogSuffix: String = "changelog"
- private def getColFamilyIdBytes: Option[Array[Byte]] = {
- if (colFamilyNameOpt.isDefined) {
- val colFamilyName = colFamilyNameOpt.get
- if (!keyValueEncoderMap.containsKey(colFamilyName)) {
- throw new IllegalStateException(
- s"Column family $colFamilyName not found in the key value encoder map")
- }
- Some(keyValueEncoderMap.get(colFamilyName)._1.getColumnFamilyIdBytes())
- } else {
- None
- }
+ /**
+ * Encodes a virtual column family ID into a byte array suitable for RocksDB.
+ *
+ * This method creates a fixed-size byte array prefixed with the virtual column family ID,
+ * which is used to partition data within RocksDB.
+ *
+ * @param virtualColFamilyId The column family identifier to encode
+ * @return A byte array containing the encoded column family ID
+ */
+ private def getColumnFamilyIdBytes(virtualColFamilyId: Short): Array[Byte] = {
Review Comment:
We are now having two different places to handle cfId, here and RocksDB. Shall we consolidate, or extract out to new class/object if there is no good way to consolidate?
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala:
##########
@@ -1834,7 +1834,6 @@ class RocksDBStateStoreSuite extends StateStoreSuiteBase[RocksDBStateStoreProvid
store = provider.getRocksDBStateStore(2)
store.createColFamilyIfAbsent(colFamily3, keySchema, valueSchema,
NoPrefixKeyStateEncoderSpec(keySchema))
- assert(store.getColumnFamilyId(colFamily3) == 3)
Review Comment:
I'm not sure this test is still valid after you removed the check for column family ID. What's rationale about removing this?
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateEncoder.scala:
##########
@@ -1791,16 +1644,12 @@ class RangeKeyScanStateEncoder(
class NoPrefixKeyStateEncoder(
dataEncoder: RocksDBDataEncoder,
keySchema: StructType,
- useColumnFamilies: Boolean = false,
- columnFamilyInfo: Option[ColumnFamilyInfo] = None)
- extends StateRowPrefixEncoder(
- useColumnFamilies,
- columnFamilyInfo
- ) with RocksDBKeyStateEncoder with Logging {
+ useColumnFamilies: Boolean = false)
+ extends RocksDBKeyStateEncoder with Logging {
override def encodeKey(row: UnsafeRow): Array[Byte] = {
if (!useColumnFamilies) {
Review Comment:
I was in doubt why useColumnFamilies is needed for KeyStateEncoder while we move virtual column family out from encoder... and found this possible bug.
My understanding is, in else statement (useColumnFamilies = true), we write version byte "twice", because `dataEncoder.encodeKey(row)` put version byte already. If this is a bug and you just need to call `dataEncoder.encodeKey(row)`, `useColumnFamilies` isn't used for any implementation of KeyStateEncoder and you can remove it.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -467,29 +492,62 @@ class RocksDB(
this
}
+ /**
+ * Function to check if col family is internal or not based on information recorded in
+ * checkpoint metadata.
+ * @param cfName - column family name
+ * @param metadata - checkpoint metadata
+ * @return - type of column family (internal or otherwise)
+ */
+ private def checkColFamilyType(
Review Comment:
nit: since this is returning Boolean, it'd be ideal to name it as isXXX so that people can understand which true will map to and for false. Just 2 cents and OK to leave it as it is if the name is not clear in this way.
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -918,18 +968,34 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
}
// Check that snapshots and changelogs get purged correctly.
db.doMaintenance()
- assert(snapshotVersionsPresent(remoteDir) === Seq(30, 60))
+
+ if (enableStateStoreCheckpointIds && colFamiliesEnabled) {
Review Comment:
nit: isn't it duplicated with the next if statement?
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -629,7 +627,12 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
if (isChangelogCheckpointingEnabled) {
assert(changelogVersionsPresent(remoteDir) === (1 to 50))
- assert(snapshotVersionsPresent(remoteDir) === Range.inclusive(5, 50, 5))
+ if (colFamiliesEnabled) {
+ assert(snapshotVersionsPresent(remoteDir) ===
+ Seq(1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50))
Review Comment:
Wait, we should have set the flag on enforcing snapshot for change of column family before this fix. Does this mean we had a bug?
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -621,28 +700,106 @@ class RocksDB(
}
}
+ /**
+ * Function to encode state row with virtual col family id prefix
+ * @param data - passed byte array to be stored in state store
+ * @param cfName - name of column family
+ * @return - encoded byte array with virtual column family id prefix
+ */
+ private def encodeStateRowWithPrefix(
+ data: Array[Byte],
+ cfName: String): Array[Byte] = {
+ // Create result array big enough for all prefixes plus data
+ val result = new Array[Byte](StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES + data.length)
+ val offset = Platform.BYTE_ARRAY_OFFSET + StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES
+
+ val cfInfo = getColumnFamilyInfo(cfName)
+ Platform.putShort(result, Platform.BYTE_ARRAY_OFFSET, cfInfo.cfId)
+
+ // Write the actual data
+ Platform.copyMemory(
+ data, Platform.BYTE_ARRAY_OFFSET,
+ result, offset,
+ data.length
+ )
+
+ result
+ }
+
+ /**
+ * Function to decode state row with virtual col family id prefix
+ * @param data - passed byte array retrieved from state store
+ * @return - pair of decoded byte array without virtual column family id prefix
+ * and name of column family
+ */
+ private def decodeStateRowWithPrefix(data: Array[Byte]): (Array[Byte], String) = {
+ val cfId = Platform.getShort(data, Platform.BYTE_ARRAY_OFFSET)
+ val cfName = getColumnFamilyNameForId(cfId)
+ val offset = Platform.BYTE_ARRAY_OFFSET + StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES
+
+ val key = new Array[Byte](data.length - StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES)
+ Platform.copyMemory(
+ data, offset,
+ key, Platform.BYTE_ARRAY_OFFSET,
+ key.length
+ )
+
+ (key, cfName)
+ }
+
/**
* Get the value for the given key if present, or null.
* @note This will return the last written value even if it was uncommitted.
*/
- def get(key: Array[Byte]): Array[Byte] = {
- db.get(readOptions, key)
+ def get(
+ key: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Array[Byte] = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ db.get(readOptions, keyWithPrefix)
}
/**
* Put the given value for the given key.
* @note This update is not committed to disk until commit() is called.
*/
- def put(key: Array[Byte], value: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val oldValue = db.get(readOptions, key)
- if (oldValue == null) {
- numKeysOnWritingVersion += 1
+ def put(
+ key: Array[Byte],
+ value: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
Review Comment:
nit: This is same as merge, extract?
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -178,55 +183,77 @@ class RocksDB(
// This is accessed and updated only between load and commit
// which means it is implicitly guarded by acquireLock
@GuardedBy("acquireLock")
- private val colFamilyNameToIdMap = new ConcurrentHashMap[String, Short]()
+ private val colFamilyNameToInfoMap = new ConcurrentHashMap[String, ColumnFamilyInfo]()
+
+ @GuardedBy("acquireLock")
+ private val colFamilyIdToNameMap = new ConcurrentHashMap[Short, String]()
@GuardedBy("acquireLock")
private val maxColumnFamilyId: AtomicInteger = new AtomicInteger(-1)
@GuardedBy("acquireLock")
private val shouldForceSnapshot: AtomicBoolean = new AtomicBoolean(false)
- /**
- * Check whether the column family name is for internal column families.
- *
- * @param cfName - column family name
- * @return - true if the column family is for internal use, false otherwise
- */
- private def checkInternalColumnFamilies(cfName: String): Boolean = cfName.charAt(0) == '_'
+ private def getColumnFamilyInfo(cfName: String): ColumnFamilyInfo = {
+ colFamilyNameToInfoMap.get(cfName)
+ }
+
+ private def getColumnFamilyNameForId(cfId: Short): String = {
+ colFamilyIdToNameMap.get(cfId)
+ }
- // Methods to fetch column family mapping for this State Store version
- def getColumnFamilyMapping: Map[String, Short] = {
- colFamilyNameToIdMap.asScala
+ private def addToColFamilyMaps(cfName: String, cfId: Short, isInternal: Boolean): Unit = {
+ colFamilyNameToInfoMap.putIfAbsent(cfName, ColumnFamilyInfo(cfId, isInternal))
+ colFamilyIdToNameMap.putIfAbsent(cfId, cfName)
+ }
+
+ private def removeFromColFamilyMaps(cfName: String): Unit = {
+ val colFamilyInfo = colFamilyNameToInfoMap.get(cfName)
+ if (colFamilyInfo != null) {
+ colFamilyNameToInfoMap.remove(cfName)
+ colFamilyIdToNameMap.remove(colFamilyInfo.cfId)
+ }
}
- def getColumnFamilyId(cfName: String): Short = {
- colFamilyNameToIdMap.get(cfName)
+ private def clearColFamilyMaps(): Unit = {
+ colFamilyNameToInfoMap.clear()
+ colFamilyIdToNameMap.clear()
}
/**
- * Create RocksDB column family, if not created already
+ * Check if the column family exists with given name and create one if it doesn't. Users can
+ * create external column families storing user facing data as well as internal column families
+ * such as secondary indexes. Metrics for both of these types are tracked separately.
+ *
+ * @param colFamilyName - column family name
+ * @param isInternal - whether the column family is for internal use or not
+ * @return - virtual column family id
*/
- def createColFamilyIfAbsent(colFamilyName: String): Short = {
+ def createColFamilyIfAbsent(colFamilyName: String, isInternal: Boolean): Short = {
if (!checkColFamilyExists(colFamilyName)) {
val newColumnFamilyId = maxColumnFamilyId.incrementAndGet().toShort
- colFamilyNameToIdMap.putIfAbsent(colFamilyName, newColumnFamilyId)
+ addToColFamilyMaps(colFamilyName, newColumnFamilyId, isInternal)
shouldForceSnapshot.set(true)
newColumnFamilyId
} else {
- colFamilyNameToIdMap.get(colFamilyName)
+ colFamilyNameToInfoMap.get(colFamilyName).cfId
}
}
/**
* Remove RocksDB column family, if exists
* @return columnFamilyId if it exists, else None
*/
- def removeColFamilyIfExists(colFamilyName: String): Option[Short] = {
+ def removeColFamilyIfExists(colFamilyName: String): Boolean = {
if (checkColFamilyExists(colFamilyName)) {
shouldForceSnapshot.set(true)
- Some(colFamilyNameToIdMap.remove(colFamilyName))
+ prefixScan(Array.empty[Byte], colFamilyName).foreach { kv =>
Review Comment:
Likewise I commented in above, `iterator(colFamilyName)` would be lot easier to understand (higher level of operation).
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -381,9 +381,7 @@ class RocksDBStateEncoderSuite extends SparkFunSuite {
keyStateEncoderSpec,
valueSchema,
Some(testProvider),
- Some(ColumnFamilyInfo(StateStore.DEFAULT_COL_FAMILY_NAME, 0))
- )
- new AvroStateEncoder(keyStateEncoderSpec, valueSchema, None, None)
Review Comment:
Nice finding!
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -629,7 +627,12 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
if (isChangelogCheckpointingEnabled) {
assert(changelogVersionsPresent(remoteDir) === (1 to 50))
- assert(snapshotVersionsPresent(remoteDir) === Range.inclusive(5, 50, 5))
+ if (colFamiliesEnabled) {
+ assert(snapshotVersionsPresent(remoteDir) ===
+ Seq(1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50))
Review Comment:
nit: Probably easier to see the diff with `Seq(1) ++ Range.inclusive(5, 10, 5)`
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -700,7 +901,13 @@ class RocksDB(
new NextIterator[ByteArrayPair] {
override protected def getNext(): ByteArrayPair = {
if (iter.isValid) {
- byteArrayPair.set(iter.key, iter.value)
+ val key = if (useColumnFamilies) {
+ decodeStateRowWithPrefix(iter.key)._1
Review Comment:
This concerns me a lot, as not giving the column family ID/name back to the caller makes the outcome be useless for multiple column family case. Say, column family cfA and cfB somehow have the same key keyA. This doesn't even give any meaning to global scan.
That's why I commented like the above.
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -1074,14 +1140,25 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
db.load(version, versionToUniqueId.get(version))
assert(db.iterator().map(toStr).toSet === Set((version.toString, version.toString)))
}
+
for (version <- 31 to 60) {
db.load(version - 1, versionToUniqueId.get(version - 1))
db.put(version.toString, version.toString)
db.remove((version - 1).toString)
db.commit()
}
assert(changelogVersionsPresent(remoteDir) === (1 to 30))
- assert(snapshotVersionsPresent(remoteDir) === (31 to 60))
+
+ var result: Seq[Long] = if (colFamiliesEnabled) {
+ Seq(1)
+ } else {
+ Seq.empty
+ }
+
+ (31 to 60).foreach { i =>
Review Comment:
nit: doesn't this work? `result ++ (31 to 60)`
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -902,7 +946,13 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
db.remove((version - 1).toString)
db.commit()
}
- assert(snapshotVersionsPresent(remoteDir) === (1 to 30))
+
+ if (enableStateStoreCheckpointIds && colFamiliesEnabled) {
+ assert(snapshotVersionsPresent(remoteDir) === (1 to 30) :+ 30 :+ 31)
Review Comment:
Is this because 30 is executed two times and snapshot does not overwrite in checkpoint v2?
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala:
##########
@@ -702,6 +696,8 @@ object StateStore extends Logging {
val DEFAULT_COL_FAMILY_NAME = "default"
+ val VIRTUAL_COL_FAMILY_PREFIX_BYTES = 2
Review Comment:
This is bound to only of StateStore "implementation" (RocksDBStateStoreProvider), not used in StateStore interface and the base implementation.
Move this to RocksDBStateStoreProvider or some utility class which is referenced by both RocksDB and RocksDBStateStoreProvider.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -656,31 +803,75 @@ class RocksDB(
*
* @note This update is not committed to disk until commit() is called.
*/
- def merge(key: Array[Byte], value: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val oldValue = db.get(readOptions, key)
- if (oldValue == null) {
- numKeysOnWritingVersion += 1
+ def merge(
+ key: Array[Byte],
+ value: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion += 1
+ } else {
+ numKeysOnWritingVersion += 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ numKeysOnWritingVersion += 1
+ }
}
}
- db.merge(writeOptions, key, value)
- changelogWriter.foreach(_.merge(key, value))
+ db.merge(writeOptions, keyWithPrefix, value)
+ changelogWriter.foreach(_.merge(keyWithPrefix, value))
}
/**
* Remove the key if present.
* @note This update is not committed to disk until commit() is called.
*/
- def remove(key: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val value = db.get(readOptions, key)
- if (value != null) {
- numKeysOnWritingVersion -= 1
+ def remove(key: Array[Byte], cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue != null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion -= 1
+ } else {
+ numKeysOnWritingVersion -= 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val value = db.get(readOptions, keyWithPrefix)
+ if (value != null) {
+ numKeysOnWritingVersion -= 1
+ }
}
}
- db.delete(writeOptions, key)
- changelogWriter.foreach(_.delete(key))
+
+ db.delete(writeOptions, keyWithPrefix)
+ changelogWriter.foreach(_.delete(keyWithPrefix))
}
/**
Review Comment:
I'm saying that UX isn't changed quite differently for all other methods except this. Most methods are changed to be aware of column family, while this isn't.
For multiple column families case, we do not expect global scan for the entire key space, so this method is mostly unused. They are now having to use `prefixScan()` when they were using `iterator()`, which relies on implementation details how we construct virtual column family. I'd rather say, it'd be better to have column family aware version of `iterator()`.
I'd leave it to you to think which is better 1) having both `iterator()` and `iterator(cfName: String)`, and leave `iterator()` to perform full scan regardless of multiple column families, 2) only having `iterator(cfName: String)`, and whether to full scan or just delegate to `prefixScan(cfName)` based on internal flag (multiple column families).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "ericm-db (via GitHub)" <gi...@apache.org>.
ericm-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1901291265
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBFileManager.scala:
##########
@@ -931,7 +941,8 @@ case class RocksDBCheckpointMetadata(
sstFiles: Seq[RocksDBSstFile],
logFiles: Seq[RocksDBLogFile],
numKeys: Long,
- columnFamilyMapping: Option[Map[String, Short]] = None,
+ numInternalKeys: Long,
+ columnFamilyMapping: Option[Map[String, ColumnFamilyInfo]] = None,
Review Comment:
Why None instead of Map.empty?
Is the test change too invasive?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1901303840
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBFileManager.scala:
##########
@@ -931,7 +941,8 @@ case class RocksDBCheckpointMetadata(
sstFiles: Seq[RocksDBSstFile],
logFiles: Seq[RocksDBLogFile],
numKeys: Long,
- columnFamilyMapping: Option[Map[String, Short]] = None,
+ numInternalKeys: Long,
+ columnFamilyMapping: Option[Map[String, ColumnFamilyInfo]] = None,
Review Comment:
Dont think I changed this
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on PR #49304:
URL: https://github.com/apache/spark/pull/49304#issuecomment-2568472641
@HeartSaVioR - could you PTAL, thanks !
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1959235264
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -178,55 +183,77 @@ class RocksDB(
// This is accessed and updated only between load and commit
// which means it is implicitly guarded by acquireLock
@GuardedBy("acquireLock")
- private val colFamilyNameToIdMap = new ConcurrentHashMap[String, Short]()
+ private val colFamilyNameToInfoMap = new ConcurrentHashMap[String, ColumnFamilyInfo]()
+
+ @GuardedBy("acquireLock")
+ private val colFamilyIdToNameMap = new ConcurrentHashMap[Short, String]()
@GuardedBy("acquireLock")
private val maxColumnFamilyId: AtomicInteger = new AtomicInteger(-1)
@GuardedBy("acquireLock")
private val shouldForceSnapshot: AtomicBoolean = new AtomicBoolean(false)
- /**
- * Check whether the column family name is for internal column families.
- *
- * @param cfName - column family name
- * @return - true if the column family is for internal use, false otherwise
- */
- private def checkInternalColumnFamilies(cfName: String): Boolean = cfName.charAt(0) == '_'
+ private def getColumnFamilyInfo(cfName: String): ColumnFamilyInfo = {
+ colFamilyNameToInfoMap.get(cfName)
+ }
+
+ private def getColumnFamilyNameForId(cfId: Short): String = {
+ colFamilyIdToNameMap.get(cfId)
+ }
- // Methods to fetch column family mapping for this State Store version
- def getColumnFamilyMapping: Map[String, Short] = {
- colFamilyNameToIdMap.asScala
+ private def addToColFamilyMaps(cfName: String, cfId: Short, isInternal: Boolean): Unit = {
+ colFamilyNameToInfoMap.putIfAbsent(cfName, ColumnFamilyInfo(cfId, isInternal))
+ colFamilyIdToNameMap.putIfAbsent(cfId, cfName)
+ }
+
+ private def removeFromColFamilyMaps(cfName: String): Unit = {
+ val colFamilyInfo = colFamilyNameToInfoMap.get(cfName)
+ if (colFamilyInfo != null) {
+ colFamilyNameToInfoMap.remove(cfName)
+ colFamilyIdToNameMap.remove(colFamilyInfo.cfId)
+ }
}
- def getColumnFamilyId(cfName: String): Short = {
- colFamilyNameToIdMap.get(cfName)
+ private def clearColFamilyMaps(): Unit = {
+ colFamilyNameToInfoMap.clear()
+ colFamilyIdToNameMap.clear()
}
/**
- * Create RocksDB column family, if not created already
+ * Check if the column family exists with given name and create one if it doesn't. Users can
+ * create external column families storing user facing data as well as internal column families
+ * such as secondary indexes. Metrics for both of these types are tracked separately.
+ *
+ * @param colFamilyName - column family name
+ * @param isInternal - whether the column family is for internal use or not
+ * @return - virtual column family id
*/
- def createColFamilyIfAbsent(colFamilyName: String): Short = {
+ def createColFamilyIfAbsent(colFamilyName: String, isInternal: Boolean): Short = {
if (!checkColFamilyExists(colFamilyName)) {
val newColumnFamilyId = maxColumnFamilyId.incrementAndGet().toShort
- colFamilyNameToIdMap.putIfAbsent(colFamilyName, newColumnFamilyId)
+ addToColFamilyMaps(colFamilyName, newColumnFamilyId, isInternal)
shouldForceSnapshot.set(true)
newColumnFamilyId
} else {
- colFamilyNameToIdMap.get(colFamilyName)
+ colFamilyNameToInfoMap.get(colFamilyName).cfId
}
}
/**
* Remove RocksDB column family, if exists
* @return columnFamilyId if it exists, else None
*/
- def removeColFamilyIfExists(colFamilyName: String): Option[Short] = {
+ def removeColFamilyIfExists(colFamilyName: String): Boolean = {
if (checkColFamilyExists(colFamilyName)) {
shouldForceSnapshot.set(true)
- Some(colFamilyNameToIdMap.remove(colFamilyName))
+ prefixScan(Array.empty[Byte], colFamilyName).foreach { kv =>
Review Comment:
Done
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -467,29 +492,62 @@ class RocksDB(
this
}
+ /**
+ * Function to check if col family is internal or not based on information recorded in
+ * checkpoint metadata.
+ * @param cfName - column family name
+ * @param metadata - checkpoint metadata
+ * @return - type of column family (internal or otherwise)
+ */
+ private def checkColFamilyType(
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1960627000
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -700,7 +901,13 @@ class RocksDB(
new NextIterator[ByteArrayPair] {
override protected def getNext(): ByteArrayPair = {
if (iter.isValid) {
- byteArrayPair.set(iter.key, iter.value)
+ val key = if (useColumnFamilies) {
+ decodeStateRowWithPrefix(iter.key)._1
Review Comment:
have added the new iterator API now. so I guess we should be good here ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1960635805
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -902,7 +946,13 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
db.remove((version - 1).toString)
db.commit()
}
- assert(snapshotVersionsPresent(remoteDir) === (1 to 30))
+
+ if (enableStateStoreCheckpointIds && colFamiliesEnabled) {
+ assert(snapshotVersionsPresent(remoteDir) === (1 to 30) :+ 30 :+ 31)
Review Comment:
yes - added a comment
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "HeartSaVioR (via GitHub)" <gi...@apache.org>.
HeartSaVioR closed pull request #49304: [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder
URL: https://github.com/apache/spark/pull/49304
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1946050169
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -237,22 +272,22 @@ class RocksDB(
* @return - true if the column family exists, false otherwise
*/
def checkColFamilyExists(colFamilyName: String): Boolean = {
- colFamilyNameToIdMap.containsKey(colFamilyName)
+ db != null && colFamilyNameToInfoMap.containsKey(colFamilyName)
Review Comment:
IIRC it was for a test case where the db was being reset but entries could still be present in the map
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1946041129
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala:
##########
@@ -702,6 +696,8 @@ object StateStore extends Logging {
val DEFAULT_COL_FAMILY_NAME = "default"
+ val VIRTUAL_COL_FAMILY_PREFIX_BYTES = 2
Review Comment:
We need to use it in the provider above right ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "ericm-db (via GitHub)" <gi...@apache.org>.
ericm-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1901294730
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -723,20 +888,42 @@ class RocksDB(
iter.seekToFirst()
var keys = 0L
- while (iter.isValid) {
- keys += 1
- iter.next()
+ var internalKeys = 0L
+
+ if (!useColumnFamilies) {
+ while (iter.isValid) {
+ keys += 1
+ iter.next()
+ }
+ } else {
+ while (iter.isValid) {
+ val (_, cfName) = decodeStateRowWithPrefix(iter.key)
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ internalKeys += 1
+ } else {
+ keys += 1
+ }
+ iter.next()
+ }
}
- keys
+ (keys, internalKeys)
} finally {
iter.close()
}
}
- def prefixScan(prefix: Array[Byte]): Iterator[ByteArrayPair] = {
+ def prefixScan(
+ prefix: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Iterator[ByteArrayPair] = {
val iter = db.newIterator()
- iter.seek(prefix)
+ var updatedPrefix = prefix
Review Comment:
```suggestion
val updatedPrefix = if (useColumnFamilies) {
encodeStateRowWithPrefix(prefix, cfName)
} else {
prefix
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1924463394
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -194,39 +202,60 @@ class RocksDB(
*/
private def checkInternalColumnFamilies(cfName: String): Boolean = cfName.charAt(0) == '_'
- // Methods to fetch column family mapping for this State Store version
- def getColumnFamilyMapping: Map[String, Short] = {
- colFamilyNameToIdMap.asScala
+ private def getColumnFamilyInfo(cfName: String): ColumnFamilyInfo = {
+ colFamilyNameToInfoMap.get(cfName)
+ }
+
+ private def getColumnFamilyNameForId(cfId: Short): String = {
+ colFamilyIdToNameMap.get(cfId)
}
- def getColumnFamilyId(cfName: String): Short = {
- colFamilyNameToIdMap.get(cfName)
+ private def addToColFamilyMaps(cfName: String, cfId: Short, isInternal: Boolean): Unit = {
+ colFamilyNameToInfoMap.putIfAbsent(cfName, ColumnFamilyInfo(cfId, isInternal))
+ colFamilyIdToNameMap.putIfAbsent(cfId, cfName)
+ }
+
+ private def removeFromColFamilyMaps(cfName: String): Unit = {
+ val colFamilyInfo = colFamilyNameToInfoMap.get(cfName)
+ if (colFamilyInfo != null) {
+ colFamilyNameToInfoMap.remove(cfName)
+ colFamilyIdToNameMap.remove(colFamilyInfo.cfId)
+ }
+ }
+
+ private def clearColFamilyMaps(): Unit = {
+ colFamilyNameToInfoMap.clear()
+ colFamilyIdToNameMap.clear()
}
/**
* Create RocksDB column family, if not created already
*/
- def createColFamilyIfAbsent(colFamilyName: String): Short = {
+ def createColFamilyIfAbsent(colFamilyName: String, isInternal: Boolean): Short = {
Review Comment:
Added some comments around this
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "micheal-o (via GitHub)" <gi...@apache.org>.
micheal-o commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1919541380
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -194,39 +202,60 @@ class RocksDB(
*/
private def checkInternalColumnFamilies(cfName: String): Boolean = cfName.charAt(0) == '_'
- // Methods to fetch column family mapping for this State Store version
- def getColumnFamilyMapping: Map[String, Short] = {
- colFamilyNameToIdMap.asScala
+ private def getColumnFamilyInfo(cfName: String): ColumnFamilyInfo = {
+ colFamilyNameToInfoMap.get(cfName)
+ }
+
+ private def getColumnFamilyNameForId(cfId: Short): String = {
+ colFamilyIdToNameMap.get(cfId)
}
- def getColumnFamilyId(cfName: String): Short = {
- colFamilyNameToIdMap.get(cfName)
+ private def addToColFamilyMaps(cfName: String, cfId: Short, isInternal: Boolean): Unit = {
+ colFamilyNameToInfoMap.putIfAbsent(cfName, ColumnFamilyInfo(cfId, isInternal))
+ colFamilyIdToNameMap.putIfAbsent(cfId, cfName)
+ }
+
+ private def removeFromColFamilyMaps(cfName: String): Unit = {
+ val colFamilyInfo = colFamilyNameToInfoMap.get(cfName)
+ if (colFamilyInfo != null) {
+ colFamilyNameToInfoMap.remove(cfName)
+ colFamilyIdToNameMap.remove(colFamilyInfo.cfId)
+ }
+ }
+
+ private def clearColFamilyMaps(): Unit = {
+ colFamilyNameToInfoMap.clear()
+ colFamilyIdToNameMap.clear()
}
/**
* Create RocksDB column family, if not created already
*/
- def createColFamilyIfAbsent(colFamilyName: String): Short = {
+ def createColFamilyIfAbsent(colFamilyName: String, isInternal: Boolean): Short = {
Review Comment:
nit: Should we also have a comment for this param, telling what it does?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1960643780
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -1074,14 +1140,25 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures with SharedSparkSession
db.load(version, versionToUniqueId.get(version))
assert(db.iterator().map(toStr).toSet === Set((version.toString, version.toString)))
}
+
for (version <- 31 to 60) {
db.load(version - 1, versionToUniqueId.get(version - 1))
db.put(version.toString, version.toString)
db.remove((version - 1).toString)
db.commit()
}
assert(changelogVersionsPresent(remoteDir) === (1 to 30))
- assert(snapshotVersionsPresent(remoteDir) === (31 to 60))
+
+ var result: Seq[Long] = if (colFamiliesEnabled) {
+ Seq(1)
+ } else {
+ Seq.empty
+ }
+
+ (31 to 60).foreach { i =>
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1960599849
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala:
##########
@@ -702,6 +696,8 @@ object StateStore extends Logging {
val DEFAULT_COL_FAMILY_NAME = "default"
+ val VIRTUAL_COL_FAMILY_PREFIX_BYTES = 2
Review Comment:
Done
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala:
##########
@@ -678,7 +654,7 @@ object RocksDBStateStoreProvider {
// Version as a single byte that specifies the encoding of the row data in RocksDB
val STATE_ENCODING_NUM_VERSION_BYTES = 1
val STATE_ENCODING_VERSION: Byte = 0
- val VIRTUAL_COL_FAMILY_PREFIX_BYTES = 2
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1960644385
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateEncoder.scala:
##########
@@ -1791,16 +1644,12 @@ class RangeKeyScanStateEncoder(
class NoPrefixKeyStateEncoder(
dataEncoder: RocksDBDataEncoder,
keySchema: StructType,
- useColumnFamilies: Boolean = false,
- columnFamilyInfo: Option[ColumnFamilyInfo] = None)
- extends StateRowPrefixEncoder(
- useColumnFamilies,
- columnFamilyInfo
- ) with RocksDBKeyStateEncoder with Logging {
+ useColumnFamilies: Boolean = false)
+ extends RocksDBKeyStateEncoder with Logging {
override def encodeKey(row: UnsafeRow): Array[Byte] = {
if (!useColumnFamilies) {
Review Comment:
Yes as discussed - this does seem like a bug. So we will retain this behavior as it is and do the version bump change later
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1960950946
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateEncoder.scala:
##########
@@ -1814,18 +1663,17 @@ class NoPrefixKeyStateEncoder(
rowBytes.length
)
- encodeStateRowWithPrefix(dataWithVersion)
+ dataWithVersion
}
}
override def decodeKey(keyBytes: Array[Byte]): UnsafeRow = {
if (!useColumnFamilies) {
- dataEncoder.decodeKey(decodeStateRowData(keyBytes))
+ dataEncoder.decodeKey(keyBytes)
Review Comment:
Same as above
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "HeartSaVioR (via GitHub)" <gi...@apache.org>.
HeartSaVioR commented on PR #49304:
URL: https://github.com/apache/spark/pull/49304#issuecomment-2668513553
Thanks! Merging to master/4.0.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1941762050
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBFileManager.scala:
##########
@@ -923,6 +924,15 @@ object RocksDBFileManagerMetrics {
val EMPTY_METRICS = RocksDBFileManagerMetrics(0L, 0L, 0L, None)
}
+/**
+ * Case class to keep track of column family info within checkpoint metadata.
+ * @param cfId - virtual column family id
+ * @param isInternal - whether the column family is internal or not
+ */
+case class ColumnFamilyInfo(
+ cfId: Short,
+ isInternal: Boolean)
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on PR #49304:
URL: https://github.com/apache/spark/pull/49304#issuecomment-2605906209
> Would you mind fixing conflicts? Thanks!
@HeartSaVioR - done ! PTAL
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1955675678
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBFileManager.scala:
##########
@@ -931,7 +941,8 @@ case class RocksDBCheckpointMetadata(
sstFiles: Seq[RocksDBSstFile],
logFiles: Seq[RocksDBLogFile],
numKeys: Long,
- columnFamilyMapping: Option[Map[String, Short]] = None,
+ numInternalKeys: Long,
+ columnFamilyMapping: Option[Map[String, ColumnFamilyInfo]] = None,
Review Comment:
Done - added format validation tests for serde
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1960600195
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala:
##########
@@ -847,35 +828,47 @@ class RocksDBStateStoreChangeDataReader(
endVersion: Long,
compressionCodec: CompressionCodec,
keyValueEncoderMap:
- ConcurrentHashMap[String, (RocksDBKeyStateEncoder, RocksDBValueStateEncoder)],
+ ConcurrentHashMap[String, (RocksDBKeyStateEncoder, RocksDBValueStateEncoder, Short)],
colFamilyNameOpt: Option[String] = None)
extends StateStoreChangeDataReader(
fm, stateLocation, startVersion, endVersion, compressionCodec, colFamilyNameOpt) {
override protected var changelogSuffix: String = "changelog"
- private def getColFamilyIdBytes: Option[Array[Byte]] = {
- if (colFamilyNameOpt.isDefined) {
- val colFamilyName = colFamilyNameOpt.get
- if (!keyValueEncoderMap.containsKey(colFamilyName)) {
- throw new IllegalStateException(
- s"Column family $colFamilyName not found in the key value encoder map")
- }
- Some(keyValueEncoderMap.get(colFamilyName)._1.getColumnFamilyIdBytes())
- } else {
- None
- }
+ /**
+ * Encodes a virtual column family ID into a byte array suitable for RocksDB.
+ *
+ * This method creates a fixed-size byte array prefixed with the virtual column family ID,
+ * which is used to partition data within RocksDB.
+ *
+ * @param virtualColFamilyId The column family identifier to encode
+ * @return A byte array containing the encoded column family ID
+ */
+ private def getColumnFamilyIdBytes(virtualColFamilyId: Short): Array[Byte] = {
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "HeartSaVioR (via GitHub)" <gi...@apache.org>.
HeartSaVioR commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1957574278
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -656,31 +803,75 @@ class RocksDB(
*
* @note This update is not committed to disk until commit() is called.
*/
- def merge(key: Array[Byte], value: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val oldValue = db.get(readOptions, key)
- if (oldValue == null) {
- numKeysOnWritingVersion += 1
+ def merge(
+ key: Array[Byte],
+ value: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion += 1
+ } else {
+ numKeysOnWritingVersion += 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ numKeysOnWritingVersion += 1
+ }
}
}
- db.merge(writeOptions, key, value)
- changelogWriter.foreach(_.merge(key, value))
+ db.merge(writeOptions, keyWithPrefix, value)
+ changelogWriter.foreach(_.merge(keyWithPrefix, value))
}
/**
* Remove the key if present.
* @note This update is not committed to disk until commit() is called.
*/
- def remove(key: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val value = db.get(readOptions, key)
- if (value != null) {
- numKeysOnWritingVersion -= 1
+ def remove(key: Array[Byte], cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue != null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion -= 1
+ } else {
+ numKeysOnWritingVersion -= 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val value = db.get(readOptions, keyWithPrefix)
+ if (value != null) {
+ numKeysOnWritingVersion -= 1
+ }
}
}
- db.delete(writeOptions, key)
- changelogWriter.foreach(_.delete(key))
+
+ db.delete(writeOptions, keyWithPrefix)
+ changelogWriter.foreach(_.delete(keyWithPrefix))
}
/**
Review Comment:
I'm saying that UX isn't changed quite differently for all other methods except this. Most methods are changed to be aware of column family, while this isn't.
For multiple column families case, we do not expect global scan for the entire key space, so this method is mostly unused. They are now having to use `prefixScan()` when they were using `iterator()`, which relies on implementation details how we construct virtual column family (and prefixScan wasn't intended to perform scan without actual prefix data). I'd rather say, it'd be better to have column family aware version of `iterator()`.
I'd leave it to you to think which is better 1) having both `iterator()` and `iterator(cfName: String)`, and leave `iterator()` to perform full scan regardless of multiple column families, 2) only having `iterator(cfName: String)`, and whether to full scan or just delegate to `prefixScan(cfName)` based on internal flag (multiple column families).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "HeartSaVioR (via GitHub)" <gi...@apache.org>.
HeartSaVioR commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1957584484
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateEncoder.scala:
##########
@@ -1814,18 +1663,17 @@ class NoPrefixKeyStateEncoder(
rowBytes.length
)
- encodeStateRowWithPrefix(dataWithVersion)
+ dataWithVersion
}
}
override def decodeKey(keyBytes: Array[Byte]): UnsafeRow = {
if (!useColumnFamilies) {
- dataEncoder.decodeKey(decodeStateRowData(keyBytes))
+ dataEncoder.decodeKey(keyBytes)
Review Comment:
ditto if the above is turned out to be a bug.
If the above is a bug, this cannot read from existing checkpoint from prior Spark versions. (Though probably we didn't make release of multiple column family?)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1960626246
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -621,28 +700,106 @@ class RocksDB(
}
}
+ /**
+ * Function to encode state row with virtual col family id prefix
+ * @param data - passed byte array to be stored in state store
+ * @param cfName - name of column family
+ * @return - encoded byte array with virtual column family id prefix
+ */
+ private def encodeStateRowWithPrefix(
+ data: Array[Byte],
+ cfName: String): Array[Byte] = {
+ // Create result array big enough for all prefixes plus data
+ val result = new Array[Byte](StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES + data.length)
+ val offset = Platform.BYTE_ARRAY_OFFSET + StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES
+
+ val cfInfo = getColumnFamilyInfo(cfName)
+ Platform.putShort(result, Platform.BYTE_ARRAY_OFFSET, cfInfo.cfId)
+
+ // Write the actual data
+ Platform.copyMemory(
+ data, Platform.BYTE_ARRAY_OFFSET,
+ result, offset,
+ data.length
+ )
+
+ result
+ }
+
+ /**
+ * Function to decode state row with virtual col family id prefix
+ * @param data - passed byte array retrieved from state store
+ * @return - pair of decoded byte array without virtual column family id prefix
+ * and name of column family
+ */
+ private def decodeStateRowWithPrefix(data: Array[Byte]): (Array[Byte], String) = {
+ val cfId = Platform.getShort(data, Platform.BYTE_ARRAY_OFFSET)
+ val cfName = getColumnFamilyNameForId(cfId)
+ val offset = Platform.BYTE_ARRAY_OFFSET + StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES
+
+ val key = new Array[Byte](data.length - StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES)
+ Platform.copyMemory(
+ data, offset,
+ key, Platform.BYTE_ARRAY_OFFSET,
+ key.length
+ )
+
+ (key, cfName)
+ }
+
/**
* Get the value for the given key if present, or null.
* @note This will return the last written value even if it was uncommitted.
*/
- def get(key: Array[Byte]): Array[Byte] = {
- db.get(readOptions, key)
+ def get(
+ key: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Array[Byte] = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ db.get(readOptions, keyWithPrefix)
}
/**
* Put the given value for the given key.
* @note This update is not committed to disk until commit() is called.
*/
- def put(key: Array[Byte], value: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val oldValue = db.get(readOptions, key)
- if (oldValue == null) {
- numKeysOnWritingVersion += 1
+ def put(
+ key: Array[Byte],
+ value: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
Review Comment:
Done
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -656,31 +813,75 @@ class RocksDB(
*
* @note This update is not committed to disk until commit() is called.
*/
- def merge(key: Array[Byte], value: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val oldValue = db.get(readOptions, key)
- if (oldValue == null) {
- numKeysOnWritingVersion += 1
+ def merge(
+ key: Array[Byte],
+ value: Array[Byte],
+ cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ val cfInfo = getColumnFamilyInfo(cfName)
+ if (cfInfo.isInternal) {
+ numInternalKeysOnWritingVersion += 1
+ } else {
+ numKeysOnWritingVersion += 1
+ }
+ }
+ }
+ } else {
+ if (conf.trackTotalNumberOfRows) {
+ val oldValue = db.get(readOptions, keyWithPrefix)
+ if (oldValue == null) {
+ numKeysOnWritingVersion += 1
+ }
}
}
- db.merge(writeOptions, key, value)
- changelogWriter.foreach(_.merge(key, value))
+ db.merge(writeOptions, keyWithPrefix, value)
+ changelogWriter.foreach(_.merge(keyWithPrefix, value))
}
/**
* Remove the key if present.
* @note This update is not committed to disk until commit() is called.
*/
- def remove(key: Array[Byte]): Unit = {
- if (conf.trackTotalNumberOfRows) {
- val value = db.get(readOptions, key)
- if (value != null) {
- numKeysOnWritingVersion -= 1
+ def remove(key: Array[Byte], cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): Unit = {
+ val keyWithPrefix = if (useColumnFamilies) {
+ encodeStateRowWithPrefix(key, cfName)
+ } else {
+ key
+ }
+
+ if (useColumnFamilies) {
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "anishshri-db (via GitHub)" <gi...@apache.org>.
anishshri-db commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1959234822
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala:
##########
@@ -847,35 +828,47 @@ class RocksDBStateStoreChangeDataReader(
endVersion: Long,
compressionCodec: CompressionCodec,
keyValueEncoderMap:
- ConcurrentHashMap[String, (RocksDBKeyStateEncoder, RocksDBValueStateEncoder)],
+ ConcurrentHashMap[String, (RocksDBKeyStateEncoder, RocksDBValueStateEncoder, Short)],
colFamilyNameOpt: Option[String] = None)
extends StateStoreChangeDataReader(
fm, stateLocation, startVersion, endVersion, compressionCodec, colFamilyNameOpt) {
override protected var changelogSuffix: String = "changelog"
- private def getColFamilyIdBytes: Option[Array[Byte]] = {
- if (colFamilyNameOpt.isDefined) {
- val colFamilyName = colFamilyNameOpt.get
- if (!keyValueEncoderMap.containsKey(colFamilyName)) {
- throw new IllegalStateException(
- s"Column family $colFamilyName not found in the key value encoder map")
- }
- Some(keyValueEncoderMap.get(colFamilyName)._1.getColumnFamilyIdBytes())
- } else {
- None
- }
+ /**
+ * Encodes a virtual column family ID into a byte array suitable for RocksDB.
+ *
+ * This method creates a fixed-size byte array prefixed with the virtual column family ID,
+ * which is used to partition data within RocksDB.
+ *
+ * @param virtualColFamilyId The column family identifier to encode
+ * @return A byte array containing the encoded column family ID
+ */
+ private def getColumnFamilyIdBytes(virtualColFamilyId: Short): Array[Byte] = {
Review Comment:
I dont think I can consolidate without some additional refactoring. Will prefer to do this in a separate PR
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala:
##########
@@ -847,35 +828,47 @@ class RocksDBStateStoreChangeDataReader(
endVersion: Long,
compressionCodec: CompressionCodec,
keyValueEncoderMap:
- ConcurrentHashMap[String, (RocksDBKeyStateEncoder, RocksDBValueStateEncoder)],
+ ConcurrentHashMap[String, (RocksDBKeyStateEncoder, RocksDBValueStateEncoder, Short)],
colFamilyNameOpt: Option[String] = None)
extends StateStoreChangeDataReader(
fm, stateLocation, startVersion, endVersion, compressionCodec, colFamilyNameOpt) {
override protected var changelogSuffix: String = "changelog"
- private def getColFamilyIdBytes: Option[Array[Byte]] = {
- if (colFamilyNameOpt.isDefined) {
- val colFamilyName = colFamilyNameOpt.get
- if (!keyValueEncoderMap.containsKey(colFamilyName)) {
- throw new IllegalStateException(
- s"Column family $colFamilyName not found in the key value encoder map")
- }
- Some(keyValueEncoderMap.get(colFamilyName)._1.getColumnFamilyIdBytes())
- } else {
- None
- }
+ /**
+ * Encodes a virtual column family ID into a byte array suitable for RocksDB.
+ *
+ * This method creates a fixed-size byte array prefixed with the virtual column family ID,
+ * which is used to partition data within RocksDB.
+ *
+ * @param virtualColFamilyId The column family identifier to encode
+ * @return A byte array containing the encoded column family ID
+ */
+ private def getColumnFamilyIdBytes(virtualColFamilyId: Short): Array[Byte] = {
+ val encodedBytes = new Array[Byte](StateStore.VIRTUAL_COL_FAMILY_PREFIX_BYTES)
+ Platform.putShort(encodedBytes, Platform.BYTE_ARRAY_OFFSET, virtualColFamilyId)
+ encodedBytes
}
- private val colFamilyIdBytesOpt: Option[Array[Byte]] = getColFamilyIdBytes
+ private def getExtractedKey(data: Array[Byte]): Array[Byte] = {
Review Comment:
Same here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "HeartSaVioR (via GitHub)" <gi...@apache.org>.
HeartSaVioR commented on PR #49304:
URL: https://github.com/apache/spark/pull/49304#issuecomment-2597377563
@anishshri-db Would you mind fixing conflicts? Thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
Re: [PR] [SPARK-50655][SS] Move virtual col family related mapping into db layer instead of encoder [spark]
Posted by "micheal-o (via GitHub)" <gi...@apache.org>.
micheal-o commented on code in PR #49304:
URL: https://github.com/apache/spark/pull/49304#discussion_r1919539767
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -194,39 +202,60 @@ class RocksDB(
*/
private def checkInternalColumnFamilies(cfName: String): Boolean = cfName.charAt(0) == '_'
- // Methods to fetch column family mapping for this State Store version
- def getColumnFamilyMapping: Map[String, Short] = {
- colFamilyNameToIdMap.asScala
+ private def getColumnFamilyInfo(cfName: String): ColumnFamilyInfo = {
+ colFamilyNameToInfoMap.get(cfName)
+ }
+
+ private def getColumnFamilyNameForId(cfId: Short): String = {
+ colFamilyIdToNameMap.get(cfId)
}
- def getColumnFamilyId(cfName: String): Short = {
- colFamilyNameToIdMap.get(cfName)
+ private def addToColFamilyMaps(cfName: String, cfId: Short, isInternal: Boolean): Unit = {
+ colFamilyNameToInfoMap.putIfAbsent(cfName, ColumnFamilyInfo(cfId, isInternal))
+ colFamilyIdToNameMap.putIfAbsent(cfId, cfName)
+ }
+
+ private def removeFromColFamilyMaps(cfName: String): Unit = {
+ val colFamilyInfo = colFamilyNameToInfoMap.get(cfName)
+ if (colFamilyInfo != null) {
+ colFamilyNameToInfoMap.remove(cfName)
+ colFamilyIdToNameMap.remove(colFamilyInfo.cfId)
+ }
+ }
+
+ private def clearColFamilyMaps(): Unit = {
+ colFamilyNameToInfoMap.clear()
+ colFamilyIdToNameMap.clear()
}
/**
* Create RocksDB column family, if not created already
*/
- def createColFamilyIfAbsent(colFamilyName: String): Short = {
+ def createColFamilyIfAbsent(colFamilyName: String, isInternal: Boolean): Short = {
Review Comment:
nit: why are we calling this `isInternal`? This is a public method and it isn't obvious what this means. Do you mean `isVirtual` here? If so, why not be explicit with the name?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org