You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xu...@apache.org on 2021/12/06 08:07:11 UTC
[iotdb] branch xkf_id_table updated: improve
This is an automated email from the ASF dual-hosted git repository.
xuekaifeng pushed a commit to branch xkf_id_table
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/xkf_id_table by this push:
new fa034b5 improve
fa034b5 is described below
commit fa034b5181c15fbbd192fd4c9ca39d3def3fe287
Author: 151250176 <15...@smail.nju.edu.cn>
AuthorDate: Mon Dec 6 16:05:36 2021 +0800
improve
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 22 ++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 27 ++--
.../iotdb/db/metadata/id_table/IDManager.java | 11 +-
.../iotdb/db/metadata/id_table/IDManagerImpl.java | 5 +-
.../apache/iotdb/db/metadata/id_table/IDTable.java | 148 +++++++++++++++++++--
.../db/metadata/id_table/entry/DeviceEntry.java | 26 ++++
.../metadata/id_table/entry/DeviceIDFactory.java | 70 ++++++++++
.../db/metadata/id_table/entry/PlainDeviceID.java | 5 +
.../db/metadata/id_table/entry/SHA256DeviceID.java | 38 ++++++
.../db/metadata/id_table/entry/SchemaEntry.java | 75 ++++++++++-
.../db/metadata/id_table/entry/TimeseriesID.java | 27 +++-
.../iotdb/db/metadata/id_table/IDManagerTest.java | 5 +
.../iotdb/db/metadata/id_table/IDTableTest.java | 5 +
.../db/metadata/id_table/entry/DeviceIDTest.java | 44 ++++++
.../metadata/id_table/entry/SchemaEntryTest.java | 71 ++++++++++
15 files changed, 541 insertions(+), 38 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index d234205..a007cc1 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -18,6 +18,11 @@
*/
package org.apache.iotdb.db.conf;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
+
+import java.io.File;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.compaction.CompactionPriority;
import org.apache.iotdb.db.engine.compaction.cross.CrossCompactionStrategy;
@@ -34,16 +39,9 @@ import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.fileSystem.FSType;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
-
public class IoTDBConfig {
/* Names of Watermark methods */
@@ -770,6 +768,8 @@ public class IoTDBConfig {
private String adminPassword = "root";
+ private String deviceIDTransformationMethod = "Plain";
+
public IoTDBConfig() {
// empty constructor
}
@@ -2437,4 +2437,12 @@ public class IoTDBConfig {
public void setCompactionSubmissionInterval(long interval) {
compactionSubmissionInterval = interval;
}
+
+ public String getDeviceIDTransformationMethod() {
+ return deviceIDTransformationMethod;
+ }
+
+ public void setDeviceIDTransformationMethod(String deviceIDTransformationMethod) {
+ this.deviceIDTransformationMethod = deviceIDTransformationMethod;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 17deb09..bbf74ac 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -18,6 +18,16 @@
*/
package org.apache.iotdb.db.conf;
+import com.google.common.net.InetAddresses;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.Properties;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.compaction.CompactionPriority;
@@ -31,21 +41,9 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.fileSystem.FSType;
import org.apache.iotdb.tsfile.utils.FilePathUtils;
-
-import com.google.common.net.InetAddresses;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetAddress;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.util.Properties;
-
public class IoTDBDescriptor {
private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
@@ -708,6 +706,11 @@ public class IoTDBDescriptor {
conf.setMaxPendingWindowEvaluationTasks(64);
}
+ // device id transformation method
+ conf.setDeviceIDTransformationMethod(
+ properties.getProperty(
+ "device_id_transformation_method", conf.getDeviceIDTransformationMethod()));
+
// mqtt
if (properties.getProperty(IoTDBConstant.MQTT_HOST_NAME) != null) {
conf.setMqttHost(properties.getProperty(IoTDBConstant.MQTT_HOST_NAME));
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManager.java
index 3bef50a..9cc5406 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManager.java
@@ -18,8 +18,10 @@
*/
package org.apache.iotdb.db.metadata.id_table;
-import com.sun.tools.javac.util.Pair;
+import org.apache.iotdb.db.metadata.id_table.entry.IDeviceID;
import org.apache.iotdb.db.metadata.id_table.entry.TimeseriesID;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+import org.apache.iotdb.tsfile.utils.Pair;
public interface IDManager {
@@ -30,7 +32,7 @@ public interface IDManager {
* @param seriesKey full path of the time series
* @return timeseries ID of this time series
*/
- public TimeseriesID checkOrCreateIfNotExist(String seriesKey);
+ public TimeseriesID checkOrCreateIfNotExist(PartialPath seriesKey);
/**
* upatde latest flushed time of one timeseries
@@ -63,4 +65,9 @@ public interface IDManager {
* @param lastTimeValue latest time value pair of one timeseries
*/
public void updateLastTimeValuePair(TimeseriesID timeseriesID, Pair<Long, Object> lastTimeValue);
+
+ public static IDeviceID getDeviceID() {
+
+ return null;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManagerImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManagerImpl.java
index 68b537e..46babf5 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManagerImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDManagerImpl.java
@@ -19,9 +19,10 @@
package org.apache.iotdb.db.metadata.id_table;
-import com.sun.tools.javac.util.Pair;
import java.util.Map;
import org.apache.iotdb.db.metadata.id_table.entry.TimeseriesID;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+import org.apache.iotdb.tsfile.utils.Pair;
public class IDManagerImpl implements IDManager {
@@ -36,7 +37,7 @@ public class IDManagerImpl implements IDManager {
* @return timeseries ID of this time series
*/
@Override
- public TimeseriesID checkOrCreateIfNotExist(String seriesKey) {
+ public TimeseriesID checkOrCreateIfNotExist(PartialPath seriesKey) {
return null;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java
index 60ecf30..590f2243 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java
@@ -19,48 +19,174 @@
package org.apache.iotdb.db.metadata.id_table;
+import java.util.HashMap;
import java.util.Map;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.metadata.id_table.entry.DeviceEntry;
import org.apache.iotdb.db.metadata.id_table.entry.IDeviceID;
+import org.apache.iotdb.db.metadata.id_table.entry.SchemaEntry;
import org.apache.iotdb.db.metadata.id_table.entry.TimeseriesID;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+/** id table belongs to a storage group and mapping timeseries path to it's schema */
public class IDTable {
+ // number of table slot
+ private static final int NUM_OF_SLOTS = 256;
+ /** logger */
+ private static final Logger logger = LoggerFactory.getLogger(IDTable.class);
/**
* 256 hashmap for avoiding rehash performance issue and lock competition device ID ->
* (measurement name -> schema entry)
*/
private Map<IDeviceID, DeviceEntry>[] idTables;
-
/** disk schema manager to manage disk schema entry */
private DiskSchemaManager diskSchemaManager;
+ public IDTable() {
+ idTables = new Map[NUM_OF_SLOTS];
+ for (int i = 0; i < NUM_OF_SLOTS; i++) {
+ idTables[i] = new HashMap<>();
+ }
+ }
+
/**
* check whether a time series is exist if exist, check the type consistency if not exist, call
* MManager to create it
*
- * @param timeseriesID ID of the time series
- * @return type consistency
+ * @param seriesKey path of the time series
+ * @param dataType type of the time series
+ * @return time series ID of the time series or null if type is not match
*/
- public boolean checkOrCreateIfNotExist(TimeseriesID timeseriesID) {
- return false;
+ public synchronized TimeseriesID checkOrCreateIfNotExist(
+ PartialPath seriesKey, TSDataType dataType) {
+ TimeseriesID timeseriesID = new TimeseriesID(seriesKey);
+ IDeviceID deviceID = timeseriesID.getDeviceID();
+ int slot = calculateSlot(deviceID);
+
+ DeviceEntry deviceEntry =
+ idTables[slot].computeIfAbsent(deviceID, id -> new DeviceEntry(deviceID));
+ SchemaEntry schemaEntry = deviceEntry.getSchemaEntry(timeseriesID.getMeasurement());
+
+ // if not exist, we create it
+ if (schemaEntry == null) {
+ schemaEntry = new SchemaEntry(dataType);
+
+ // 1. create new timeseries in mmanager
+ try {
+ MManager.getInstance()
+ .createTimeseries(
+ seriesKey,
+ dataType,
+ schemaEntry.getTSEncoding(),
+ schemaEntry.getCompressionType(),
+ null);
+ } catch (MetadataException e) {
+ logger.error("create timeseries failed, path is:" + seriesKey + " type is: " + dataType);
+ }
+
+ // 2. insert this schema into id table
+ deviceEntry.putSchemaEntry(timeseriesID.getMeasurement(), schemaEntry);
+
+ return timeseriesID;
+ }
+
+ // type mismatch, we return null and this will be handled by upper level
+ if (!schemaEntry.getTSDataType().equals(dataType)) {
+ return null;
+ }
+
+ return timeseriesID;
}
/**
- * upatde latest flushed time of one timeseries
+ * update latest flushed time of one timeseries
*
* @param timeseriesID timeseries id
- * @param flushedTime latest flushed time
+ * @param flushTime latest flushed time
+ * @throws MetadataException throw if this timeseries is not exist
*/
- public void updateLatestFlushedTime(TimeseriesID timeseriesID, long flushedTime) {}
+ public synchronized void updateLatestFlushTime(TimeseriesID timeseriesID, long flushTime)
+ throws MetadataException {
+ getSchemaEntry(timeseriesID).updateLastedFlushTime(flushTime);
+ }
/**
- * upatde latest flushed time of one timeseries
+ * update latest flushed time of one timeseries
*
* @param timeseriesID timeseries id
* @return latest flushed time of one timeseries
+ * @throws MetadataException throw if this timeseries is not exist
*/
- public long getLatestFlushedTime(TimeseriesID timeseriesID) {
- return 0;
+ public synchronized long getLatestFlushedTime(TimeseriesID timeseriesID)
+ throws MetadataException {
+ return getSchemaEntry(timeseriesID).getFlushTime();
+ }
+
+ /**
+ * get latest time value pair of one timeseries
+ *
+ * @param timeseriesID timeseries id
+ * @return latest time value pair of one timeseries
+ * @throws MetadataException throw if this timeseries is not exist
+ */
+ public Pair<Long, Object> getLastTimeValuePair(TimeseriesID timeseriesID)
+ throws MetadataException {
+ SchemaEntry schemaEntry = getSchemaEntry(timeseriesID);
+
+ return new Pair<>(schemaEntry.getLastTime(), schemaEntry.getLastValue());
+ }
+
+ /**
+ * update latest time value pair of one timeseries
+ *
+ * @param timeseriesID timeseries id
+ * @param lastTimeValue latest time value pair of one timeseries
+ * @throws MetadataException throw if this timeseries is not exist
+ */
+ public void updateLastTimeValuePair(TimeseriesID timeseriesID, Pair<Long, Object> lastTimeValue)
+ throws MetadataException {
+ getSchemaEntry(timeseriesID).updateLastCache(lastTimeValue);
+ }
+
+ /**
+ * calculate slot that this deviceID should in
+ *
+ * @param deviceID device id
+ * @return slot number
+ */
+ private int calculateSlot(IDeviceID deviceID) {
+ return deviceID.hashCode() % NUM_OF_SLOTS;
+ }
+
+ /**
+ * get schema entry
+ *
+ * @param timeseriesID the timeseries ID
+ * @return schema entry of the timeseries
+ * @throws MetadataException throw if this timeseries is not exist
+ */
+ private SchemaEntry getSchemaEntry(TimeseriesID timeseriesID) throws MetadataException {
+ IDeviceID deviceID = timeseriesID.getDeviceID();
+ int slot = calculateSlot(deviceID);
+
+ DeviceEntry deviceEntry = idTables[slot].get(deviceID);
+ if (deviceEntry == null) {
+ throw new MetadataException(
+ "update non exist timeseries's latest flushed time, timeseries id is: " + timeseriesID);
+ }
+
+ SchemaEntry schemaEntry = deviceEntry.getSchemaEntry(timeseriesID.getMeasurement());
+ if (schemaEntry == null) {
+ throw new MetadataException(
+ "update non exist timeseries's latest flushed time, timeseries id is: " + timeseriesID);
+ }
+
+ return schemaEntry;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceEntry.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceEntry.java
index 127581a..48449f1 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceEntry.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceEntry.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.metadata.id_table.entry;
+import java.util.HashMap;
import java.util.Map;
public class DeviceEntry {
@@ -28,4 +29,29 @@ public class DeviceEntry {
/** measurement schema map */
Map<String, SchemaEntry> measurementMap;
+
+ public DeviceEntry(IDeviceID deviceID) {
+ this.deviceID = deviceID;
+ measurementMap = new HashMap<>();
+ }
+
+ /**
+ * get schema entry of the measurement
+ *
+ * @param measurementName name of the measurement
+ * @return if exist, schema entry of the measurement. if not exist, null
+ */
+ public SchemaEntry getSchemaEntry(String measurementName) {
+ return measurementMap.get(measurementName);
+ }
+
+ /**
+ * put new schema entry of the measurement
+ *
+ * @param measurementName name of the measurement
+ * @param schemaEntry schema entry of the measurement
+ */
+ public void putSchemaEntry(String measurementName, SchemaEntry schemaEntry) {
+ measurementMap.put(measurementName, schemaEntry);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDFactory.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDFactory.java
new file mode 100644
index 0000000..4967536
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDFactory.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.metadata.id_table.entry;
+
+import java.util.function.Function;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+
+/** factory to build device id according to configured algorithm */
+public class DeviceIDFactory {
+ Function<PartialPath, IDeviceID> getDeviceIDFunction;
+
+ // region DeviceIDFactory Singleton
+ private static class DeviceIDFactoryHolder {
+
+ private DeviceIDFactoryHolder() {
+ // allowed to do nothing
+ }
+
+ private static final DeviceIDFactory INSTANCE = new DeviceIDFactory();
+ }
+
+ /**
+ * get instance
+ *
+ * @return instance of the factory
+ */
+ public static DeviceIDFactory getInstance() {
+ return DeviceIDFactory.DeviceIDFactoryHolder.INSTANCE;
+ }
+
+ private DeviceIDFactory() {
+ if (IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getDeviceIDTransformationMethod()
+ .equals("SHA256")) {
+ getDeviceIDFunction = partialPath -> new SHA256DeviceID(partialPath.getDevice());
+ } else {
+ getDeviceIDFunction = partialPath -> new PlainDeviceID(partialPath.getDevice());
+ }
+ }
+ // endregion
+
+ /**
+ * get device id by full path
+ *
+ * @param fullPath full path of the timeseries
+ * @return device id of the timeseries
+ */
+ public IDeviceID getDeviceID(PartialPath fullPath) {
+ return getDeviceIDFunction.apply(fullPath);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/PlainDeviceID.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/PlainDeviceID.java
index d1572e6..e3b2e25 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/PlainDeviceID.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/PlainDeviceID.java
@@ -45,4 +45,9 @@ public class PlainDeviceID implements IDeviceID {
public int hashCode() {
return Objects.hash(deviceID);
}
+
+ @Override
+ public String toString() {
+ return "PlainDeviceID{" + "deviceID='" + deviceID + '\'' + '}';
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/SHA256DeviceID.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/SHA256DeviceID.java
index 46362d7..144b838 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/SHA256DeviceID.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/SHA256DeviceID.java
@@ -18,6 +18,9 @@
*/
package org.apache.iotdb.db.metadata.id_table.entry;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
/** Using sha 256 hash value of device path as device ID */
public class SHA256DeviceID implements IDeviceID {
long l1;
@@ -25,6 +28,26 @@ public class SHA256DeviceID implements IDeviceID {
long l3;
long l4;
+ private static MessageDigest md;
+
+ static {
+ try {
+ md = MessageDigest.getInstance("SHA-256");
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public SHA256DeviceID(String deviceID) {
+ byte[] hashVal = md.digest(deviceID.getBytes());
+ md.reset();
+
+ l1 = toLong(hashVal, 0);
+ l2 = toLong(hashVal, 8);
+ l3 = toLong(hashVal, 16);
+ l4 = toLong(hashVal, 24);
+ }
+
/** The probability that each bit of sha 256 is 0 or 1 is equal */
public int hashCode() {
return (int) l1;
@@ -41,4 +64,19 @@ public class SHA256DeviceID implements IDeviceID {
SHA256DeviceID that = (SHA256DeviceID) o;
return l1 == that.l1 && l2 == that.l2 && l3 == that.l3 && l4 == that.l4;
}
+
+ private long toLong(byte[] array, int start) {
+ long res = 0;
+ for (int i = 0; i < 8; i++) {
+ res <<= 8;
+ res |= array[start + i];
+ }
+
+ return res;
+ }
+
+ @Override
+ public String toString() {
+ return "SHA256DeviceID{" + "l1=" + l1 + ", l2=" + l2 + ", l3=" + l3 + ", l4=" + l4 + '}';
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/SchemaEntry.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/SchemaEntry.java
index f635afe..ba6479a 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/SchemaEntry.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/SchemaEntry.java
@@ -19,14 +19,21 @@
package org.apache.iotdb.db.metadata.id_table.entry;
+import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding;
+
import java.io.Serializable;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.Pair;
public class SchemaEntry implements Serializable {
- /* 1 byte of type */
- /* 1 byte of encoding */
- /* 1 byte of compressor */
/* 5 byte of disk pointer */
+ /* 1 byte of compressor */
+ /* 1 byte of encoding */
+ /* 1 byte of type */
long schema;
long lastTime;
@@ -35,10 +42,72 @@ public class SchemaEntry implements Serializable {
long flushTime;
+ public SchemaEntry(TSDataType dataType) {
+ TSEncoding encoding = getDefaultEncoding(dataType);
+ CompressionType compressionType = TSFileDescriptor.getInstance().getConfig().getCompressor();
+
+ schema |= dataType.serialize();
+ schema |= (((long) encoding.serialize()) << 8);
+ schema |= (((long) compressionType.serialize()) << 16);
+
+ lastTime = Long.MIN_VALUE;
+ flushTime = Long.MIN_VALUE;
+ }
+
public SchemaEntry(long schema, long lastTime, Object lastValue, long flushTime) {
this.schema = schema;
this.lastTime = lastTime;
this.lastValue = lastValue;
this.flushTime = flushTime;
}
+
+ /**
+ * get ts data type from long value of schema
+ *
+ * @return ts data type
+ */
+ public TSDataType getTSDataType() {
+ return TSDataType.deserialize((byte) schema);
+ }
+
+ /**
+ * get ts encoding from long value of schema
+ *
+ * @return ts encoding
+ */
+ public TSEncoding getTSEncoding() {
+ return TSEncoding.deserialize((byte) (schema >> 8));
+ }
+
+ /**
+ * get compression type from long value of schema
+ *
+ * @return compression type
+ */
+ public CompressionType getCompressionType() {
+ return CompressionType.deserialize((byte) (schema >> 16));
+ }
+
+ public void updateLastedFlushTime(long lastFlushTime) {
+ flushTime = Math.max(flushTime, lastFlushTime);
+ }
+
+ public void updateLastCache(Pair<Long, Object> lastTimeValue) {
+ if (lastTimeValue.left >= lastTime) {
+ lastTime = lastTimeValue.left;
+ lastValue = lastTimeValue.right;
+ }
+ }
+
+ public long getLastTime() {
+ return lastTime;
+ }
+
+ public Object getLastValue() {
+ return lastValue;
+ }
+
+ public long getFlushTime() {
+ return flushTime;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/TimeseriesID.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/TimeseriesID.java
index ed2386a..f02fcb5 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/TimeseriesID.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/TimeseriesID.java
@@ -18,8 +18,33 @@
*/
package org.apache.iotdb.db.metadata.id_table.entry;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+
public class TimeseriesID {
- private IDeviceID deviceID;
+ private IDeviceID deviceID;
private String measurement;
+
+ /** build timeseries id from full path */
+ public TimeseriesID(PartialPath fullPath) {
+ deviceID = DeviceIDFactory.getInstance().getDeviceID(fullPath);
+ measurement = fullPath.getMeasurement();
+ }
+
+ public IDeviceID getDeviceID() {
+ return deviceID;
+ }
+
+ public String getMeasurement() {
+ return measurement;
+ }
+
+ public void setMeasurement(String measurement) {
+ this.measurement = measurement;
+ }
+
+ @Override
+ public String toString() {
+ return "TimeseriesID{" + "deviceID=" + deviceID + ", measurement='" + measurement + '\'' + '}';
+ }
}
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDManagerTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDManagerTest.java
new file mode 100644
index 0000000..585a974
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDManagerTest.java
@@ -0,0 +1,5 @@
+package org.apache.iotdb.db.metadata.id_table;
+
+public class IDManagerTest {
+
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableTest.java
new file mode 100644
index 0000000..0c8f5f2
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableTest.java
@@ -0,0 +1,5 @@
+package org.apache.iotdb.db.metadata.id_table;
+
+public class IDTableTest {
+
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDTest.java
new file mode 100644
index 0000000..ba4096c
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.metadata.id_table.entry;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+import org.junit.Test;
+
+public class DeviceIDTest {
+ @Test
+ public void deviceIDBuildTest() throws IllegalPathException {
+ PartialPath partialPath1 = new PartialPath("root.sg1.d1.s1");
+ PartialPath partialPath2 = new PartialPath("root.sg1.d1.s2");
+ PartialPath partialPath3 = new PartialPath("root.sg1.d2.s1");
+
+ IDeviceID deviceID1 = DeviceIDFactory.getInstance().getDeviceID(partialPath1);
+ IDeviceID deviceID2 = DeviceIDFactory.getInstance().getDeviceID(partialPath2);
+ IDeviceID deviceID3 = DeviceIDFactory.getInstance().getDeviceID(partialPath3);
+
+ assertEquals(deviceID1, deviceID2);
+ assertNotEquals(deviceID1, deviceID3);
+ assertNotEquals(deviceID2, deviceID3);
+ }
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/entry/SchemaEntryTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/entry/SchemaEntryTest.java
new file mode 100644
index 0000000..34f490b
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/entry/SchemaEntryTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.metadata.id_table.entry;
+
+import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding;
+import static org.junit.Assert.assertEquals;
+
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.junit.Test;
+
+public class SchemaEntryTest {
+ @Test
+ public void schemaEntryBuildTest() throws IllegalPathException {
+ for (TSDataType type : TSDataType.values()) {
+ // omit vector
+ if (type == TSDataType.VECTOR) {
+ continue;
+ }
+
+ SchemaEntry schemaEntry = new SchemaEntry(type);
+
+ // schema check
+ assertEquals(schemaEntry.getTSDataType(), type);
+ assertEquals(schemaEntry.getTSEncoding(), getDefaultEncoding(type));
+ assertEquals(
+ schemaEntry.getCompressionType(),
+ TSFileDescriptor.getInstance().getConfig().getCompressor());
+
+ // flush time
+ schemaEntry.updateLastedFlushTime(100);
+ assertEquals(schemaEntry.getFlushTime(), 100);
+ schemaEntry.updateLastedFlushTime(50);
+ assertEquals(schemaEntry.getFlushTime(), 100);
+
+ // last cache
+ Object o1 = new Object();
+ Object o2 = new Object();
+ schemaEntry.updateLastCache(new Pair<>(100L, o1));
+ assertEquals(schemaEntry.getLastValue(), o1);
+ assertEquals(schemaEntry.getLastTime(), 100L);
+
+ schemaEntry.updateLastCache(new Pair<>(90L, o2));
+ assertEquals(schemaEntry.getLastValue(), o1);
+ assertEquals(schemaEntry.getLastTime(), 100L);
+
+ schemaEntry.updateLastCache(new Pair<>(110L, o2));
+ assertEquals(schemaEntry.getLastValue(), o2);
+ assertEquals(schemaEntry.getLastTime(), 110L);
+ }
+ }
+}