You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2019/02/28 09:02:46 UTC
[incubator-iotdb] branch master updated: [IOTDB-29] Fix bug that
reading an empty series (but has been registered). (#75)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 07c8fce [IOTDB-29] Fix bug that reading an empty series (but has been registered). (#75)
07c8fce is described below
commit 07c8fce138b398eb27cc1fb8fb9c8c7261b2250b
Author: Xiangdong Huang <hx...@qq.com>
AuthorDate: Thu Feb 28 17:02:43 2019 +0800
[IOTDB-29] Fix bug that reading an empty series (but has been registered). (#75)
* fix issue #29
---
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 11 +++++
pom.xml | 3 ++
.../tsfile/read/controller/MetadataQuerier.java | 9 ++++
.../read/controller/MetadataQuerierByFileImpl.java | 12 +++++
.../tsfile/read/query/executor/TsFileExecutor.java | 57 ++++++++++++++--------
.../series/EmptyFileSeriesReader.java} | 53 ++++++++++++++------
.../iotdb/tsfile/write/TsFileReadWriteTest.java | 36 ++++++++++++++
7 files changed, 147 insertions(+), 34 deletions(-)
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java b/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
index eb8da77..387310b 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
@@ -19,11 +19,13 @@
package org.apache.iotdb.db.integration;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import org.apache.iotdb.db.exception.FileNodeManagerException;
@@ -343,4 +345,13 @@ public class IoTDBSeriesReaderIT {
QueryTokenManager.getInstance().endQueryForCurrentRequestThread();
}
+
+ @Test
+ public void queryEmptySeriesTest() throws SQLException {
+ Statement statement = connection.createStatement();
+ statement.execute("CREATE TIMESERIES root.vehicle.d_empty.s1 WITH DATATYPE=INT64, ENCODING=RLE");
+ ResultSet resultSet = statement.executeQuery("select * from root.vehicle.d_empty");
+ assertFalse (resultSet.next());
+ resultSet.close();
+ }
}
diff --git a/pom.xml b/pom.xml
index 2a2e9ab..6f92158 100644
--- a/pom.xml
+++ b/pom.xml
@@ -201,6 +201,9 @@
<exclude>.checkstyle</exclude>
<!--Generated by Apache Release -->
<exclude>local-snapshots-dir/**</exclude>
+ <!--IoTDB data files-->
+ <exclude>iotdb/data/**</exclude>
+ <exclude>iotdb/logs/**</exclude>
</excludes>
</configuration>
</plugin>
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerier.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerier.java
index e288e58..6737896 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerier.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerier.java
@@ -20,8 +20,10 @@ package org.apache.iotdb.tsfile.read.controller;
import java.io.IOException;
import java.util.List;
+import org.apache.iotdb.tsfile.exception.write.NoMeasurementException;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Path;
public interface MetadataQuerier {
@@ -38,4 +40,11 @@ public interface MetadataQuerier {
*/
void loadChunkMetaDatas(List<Path> paths) throws IOException;
+ /**
+ *
+ * @param measurement
+ * @return the corresponding data type.
+ * @throws NoMeasurementException if the measurement not exists.
+ */
+ TSDataType getDataType(String measurement) throws NoMeasurementException;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerierByFileImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerierByFileImpl.java
index b1e16c9..acc8fc4 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerierByFileImpl.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerierByFileImpl.java
@@ -28,13 +28,16 @@ import java.util.Set;
import java.util.TreeMap;
import org.apache.iotdb.tsfile.common.cache.LRUCache;
import org.apache.iotdb.tsfile.common.constant.QueryConstant;
+import org.apache.iotdb.tsfile.exception.write.NoMeasurementException;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
public class MetadataQuerierByFileImpl implements MetadataQuerier {
@@ -178,6 +181,15 @@ public class MetadataQuerierByFileImpl implements MetadataQuerier {
}
+ @Override
+ public TSDataType getDataType(String measurement) throws NoMeasurementException {
+ MeasurementSchema measurementSchema = fileMetaData.getMeasurementSchema().get(measurement);
+ if(measurementSchema != null) {
+ return measurementSchema.getType();
+ }
+ throw new NoMeasurementException(String.format("%s not found.", measurement));
+ }
+
private List<ChunkMetaData> loadChunkMetadata(Path path) throws IOException {
if (!fileMetaData.containsDevice(path.getDevice())) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java
index 69d364f..39883af 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException;
+import org.apache.iotdb.tsfile.exception.write.NoMeasurementException;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -33,6 +34,7 @@ import org.apache.iotdb.tsfile.read.expression.impl.GlobalTimeExpression;
import org.apache.iotdb.tsfile.read.expression.util.ExpressionOptimizer;
import org.apache.iotdb.tsfile.read.query.dataset.DataSetWithoutTimeGenerator;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.iotdb.tsfile.read.reader.series.EmptyFileSeriesReader;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderWithFilter;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderWithoutFilter;
@@ -66,11 +68,15 @@ public class TsFileExecutor implements QueryExecutor {
return new ExecutorWithTimeGenerator(metadataQuerier, chunkLoader)
.execute(queryExpression);
}
- } catch (QueryFilterOptimizationException e) {
+ } catch (QueryFilterOptimizationException | NoMeasurementException e) {
throw new IOException(e);
}
} else {
- return execute(queryExpression.getSelectedSeries());
+ try {
+ return execute(queryExpression.getSelectedSeries());
+ } catch (NoMeasurementException e) {
+ throw new IOException(e);
+ }
}
}
@@ -80,18 +86,9 @@ public class TsFileExecutor implements QueryExecutor {
* @param selectedPathList all selected paths
* @return DataSet without TimeGenerator
*/
- private QueryDataSet execute(List<Path> selectedPathList) throws IOException {
- List<FileSeriesReader> readersOfSelectedSeries = new ArrayList<>();
- List<TSDataType> dataTypes = new ArrayList<>();
-
- for (Path path : selectedPathList) {
- List<ChunkMetaData> chunkMetaDataList = metadataQuerier.getChunkMetaDataList(path);
- FileSeriesReader seriesReader = new FileSeriesReaderWithoutFilter(chunkLoader,
- chunkMetaDataList);
- readersOfSelectedSeries.add(seriesReader);
- dataTypes.add(chunkMetaDataList.get(0).getTsDataType());
- }
- return new DataSetWithoutTimeGenerator(selectedPathList, dataTypes, readersOfSelectedSeries);
+ private QueryDataSet execute(List<Path> selectedPathList)
+ throws IOException, NoMeasurementException {
+ return executeMayAttachTimeFiler(selectedPathList, null);
}
/**
@@ -102,18 +99,40 @@ public class TsFileExecutor implements QueryExecutor {
* @return DataSet without TimeGenerator
*/
private QueryDataSet execute(List<Path> selectedPathList, GlobalTimeExpression timeFilter)
- throws IOException {
+ throws IOException, NoMeasurementException {
+ return executeMayAttachTimeFiler(selectedPathList, timeFilter);
+ }
+
+ /**
+ *
+ * @param selectedPathList completed path
+ * @param timeFilter a GlobalTimeExpression or null
+ * @return DataSetWithoutTimeGenerator
+ * @throws IOException
+ * @throws NoMeasurementException
+ */
+ private QueryDataSet executeMayAttachTimeFiler(List<Path> selectedPathList, GlobalTimeExpression timeFilter)
+ throws IOException, NoMeasurementException {
List<FileSeriesReader> readersOfSelectedSeries = new ArrayList<>();
List<TSDataType> dataTypes = new ArrayList<>();
for (Path path : selectedPathList) {
List<ChunkMetaData> chunkMetaDataList = metadataQuerier.getChunkMetaDataList(path);
- FileSeriesReader seriesReader = new FileSeriesReaderWithFilter(chunkLoader, chunkMetaDataList,
- timeFilter.getFilter());
+ FileSeriesReader seriesReader;
+ if (chunkMetaDataList.isEmpty()) {
+ seriesReader = new EmptyFileSeriesReader();
+ dataTypes.add(metadataQuerier.getDataType(path.getMeasurement()));
+ } else {
+ if (timeFilter == null) {
+ seriesReader = new FileSeriesReaderWithoutFilter(chunkLoader, chunkMetaDataList);
+ } else {
+ seriesReader = new FileSeriesReaderWithFilter(chunkLoader, chunkMetaDataList,
+ timeFilter.getFilter());
+ }
+ dataTypes.add(chunkMetaDataList.get(0).getTsDataType());
+ }
readersOfSelectedSeries.add(seriesReader);
- dataTypes.add(chunkMetaDataList.get(0).getTsDataType());
}
-
return new DataSetWithoutTimeGenerator(selectedPathList, dataTypes, readersOfSelectedSeries);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerier.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/EmptyFileSeriesReader.java
similarity index 52%
copy from tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerier.java
copy to tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/EmptyFileSeriesReader.java
index e288e58..26a509f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerier.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/EmptyFileSeriesReader.java
@@ -16,26 +16,49 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.tsfile.read.controller;
-import java.io.IOException;
-import java.util.List;
+package org.apache.iotdb.tsfile.read.reader.series;
+
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
-import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+
+/**
+ * this is for those series which has no data points
+ */
+public class EmptyFileSeriesReader extends FileSeriesReader {
+ BatchData data = new BatchData();
+
+ public EmptyFileSeriesReader() {
+ super(null, null);
+ }
+
+ @Override
+ protected void initChunkReader(ChunkMetaData chunkMetaData) {
+ //do nothing
+ }
-public interface MetadataQuerier {
+ @Override
+ protected boolean chunkSatisfied(ChunkMetaData chunkMetaData) {
+ return false;
+ }
- List<ChunkMetaData> getChunkMetaDataList(Path path) throws IOException;
+ @Override
+ public boolean hasNextBatch() {
+ return false;
+ }
- TsFileMetaData getWholeFileMetadata();
+ @Override
+ public BatchData nextBatch() {
+ return data;
+ }
- /**
- * this will load all chunk metadata of given paths into cache.
- *
- * <p>call this method before calling getChunkMetaDataList() will accelerate the reading of chunk
- * metadata, which will only read TsDeviceMetaData once
- */
- void loadChunkMetaDatas(List<Path> paths) throws IOException;
+ @Override
+ public BatchData currentBatch() {
+ return data;
+ }
+ @Override
+ public void close() {
+ data = null;
+ }
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java
index 53d2d02..3cb2a84 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.tsfile.write;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
@@ -216,4 +218,38 @@ public class TsFileReadWriteTest {
reader.close();
}
+ @Test
+ public void readEmptyMeasurementTest() throws IOException, WriteProcessException {
+ String path = "test.tsfile";
+ File f = new File(path);
+ if (f.exists()) {
+ assertTrue(f.delete());
+ }
+ TsFileWriter tsFileWriter = new TsFileWriter(f);
+
+ // add measurements into file schema
+ tsFileWriter
+ .addMeasurement(new MeasurementSchema("sensor_1", TSDataType.FLOAT, TSEncoding.RLE));
+ tsFileWriter
+ .addMeasurement(new MeasurementSchema("sensor_2", TSDataType.INT32, TSEncoding.TS_2DIFF));
+ // construct TSRecord
+ TSRecord tsRecord = new TSRecord(1, "device_1");
+ DataPoint dPoint1 = new FloatDataPoint("sensor_1", 1.2f);
+ tsRecord.addTuple(dPoint1);
+ // write a TSRecord to TsFile
+ tsFileWriter.write(tsRecord);
+ // close TsFile
+ tsFileWriter.close();
+
+ // read example : no filter
+ TsFileSequenceReader reader = new TsFileSequenceReader(path);
+ ReadOnlyTsFile readTsFile = new ReadOnlyTsFile(reader);
+ ArrayList<Path> paths = new ArrayList<>();
+ paths.add(new Path("device_1.sensor_2"));
+ QueryExpression queryExpression = QueryExpression.create(paths, null);
+ QueryDataSet queryDataSet = readTsFile.query(queryExpression);
+ assertFalse(queryDataSet.hasNext());
+ reader.close();
+ assertTrue(f.delete());
+ }
}