You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/07/18 10:46:58 UTC
[incubator-iotdb] branch master updated: TsFile Docs (#247)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a1b906d TsFile Docs (#247)
a1b906d is described below
commit a1b906d70ad43aaf5a4d8b35d977f60655f88b51
Author: Zihan Meng <40...@users.noreply.github.com>
AuthorDate: Thu Jul 18 18:46:53 2019 +0800
TsFile Docs (#247)
* Add TsFile Docs, delete Json schema
---
docs/Documentation/UserGuideV0.7.0/0-Content.md | 6 +-
.../UserGuideV0.7.0/7-TsFile/1-Installation.md | 93 ++++
.../UserGuideV0.7.0/7-TsFile/2-Usage.md | 525 +++++++++++++++++++++
tsfile/example/readme.md | 15 +-
.../java/org/apache/iotdb/tsfile/TsFileRead.java | 86 ++--
.../java/org/apache/iotdb/tsfile/TsFileWrite.java | 233 +++------
.../apache/iotdb/tsfile/write/TsFileWriter.java | 5 -
.../iotdb/tsfile/write/schema/FileSchema.java | 17 -
.../iotdb/tsfile/write/schema/JsonConverter.java | 203 --------
.../TsFileGeneratorForSeriesReaderByTimestamp.java | 63 +--
.../apache/iotdb/tsfile/utils/FileGenerator.java | 60 +--
.../apache/iotdb/tsfile/utils/RecordUtilsTest.java | 52 +-
.../org/apache/iotdb/tsfile/write/PerfTest.java | 38 +-
.../iotdb/tsfile/write/ReadPageInMemTest.java | 59 +--
.../org/apache/iotdb/tsfile/write/WriteTest.java | 46 +-
.../write/schema/converter/JsonConverterTest.java | 126 -----
tsfile/src/test/resources/test_schema.json | 25 -
tsfile/src/test/resources/test_write_schema.json | 31 --
18 files changed, 828 insertions(+), 855 deletions(-)
diff --git a/docs/Documentation/UserGuideV0.7.0/0-Content.md b/docs/Documentation/UserGuideV0.7.0/0-Content.md
index 9fd130c..8e40832 100644
--- a/docs/Documentation/UserGuideV0.7.0/0-Content.md
+++ b/docs/Documentation/UserGuideV0.7.0/0-Content.md
@@ -48,4 +48,8 @@
* 1-IoTDB Query Statement
* 2-Reference
# Chapter 6: JDBC API
-* 1-JDBC API
\ No newline at end of file
+* 1-JDBC API
+# Chapter 7: TsFile
+* 1-Installation
+* 2-Usage
+* 3-Hierarchy
\ No newline at end of file
diff --git a/docs/Documentation/UserGuideV0.7.0/7-TsFile/1-Installation.md b/docs/Documentation/UserGuideV0.7.0/7-TsFile/1-Installation.md
new file mode 100644
index 0000000..50c057d
--- /dev/null
+++ b/docs/Documentation/UserGuideV0.7.0/7-TsFile/1-Installation.md
@@ -0,0 +1,93 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+## Installation
+
+Before started, maven should be installed. See <a href="https://maven.apache.org/install.html">How to install maven</a>
+
+There are two ways to use TsFile in your own project.
+
+* Using as jars:
+ * Compile the source codes and build to jars
+
+ ```
+ git clone https://github.com/apache/incubator-iotdb.git
+ cd tsfile/
+ sh package.sh
+ ```
+ Then, all the jars can be get in folder named `lib/`. Import `lib/tsfile-0.9.0-SNAPSHOT-jar-with-dependencies.jar` to your project.
+
+* Using as a maven dependency:
+
+ Compile source codes and deploy to your local repository in three steps:
+
+ * Get the source codes
+
+ ```
+ git clone https://github.com/apache/incubator-iotdb.git
+ ```
+ * Compile the source codes and deploy
+
+ ```
+ cd tsfile/
+ mvn clean install -Dmaven.test.skip=true
+ ```
+ * add dependencies into your project:
+
+ ```
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>tsfile</artifactId>
+ <version>0.9.0-SNAPSHOT</version>
+ </dependency>
+ ```
+
+ Or, you can download the dependencies from official Maven repository:
+
+ * First, find your maven `settings.xml` on path: `${username}\.m2\settings.xml`
+ , add this `<profile>` to `<profiles>`:
+ ```
+ <profile>
+ <id>allow-snapshots</id>
+ <activation><activeByDefault>true</activeByDefault></activation>
+ <repositories>
+ <repository>
+ <id>apache.snapshots</id>
+ <name>Apache Development Snapshot Repository</name>
+ <url>https://repository.apache.org/content/repositories/snapshots/</url>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+ </profile>
+ ```
+ * Then add dependencies into your project:
+
+ ```
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>tsfile</artifactId>
+ <version>0.9.0-SNAPSHOT</version>
+ </dependency>
+ ```
\ No newline at end of file
diff --git a/docs/Documentation/UserGuideV0.7.0/7-TsFile/2-Usage.md b/docs/Documentation/UserGuideV0.7.0/7-TsFile/2-Usage.md
new file mode 100644
index 0000000..94c50a3
--- /dev/null
+++ b/docs/Documentation/UserGuideV0.7.0/7-TsFile/2-Usage.md
@@ -0,0 +1,525 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+Now, you are ready to start doing some awesome things with TsFile. This section demonstrates the detailed usages of TsFile.
+
+### Time-series Data
+A time-series is considered as a sequence of quadruples. A quadruple is defined as (device, measurement, time, value).
+
+* **measurement**: A physical or formal measurement that a time-series is taking, e.g., the temperature of a city, the
+sales number of some goods or the speed of a train at different times. As a traditional sensor (like a thermometer) also
+ takes a single measurement and produce a time-series, we will use measurement and sensor interchangeably below.
+
+* **device**: A device refers to an entity that is taking several measurements (producing multiple time-series), e.g.,
+a running train monitors its speed, oil meter, miles it has run, current passengers each is conveyed to a time-series.
+
+Table 1 illustrates a set of time-series data. The set showed in the following table contains one device named "device\_1"
+with three measurements named "sensor\_1", "sensor\_2" and "sensor\_3".
+
+<center>
+<table style="text-align:center">
+ <tr><th colspan="6">device_1</th></tr>
+ <tr><th colspan="2">sensor_1</th><th colspan="2">sensor_2</th><th colspan="2">sensor_3</th></tr>
+ <tr><th>time</th><th>value</td><th>time</th><th>value</td><th>time</th><th>value</td>
+ <tr><td>1</td><td>1.2</td><td>1</td><td>20</td><td>2</td><td>50</td></tr>
+ <tr><td>3</td><td>1.4</td><td>2</td><td>20</td><td>4</td><td>51</td></tr>
+ <tr><td>5</td><td>1.1</td><td>3</td><td>21</td><td>6</td><td>52</td></tr>
+ <tr><td>7</td><td>1.8</td><td>4</td><td>20</td><td>8</td><td>53</td></tr>
+</table>
+<span>A set of time-series data</span>
+</center>
+
+**One Line of Data**: In many industrial applications, a device normally contains more than one sensor and these sensors
+ may have values at a same timestamp, which is called one line of data.
+
+Formally, one line of data consists of a `device_id`, a timestamp which indicates the milliseconds since January 1,
+ 1970, 00:00:00, and several data pairs composed of `measurement_id` and corresponding `value`. All data pairs in one
+ line belong to this `device_id` and have the same timestamp. If one of the `measurements` does not have a `value`
+ in the `timestamp`, use a space instead(Actually, TsFile does not store null values). Its format is shown as follow:
+
+```
+device_id, timestamp, <measurement_id, value>...
+```
+
+An example is illustrated as follow. In this example, the data type of two measurements are `INT32`, `FLOAT` respectively.
+
+```
+device_1, 1490860659000, m1, 10, m2, 12.12
+```
+
+
+### Writing TsFile
+
+#### Generate a TsFile File.
+A TsFile can be generated by following three steps and the complete code will be given in the section "Example for writing TsFile".
+
+* First, construct a `TsFileWriter` instance.
+
+ Here are the available constructors:
+
+ * Without pre-defined schema
+ ```
+ public TsFileWriter(File file) throws IOException
+ ```
+ * With pre-defined schema
+ ```
+ public TsFileWriter(File file, FileSchema schema) throws IOException
+ ```
+ This one is for using the HDFS file system. `TsFileOutput` can be an instance of class `HDFSOutput`.
+
+ ```
+ public TsFileWriter(TsFileOutput output, FileSchema schema) throws IOException
+ ```
+ **Parameters:**
+
+ * file : The TsFile to write
+
+ * schema : The file schemas, will be introduced in next part.
+
+* Second, add measurements
+
+ Or you can make an instance of class `FileSchema` first and pass this to the constructor of class `TsFileWriter`
+
+ The class `FileSchema` contains a map whose key is the name of one measurement schema, and the value is the schema itself.
+
+ Here are the interfaces:
+ ```
+ // Create an empty FileSchema or from an existing map
+ public FileSchema()
+ public FileSchema(Map<String, MeasurementSchema> measurements)
+
+ // Use this two interfaces to add measurements
+ public void registerMeasurement(MeasurementSchema descriptor)
+ public void registerMeasurements(Map<String, MeasurementSchema> measurements)
+
+ // Some useful getter and checker
+ public TSDataType getMeasurementDataType(String measurementId)
+ public MeasurementSchema getMeasurementSchema(String measurementId)
+ public Map<String, MeasurementSchema> getAllMeasurementSchema()
+ public boolean hasMeasurement(String measurementId)
+ ```
+
+ You can always use the following interface in `TsFileWriter` class to add additional measurements:
+
+ ```
+ public void addMeasurement(MeasurementSchema measurementSchema) throws WriteProcessException
+ ```
+
+ The class `MeasurementSchema` contains the information of one measurement, there are several constructors:
+ ```
+ public MeasurementSchema(String measurementId, TSDataType type, TSEncoding encoding)
+
+ public MeasurementSchema(String measurementId, TSDataType type, TSEncoding encoding, CompressionType compressionType)
+
+ public MeasurementSchema(String measurementId, TSDataType type, TSEncoding encoding, CompressionType compressionType,
+ Map<String, String> props)
+ ```
+
+ **Parameters:**
+
+ * measurementID: The name of this measurement, typically the name of the sensor.
+
+ * type: The data type, now support six types: `BOOLEAN`, `INT32`, `INT64`, `FLOAT`, `DOUBLE`, `TEXT`;
+
+ * encoding: The data encoding. See [Chapter 2-3]('../2-Concept Key Concepts and Terminology/3-Encoding.md').
+
+ * compression: The data compression. Now supports `UNCOMPRESSED` and `SNAPPY`.
+
+ * props: Properties for special data types.Such as `max_point_number` for `FLOAT` and `DOUBLE`, `max_string_length` for
+ `TEXT`. Use as string pairs into a map such as ("max_point_number", "3").
+
+ > **Notice:** Although one measurement name can be used in multiple deltaObjects, the properties cannot be changed. I.e.
+ it's not allowed to add one measurement name for multiple times with different type or encoding.
+ Here is a bad example:
+
+ // The measurement "sensor_1" is float type
+ addMeasurement(new MeasurementSchema("sensor_1", TSDataType.FLOAT, TSEncoding.RLE));
+
+ // This call will throw a WriteProcessException exception
+ addMeasurement(new MeasurementSchema("sensor_1", TSDataType.INT32, TSEncoding.RLE));
+* Third, insert and write data continually.
+
+ Use this interface to create a new `TSRecord`(a timestamp and device pair).
+
+ ```
+ public TSRecord(long timestamp, String deviceId)
+ ```
+ Then create a `DataPoint`(a measurement and value pair), and use the addTuple method to add the DataPoint to the correct
+ TsRecord.
+
+ Use this method to write
+
+ ```
+ public void write(TSRecord record) throws IOException, WriteProcessException
+ ```
+
+* Finally, call `close` to finish this writing process.
+
+ ```
+ public void close() throws IOException
+ ```
+
+#### Example for writing a TsFile
+
+You should install TsFile to your local maven repository.
+
+See reference: [Installation](./1-Installation.md)
+
+A more thorough example can be found at `/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileWrite.java`
+
+```java
+package org.apache.iotdb.tsfile;
+
+import java.io.File;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
+import org.apache.iotdb.tsfile.write.record.datapoint.FloatDataPoint;
+import org.apache.iotdb.tsfile.write.record.datapoint.IntDataPoint;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+/**
+ * An example of writing data to TsFile
+ * It uses the interface:
+ * public void addMeasurement(MeasurementSchema MeasurementSchema) throws WriteProcessException
+ */
+public class TsFileWrite {
+
+ public static void main(String args[]) {
+ try {
+ String path = "test.tsfile";
+ File f = new File(path);
+ if (f.exists()) {
+ f.delete();
+ }
+ TsFileWriter tsFileWriter = new TsFileWriter(f);
+
+ // add measurements into file schema
+ tsFileWriter
+ .addMeasurement(new MeasurementSchema("sensor_1", TSDataType.FLOAT, TSEncoding.RLE));
+ tsFileWriter
+ .addMeasurement(new MeasurementSchema("sensor_2", TSDataType.INT32, TSEncoding.TS_2DIFF));
+ tsFileWriter
+ .addMeasurement(new MeasurementSchema("sensor_3", TSDataType.INT32, TSEncoding.TS_2DIFF));
+ // construct TSRecord
+ TSRecord tsRecord = new TSRecord(1, "device_1");
+ DataPoint dPoint1 = new FloatDataPoint("sensor_1", 1.2f);
+ DataPoint dPoint2 = new IntDataPoint("sensor_2", 20);
+
+ //For time 1 in device_1, the data will be 1.2, 20, null
+ tsRecord.addTuple(dPoint1);
+ tsRecord.addTuple(dPoint2);
+
+ // write a TSRecord to TsFile
+ tsFileWriter.write(tsRecord);
+ // close TsFile
+ tsFileWriter.close();
+ } catch (Throwable e) {
+ e.printStackTrace();
+ System.out.println(e.getMessage());
+ }
+ }
+}
+
+```
+
+### Interface for Reading TsFile
+
+#### Before the Start
+
+The set of time-series data in section "Time-series Data" is used here for a concrete introduction in this section. The set showed in the following table contains one deltaObject named "device\_1" with three measurements named "sensor\_1", "sensor\_2" and "sensor\_3". And the measurements has been simplified to do a simple illustration, which contains only 4 time-value pairs each.
+
+<center>
+<table style="text-align:center">
+ <tr><th colspan="6">device_1</th></tr>
+ <tr><th colspan="2">sensor_1</th><th colspan="2">sensor_2</th><th colspan="2">sensor_3</th></tr>
+ <tr><th>time</th><th>value</td><th>time</th><th>value</td><th>time</th><th>value</td>
+ <tr><td>1</td><td>1.2</td><td>1</td><td>20</td><td>2</td><td>50</td></tr>
+ <tr><td>3</td><td>1.4</td><td>2</td><td>20</td><td>4</td><td>51</td></tr>
+ <tr><td>5</td><td>1.1</td><td>3</td><td>21</td><td>6</td><td>52</td></tr>
+ <tr><td>7</td><td>1.8</td><td>4</td><td>20</td><td>8</td><td>53</td></tr>
+</table>
+<span>A set of time-series data</span>
+</center>
+
+#### Definition of Path
+
+A path is a dot-separated string which uniquely identifies a time-series in TsFile, e.g., "root.area_1.device_1.sensor_1".
+The last section "sensor_1" is called "measurementId" while the remaining parts "root.area_1.device_1" is called deviceId.
+As mentioned above, the same measurement in different devices has the same data type and encoding, and devices are also unique.
+
+In read interfaces, The parameter ```paths``` indicates the measurements to be selected.
+
+Path instance can be easily constructed through the class ```Path```. For example:
+
+```
+Path p = new Path("device_1.sensor_1");
+```
+
+We will pass an ArrayList of paths for final query call to support multiple paths.
+
+```
+List<Path> paths = new ArrayList<Path>();
+paths.add(new Path("device_1.sensor_1"));
+paths.add(new Path("device_1.sensor_3"));
+```
+
+> **Notice:** When constructing a Path, the format of the parameter should be a dot-separated string, the last part will
+ be recognized as measurementId while the remaining parts will be recognized as deviceId.
+
+
+#### Definition of Filter
+
+##### Usage Scenario
+Filter is used in TsFile reading process to select data satisfying one or more given condition(s).
+
+##### IExpression
+The `IExpression` is a filter expression interface and it will be passed to our final query call.
+We create one or more filter expressions and may use binary filter operators to link them to our final expression.
+
+* **Create a Filter Expression**
+
+ There are two types of filters.
+
+ * TimeFilter: A filter for `time` in time-series data.
+ ```
+ IExpression timeFilterExpr = new GlobalTimeExpression(TimeFilter);
+ ```
+ Use the following relationships to get a `TimeFilter` object (value is a long int variable).
+ <center>
+ <table style="text-align:center">
+ <tr><th>Relationship</th><th>Description</td></tr>
+ <tr><td>TimeFilter.eq(value)</td><td>Choose the time equal to the value</td>
+ <tr><td>TimeFilter.lt(value)</td><td>Choose the time less than the value</td>
+ <tr><td>TimeFilter.gt(value)</td><td>Choose the time greater than the value</td>
+ <tr><td>TimeFilter.ltEq(value)</td><td>Choose the time less than or equal to the value</td>
+ <tr><td>TimeFilter.gtEq(value)</td><td>Choose the time greater than or equal to the value</td>
+ <tr><td>TimeFilter.notEq(value)</td><td>Choose the time not equal to the value</td>
+ <tr><td>TimeFilter.not(TimeFilter)</td><td>Choose the time not satisfy another TimeFilter</td>
+ </table>
+ </center>
+
+ * ValueFilter: A filter for `value` in time-series data.
+
+ ```
+ IExpression valueFilterExpr = new SingleSeriesExpression(Path, ValueFilter);
+ ```
+ The usage of `ValueFilter` is the same as using `TimeFilter`, just to make sure that the type of the value
+ equal to the measurement's(defined in the path).
+
+* **Binary Filter Operators**
+
+ Binary filter operators can be used to link two single expressions.
+
+ * BinaryExpression.and(Expression, Expression): Choose the value satisfy for both expressions.
+ * BinaryExpression.or(Expression, Expression): Choose the value satisfy for at least one expression.
+
+
+##### Filter Expression Examples
+
+* **TimeFilterExpression Examples**
+
+ ```
+ IExpression timeFilterExpr = new GlobalTimeExpression(TimeFilter.eq(15)); // series time = 15
+
+ ```
+ ```
+ IExpression timeFilterExpr = new GlobalTimeExpression(TimeFilter.ltEq(15)); // series time <= 15
+
+ ```
+ ```
+ IExpression timeFilterExpr = new GlobalTimeExpression(TimeFilter.lt(15)); // series time < 15
+
+ ```
+ ```
+ IExpression timeFilterExpr = new GlobalTimeExpression(TimeFilter.gtEq(15)); // series time >= 15
+
+ ```
+ ```
+ IExpression timeFilterExpr = new GlobalTimeExpression(TimeFilter.notEq(15)); // series time != 15
+
+ ```
+ ```
+ IExpression timeFilterExpr = BinaryExpression.and(new GlobalTimeExpression(TimeFilter.gtEq(15L)),
+ new GlobalTimeExpression(TimeFilter.lt(25L))); // 15 <= series time < 25
+ ```
+ ```
+ IExpression timeFilterExpr = BinaryExpression.or(new GlobalTimeExpression(TimeFilter.gtEq(15L)),
+ new GlobalTimeExpression(TimeFilter.lt(25L))); // series time >= 15 or series time < 25
+ ```
+#### Read Interface
+
+First, we open the TsFile and get a `ReadOnlyTsFile` instance from a file path string `path`.
+
+```
+TsFileSequenceReader reader = new TsFileSequenceReader(path);
+
+ReadOnlyTsFile readTsFile = new ReadOnlyTsFile(reader);
+```
+Next, we prepare the path array and query expression, then get final `QueryExpression` object by this interface:
+
+```
+QueryExpression queryExpression = QueryExpression.create(paths, statement);
+```
+
+The ReadOnlyTsFile class has two `query` method to perform a query.
+* **Method 1**
+
+ ```
+ public QueryDataSet query(QueryExpression queryExpression) throws IOException
+ ```
+
+* **Method 2**
+
+ ```
+ public QueryDataSet query(QueryExpression queryExpression, long partitionStartOffset, long partitionEndOffset) throws IOException
+ ```
+
+ This method is designed for advanced applications such as the TsFile-Spark Connector.
+
+ * **params** : For method 2, two additional parameters are added to support partial query:
+ * ```partitionStartOffset```: start offset for a TsFile
+ * ```partitionEndOffset```: end offset for a TsFile
+
+ > **What is Partial Query ?**
+ >
+ > In some distributed file systems(e.g. HDFS), a file is split into severval parts which are called "Blocks" and stored in different nodes. Executing a query paralleled in each nodes involved makes better efficiency. Thus Partial Query is needed. Paritial Query only selects the results stored in the part split by ```QueryConstant.PARTITION_START_OFFSET``` and ```QueryConstant.PARTITION_END_OFFSET``` for a TsFile.
+
+#### QueryDataset Interface
+
+The query performed above will return a `QueryDataset` object.
+
+Here's the useful interfaces for user.
+
+
+* `bool hasNext();`
+
+ Return true if this dataset still has elements.
+* `List<Path> getPaths()`
+
+ Get the paths in this data set.
+* `List<TSDataType> getDataTypes();`
+
+ Get the data types. The class TSDataType is an enum class, the value will be one of the following:
+
+ BOOLEAN,
+ INT32,
+ INT64,
+ FLOAT,
+ DOUBLE,
+ TEXT;
+ * `RowRecord next() throws IOException;`
+
+ Get the next record.
+
+ The class `RowRecord` consists of a `long` timestamp and a `List<Field>` for data in different sensors,
+ we can use two getter methods to get them.
+
+ ```
+ long getTimestamp();
+ List<Field> getFields();
+ ```
+
+ To get data from one Field, use these methods:
+
+ ```
+ TSDataType getDataType();
+ Object getObjectValue();
+ ```
+
+#### Example for reading an existing TsFile
+
+
+You should install TsFile to your local maven repository.
+
+See reference: [Installation](./1-Installation.md)
+
+A more thorough example with query statement can be found at
+`/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileRead.java`
+
+```java
+package org.apache.iotdb.tsfile;
+import java.io.IOException;
+import java.util.ArrayList;
+import org.apache.iotdb.tsfile.read.ReadOnlyTsFile;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.expression.IExpression;
+import org.apache.iotdb.tsfile.read.expression.QueryExpression;
+import org.apache.iotdb.tsfile.read.expression.impl.BinaryExpression;
+import org.apache.iotdb.tsfile.read.expression.impl.GlobalTimeExpression;
+import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
+import org.apache.iotdb.tsfile.read.filter.TimeFilter;
+import org.apache.iotdb.tsfile.read.filter.ValueFilter;
+import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
+
+/**
+ * The class is to show how to read TsFile file named "test.tsfile".
+ * The TsFile file "test.tsfile" is generated from class TsFileWrite.
+ * Run TsFileWrite to generate the test.tsfile first
+ */
+public class TsFileRead {
+ private static void queryAndPrint(ArrayList<Path> paths, ReadOnlyTsFile readTsFile, IExpression statement)
+ throws IOException {
+ QueryExpression queryExpression = QueryExpression.create(paths, statement);
+ QueryDataSet queryDataSet = readTsFile.query(queryExpression);
+ while (queryDataSet.hasNext()) {
+ System.out.println(queryDataSet.next());
+ }
+ System.out.println("------------");
+ }
+
+ public static void main(String[] args) throws IOException {
+
+ // file path
+ String path = "test.tsfile";
+
+ // create reader and get the readTsFile interface
+ TsFileSequenceReader reader = new TsFileSequenceReader(path);
+ ReadOnlyTsFile readTsFile = new ReadOnlyTsFile(reader);
+ // use these paths(all sensors) for all the queries
+ ArrayList<Path> paths = new ArrayList<>();
+ paths.add(new Path("device_1.sensor_1"));
+ paths.add(new Path("device_1.sensor_2"));
+ paths.add(new Path("device_1.sensor_3"));
+
+ // no query statement
+ queryAndPrint(paths, readTsFile, null);
+
+ //close the reader when you left
+ reader.close();
+ }
+}
+
+```
+
+### User-specified config file path
+
+Default config file `tsfile-format.properties.template` is located at `/tsfile/src/main/resources` directory. If you want to use your own path, you can:
+```
+System.setProperty(TsFileConstant.TSFILE_CONF, "your config file path");
+```
+and then call:
+```
+TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
+```
+
+
diff --git a/tsfile/example/readme.md b/tsfile/example/readme.md
index 0e20ec8..0a3849a 100644
--- a/tsfile/example/readme.md
+++ b/tsfile/example/readme.md
@@ -68,10 +68,17 @@ The example is to show how to write and read a TsFile File.
## Run TsFileRead.java
-```
- The class is to show how to read TsFile file named "test.tsfile".
- The TsFile file "test.tsfile" is generated from class TsFileWrite1 or class TsFileWrite2, they generate the same TsFile file by two different ways
-```
+ This class is to show how to read TsFile file named "testDirect.tsfile".
+
+ The TsFile file "testDirect.tsfile" is generated from class TsFileWrite.
+
+ It generates the same TsFile(testDirect.tsfile and testWithJson.tsfile) file by two different ways
+
+ Run TsFileWrite to generate the testDirect.tsfile first
+
+## Run TsFileSequenceRead.java
+
+ This class is to show the structure of a TsFile.
### Notice
For detail, please refer to https://github.com/thulab/tsfile/wiki/Get-Started.
diff --git a/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileRead.java b/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileRead.java
index ce341db..264ab88 100644
--- a/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileRead.java
+++ b/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileRead.java
@@ -17,7 +17,6 @@
* under the License.
*/
package org.apache.iotdb.tsfile;
-
import java.io.IOException;
import java.util.ArrayList;
import org.apache.iotdb.tsfile.read.ReadOnlyTsFile;
@@ -34,84 +33,55 @@ import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
/**
* The class is to show how to read TsFile file named "test.tsfile".
- * The TsFile file "test.tsfile" is generated from class TsFileWrite2 or class TsFileWrite,
- * they generate the same TsFile file by two different ways
- * <p>
- * Run TsFileWrite1 or TsFileWrite to generate the test.tsfile first
+ * The TsFile file "test.tsfile" is generated from class TsFileWrite.
+ * Run TsFileWrite to generate the test.tsfile first
*/
public class TsFileRead {
+ private static void queryAndPrint(ArrayList<Path> paths, ReadOnlyTsFile readTsFile, IExpression statement)
+ throws IOException {
+ QueryExpression queryExpression = QueryExpression.create(paths, statement);
+ QueryDataSet queryDataSet = readTsFile.query(queryExpression);
+ while (queryDataSet.hasNext()) {
+ System.out.println(queryDataSet.next());
+ }
+ System.out.println("------------");
+ }
public static void main(String[] args) throws IOException {
// file path
String path = "test.tsfile";
- // read example : no filter
+ // create reader and get the readTsFile interface
TsFileSequenceReader reader = new TsFileSequenceReader(path);
ReadOnlyTsFile readTsFile = new ReadOnlyTsFile(reader);
+ // use these paths(all sensors) for all the queries
ArrayList<Path> paths = new ArrayList<>();
paths.add(new Path("device_1.sensor_1"));
paths.add(new Path("device_1.sensor_2"));
paths.add(new Path("device_1.sensor_3"));
- QueryExpression queryExpression = QueryExpression.create(paths, null);
- QueryDataSet queryDataSet = readTsFile.query(queryExpression);
- while (queryDataSet.hasNext()) {
- System.out.println(queryDataSet.next());
- }
- System.out.println("------------");
- reader.close();
- // time filter : 4 <= time <= 10
+ // no filter, should select 1 2 3 4 6 7 8
+ queryAndPrint(paths, readTsFile, null);
+
+ // time filter : 4 <= time <= 10, should select 4 6 7 8
IExpression timeFilter = BinaryExpression.and(new GlobalTimeExpression(TimeFilter.gtEq(4L)),
- new GlobalTimeExpression(TimeFilter.ltEq(10L)));
- reader = new TsFileSequenceReader(path);
- readTsFile = new ReadOnlyTsFile(reader);
- paths = new ArrayList<>();
- paths.add(new Path("device_1.sensor_1"));
- paths.add(new Path("device_1.sensor_2"));
- paths.add(new Path("device_1.sensor_3"));
- queryExpression = QueryExpression.create(paths, timeFilter);
- queryDataSet = readTsFile.query(queryExpression);
- while (queryDataSet.hasNext()) {
- System.out.println(queryDataSet.next());
- }
- System.out.println("------------");
- reader.close();
+ new GlobalTimeExpression(TimeFilter.ltEq(10L)));
+ queryAndPrint(paths, readTsFile, timeFilter);
- // value filter : device_1.sensor_2 <= 20
+ // value filter : device_1.sensor_2 <= 20, should select 1 2 4 6 7
IExpression valueFilter = new SingleSeriesExpression(new Path("device_1.sensor_2"),
- ValueFilter.ltEq(20));
- reader = new TsFileSequenceReader(path);
- readTsFile = new ReadOnlyTsFile(reader);
- paths = new ArrayList<>();
- paths.add(new Path("device_1.sensor_1"));
- paths.add(new Path("device_1.sensor_2"));
- paths.add(new Path("device_1.sensor_3"));
- queryExpression = QueryExpression.create(paths, valueFilter);
- queryDataSet = readTsFile.query(queryExpression);
- while (queryDataSet.hasNext()) {
- System.out.println(queryDataSet.next());
- }
- System.out.println("------------");
- reader.close();
+ ValueFilter.ltEq(20));
+ queryAndPrint(paths, readTsFile, valueFilter);
- // time filter : 4 <= time <= 10, value filter : device_1.sensor_3 >= 20
+ // time filter : 4 <= time <= 10, value filter : device_1.sensor_3 >= 20, should select 4 7 8
timeFilter = BinaryExpression.and(new GlobalTimeExpression(TimeFilter.gtEq(4L)),
- new GlobalTimeExpression(TimeFilter.ltEq(10L)));
+ new GlobalTimeExpression(TimeFilter.ltEq(10L)));
valueFilter = new SingleSeriesExpression(new Path("device_1.sensor_3"), ValueFilter.gtEq(20));
- reader = new TsFileSequenceReader(path);
- readTsFile = new ReadOnlyTsFile(reader);
- paths = new ArrayList<>();
- paths.add(new Path("device_1.sensor_1"));
- paths.add(new Path("device_1.sensor_2"));
- paths.add(new Path("device_1.sensor_3"));
IExpression finalFilter = BinaryExpression.and(timeFilter, valueFilter);
- queryExpression = QueryExpression.create(paths, finalFilter);
- queryDataSet = readTsFile.query(queryExpression);
- while (queryDataSet.hasNext()) {
- System.out.println(queryDataSet.next());
- }
+ queryAndPrint(paths, readTsFile, finalFilter);
+
+ //close the reader when you left
reader.close();
}
-
-}
\ No newline at end of file
+}
diff --git a/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileWrite.java b/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileWrite.java
index 58a3327..3b6cfeb 100644
--- a/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileWrite.java
+++ b/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileWrite.java
@@ -20,10 +20,6 @@
package org.apache.iotdb.tsfile;
import java.io.File;
-import java.io.IOException;
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.write.TsFileWriter;
@@ -34,171 +30,90 @@ import org.apache.iotdb.tsfile.write.record.datapoint.IntDataPoint;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
/**
* An example of writing data to TsFile
+ * It uses the interface:
+ * public void addMeasurement(MeasurementSchema MeasurementSchema) throws WriteProcessException
*/
public class TsFileWrite {
- private static void writeData(TsFileWriter tsFileWriter) throws IOException, WriteProcessException {
- // construct TSRecord
- TSRecord tsRecord = new TSRecord(1, "device_1");
- DataPoint dPoint1 = new FloatDataPoint("sensor_1", 1.2f);
- DataPoint dPoint2 = new IntDataPoint("sensor_2", 20);
- DataPoint dPoint3;
- tsRecord.addTuple(dPoint1);
- tsRecord.addTuple(dPoint2);
- // write a TSRecord to TsFile
- tsFileWriter.write(tsRecord);
-
- tsRecord = new TSRecord(2, "device_1");
- dPoint2 = new IntDataPoint("sensor_2", 20);
- dPoint3 = new IntDataPoint("sensor_3", 50);
- tsRecord.addTuple(dPoint2);
- tsRecord.addTuple(dPoint3);
- tsFileWriter.write(tsRecord);
+ public static void main(String args[]) {
+ try {
+ String path = "test.tsfile";
+ File f = new File(path);
+ if (f.exists()) {
+ f.delete();
+ }
+ TsFileWriter tsFileWriter = new TsFileWriter(f);
- tsRecord = new TSRecord(3, "device_1");
- dPoint1 = new FloatDataPoint("sensor_1", 1.4f);
- dPoint2 = new IntDataPoint("sensor_2", 21);
- tsRecord.addTuple(dPoint1);
- tsRecord.addTuple(dPoint2);
- tsFileWriter.write(tsRecord);
+ // add measurements into file schema
+ tsFileWriter
+ .addMeasurement(new MeasurementSchema("sensor_1", TSDataType.FLOAT, TSEncoding.RLE));
+ tsFileWriter
+ .addMeasurement(new MeasurementSchema("sensor_2", TSDataType.INT32, TSEncoding.TS_2DIFF));
+ tsFileWriter
+ .addMeasurement(new MeasurementSchema("sensor_3", TSDataType.INT32, TSEncoding.TS_2DIFF));
+ // construct TSRecord
+ TSRecord tsRecord = new TSRecord(1, "device_1");
+ DataPoint dPoint1 = new FloatDataPoint("sensor_1", 1.2f);
+ DataPoint dPoint2 = new IntDataPoint("sensor_2", 20);
+ DataPoint dPoint3;
+ tsRecord.addTuple(dPoint1);
+ tsRecord.addTuple(dPoint2);
- tsRecord = new TSRecord(4, "device_1");
- dPoint1 = new FloatDataPoint("sensor_1", 1.2f);
- dPoint2 = new IntDataPoint("sensor_2", 20);
- dPoint3 = new IntDataPoint("sensor_3", 51);
- tsRecord.addTuple(dPoint1);
- tsRecord.addTuple(dPoint2);
- tsRecord.addTuple(dPoint3);
- tsFileWriter.write(tsRecord);
+ // write a TSRecord to TsFile
+ tsFileWriter.write(tsRecord);
- tsRecord = new TSRecord(6, "device_1");
- dPoint1 = new FloatDataPoint("sensor_1", 7.2f);
- dPoint2 = new IntDataPoint("sensor_2", 10);
- dPoint3 = new IntDataPoint("sensor_3", 11);
- tsRecord.addTuple(dPoint1);
- tsRecord.addTuple(dPoint2);
- tsRecord.addTuple(dPoint3);
- tsFileWriter.write(tsRecord);
+ tsRecord = new TSRecord(2, "device_1");
+ dPoint2 = new IntDataPoint("sensor_2", 20);
+ dPoint3 = new IntDataPoint("sensor_3", 50);
+ tsRecord.addTuple(dPoint2);
+ tsRecord.addTuple(dPoint3);
+ tsFileWriter.write(tsRecord);
- tsRecord = new TSRecord(7, "device_1");
- dPoint1 = new FloatDataPoint("sensor_1", 6.2f);
- dPoint2 = new IntDataPoint("sensor_2", 20);
- dPoint3 = new IntDataPoint("sensor_3", 21);
- tsRecord.addTuple(dPoint1);
- tsRecord.addTuple(dPoint2);
- tsRecord.addTuple(dPoint3);
- tsFileWriter.write(tsRecord);
+ tsRecord = new TSRecord(3, "device_1");
+ dPoint1 = new FloatDataPoint("sensor_1", 1.4f);
+ dPoint2 = new IntDataPoint("sensor_2", 21);
+ tsRecord.addTuple(dPoint1);
+ tsRecord.addTuple(dPoint2);
+ tsFileWriter.write(tsRecord);
- tsRecord = new TSRecord(8, "device_1");
- dPoint1 = new FloatDataPoint("sensor_1", 9.2f);
- dPoint2 = new IntDataPoint("sensor_2", 30);
- dPoint3 = new IntDataPoint("sensor_3", 31);
- tsRecord.addTuple(dPoint1);
- tsRecord.addTuple(dPoint2);
- tsRecord.addTuple(dPoint3);
- tsFileWriter.write(tsRecord);
+ tsRecord = new TSRecord(4, "device_1");
+ dPoint1 = new FloatDataPoint("sensor_1", 1.2f);
+ dPoint2 = new IntDataPoint("sensor_2", 20);
+ dPoint3 = new IntDataPoint("sensor_3", 51);
+ tsRecord.addTuple(dPoint1);
+ tsRecord.addTuple(dPoint2);
+ tsRecord.addTuple(dPoint3);
+ tsFileWriter.write(tsRecord);
- // close TsFile
- tsFileWriter.close();
- }
- /**
- * There are two ways to construct a TsFile instance,they generate the identical TsFile file.
- * This method uses the first interface:
- * public void addMeasurementByJson(JSONObject measurement) throws WriteProcessException
- * The corresponding json string is provided below.
- * {
- * "schema": [
- * {
- * "measurement_id": "sensor_1",
- * "data_type": "FLOAT",
- * "encoding": "RLE",
- * "compressor" : "UNCOMPRESSED"
- * },
- * {
- * "measurement_id": "sensor_2",
- * "data_type": "INT32",
- * "encoding": "TS_2DIFF",
- * "compressor" : "UNCOMPRESSED"
- * },
- * {
- * "measurement_id": "sensor_3",
- * "data_type": "INT32",
- * "encoding": "TS_2DIFF",
- * "compressor" : "UNCOMPRESSED"
- * }
- * ]
- * }
- */
- private static void tsFileWriteWithJson() throws IOException,WriteProcessException {
- String path = "testWithJson.tsfile";
- String jsonText = "{\n" +
- " \"schema\": [\n" +
- " {\n" +
- " \"measurement_id\": \"sensor_1\",\n" +
- " \"data_type\": \"FLOAT\",\n" +
- " \"encoding\": \"RLE\",\n" +
- " \"compressor\" : \"UNCOMPRESSED\"\n" +
- " },\n" +
- " {\n" +
- " \"measurement_id\": \"sensor_2\",\n" +
- " \"data_type\": \"INT32\",\n" +
- " \"encoding\": \"TS_2DIFF\",\n" +
- " \"compressor\" : \"UNCOMPRESSED\"\n" +
- "\n" +
- " },\n" +
- " {\n" +
- " \"measurement_id\": \"sensor_3\",\n" +
- " \"data_type\": \"INT32\",\n" +
- " \"encoding\": \"TS_2DIFF\", \n" +
- " \"compressor\" : \"UNCOMPRESSED\"\n" +
- "\n" +
- " }\n" +
- " ]\n" +
- "}";
- File f = new File(path);
- if (f.exists()) {
- f.delete();
- }
- TsFileWriter tsFileWriter = new TsFileWriter(f);
- JSONObject j = JSONObject.parseObject(jsonText);
- JSONArray schemas = j.getJSONArray("schema");
- // add measurements into file schema
- for (int i = 0; i < schemas.size(); ++i) {
- tsFileWriter.addMeasurementByJson(schemas.getJSONObject(i));
- }
- writeData(tsFileWriter);
- }
+ tsRecord = new TSRecord(6, "device_1");
+ dPoint1 = new FloatDataPoint("sensor_1", 7.2f);
+ dPoint2 = new IntDataPoint("sensor_2", 10);
+ dPoint3 = new IntDataPoint("sensor_3", 11);
+ tsRecord.addTuple(dPoint1);
+ tsRecord.addTuple(dPoint2);
+ tsRecord.addTuple(dPoint3);
+ tsFileWriter.write(tsRecord);
+
+ tsRecord = new TSRecord(7, "device_1");
+ dPoint1 = new FloatDataPoint("sensor_1", 6.2f);
+ dPoint2 = new IntDataPoint("sensor_2", 20);
+ dPoint3 = new IntDataPoint("sensor_3", 21);
+ tsRecord.addTuple(dPoint1);
+ tsRecord.addTuple(dPoint2);
+ tsRecord.addTuple(dPoint3);
+ tsFileWriter.write(tsRecord);
- /**
- * There are two ways to construct a TsFile instance,they generate the identical TsFile file.
- * This method uses the second interface:
- * public void addMeasurement(MeasurementSchema MeasurementSchema) throws WriteProcessException
- * The measurements are identical to the json string provided above.
- */
- private static void tsFileWriteDirect() throws IOException,WriteProcessException {
- String path = "testDirect.tsfile";
- File f = new File(path);
- if (f.exists()) {
- f.delete();
- }
- TsFileWriter tsFileWriter = new TsFileWriter(f);
-
- // add measurements into file schema
- tsFileWriter
- .addMeasurement(new MeasurementSchema("sensor_1", TSDataType.FLOAT, TSEncoding.RLE));
- tsFileWriter
- .addMeasurement(new MeasurementSchema("sensor_2", TSDataType.INT32, TSEncoding.TS_2DIFF));
- tsFileWriter
- .addMeasurement(new MeasurementSchema("sensor_3", TSDataType.INT32, TSEncoding.TS_2DIFF));
- writeData(tsFileWriter);
- }
-
- public static void main(String args[]) {
- try {
- // Use a json string for all the measurements to write a TsFile
- tsFileWriteWithJson();
- // Write a TsFile by adding the measurements directly in the method
- tsFileWriteDirect();
+ tsRecord = new TSRecord(8, "device_1");
+ dPoint1 = new FloatDataPoint("sensor_1", 9.2f);
+ dPoint2 = new IntDataPoint("sensor_2", 30);
+ dPoint3 = new IntDataPoint("sensor_3", 31);
+ tsRecord.addTuple(dPoint1);
+ tsRecord.addTuple(dPoint2);
+ tsRecord.addTuple(dPoint3);
+ tsFileWriter.write(tsRecord);
+
+ // close TsFile
+ tsFileWriter.close();
} catch (Throwable e) {
e.printStackTrace();
System.out.println(e.getMessage());
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
index 94bfce1..62b740f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import com.alibaba.fastjson.JSONObject;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.exception.write.NoMeasurementException;
@@ -34,7 +33,6 @@ import org.apache.iotdb.tsfile.write.chunk.IChunkGroupWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
-import org.apache.iotdb.tsfile.write.schema.JsonConverter;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
import org.apache.iotdb.tsfile.write.writer.TsFileOutput;
@@ -166,9 +164,6 @@ public class TsFileWriter implements AutoCloseable{
* "encoding": "RLE" "compressor":"SNAPPY" }
* @throws WriteProcessException if the json is illegal or the measurement exists
*/
- public void addMeasurementByJson(JSONObject measurement) throws WriteProcessException {
- addMeasurement(JsonConverter.convertJsonToMeasurementSchema(measurement));
- }
/**
* Confirm whether the record is legal. If legal, add it into this RecordWriter.
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/FileSchema.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/FileSchema.java
index 03e05fd..db80efa 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/FileSchema.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/FileSchema.java
@@ -21,8 +21,6 @@ package org.apache.iotdb.tsfile.write.schema;
import java.util.HashMap;
import java.util.Map;
-import com.alibaba.fastjson.JSONObject;
-import org.apache.iotdb.tsfile.exception.write.InvalidJsonSchemaException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
/**
@@ -48,21 +46,6 @@ public class FileSchema {
}
/**
- * @deprecated
- * example: { "measurement_id": "sensor_cpu_50", "data_type": "INT32", "encoding": "RLE" }.
- * {"schema": [ { "measurement_id": "sensor_1", "data_type": "FLOAT", "encoding": "RLE" },
- * { "measurement_id": "sensor_2", "data_type": "INT32", "encoding": "TS_2DIFF" },
- * { "measurement_id": "sensor_3", "data_type": "INT32","encoding": "TS_2DIFF" } ] };
- *
- * @param jsonSchema
- * file schema in json format
- */
- @Deprecated
- public FileSchema(JSONObject jsonSchema) throws InvalidJsonSchemaException {
- this(JsonConverter.converterJsonToMeasurementSchemas(jsonSchema));
- }
-
- /**
* Construct a FileSchema using provided schema map.
* @param measurements a map whose key is the measurementId and value is the schema of
* the measurement.
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/JsonConverter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/JsonConverter.java
deleted file mode 100644
index 1156e4a..0000000
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/JsonConverter.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.tsfile.write.schema;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant;
-import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder;
-import org.apache.iotdb.tsfile.exception.write.InvalidJsonSchemaException;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p> JsonConverter is used to convert JsonObject to TSFile Schema which is a java class defined
- * in
- * tsfile project. the main function of this converter is to receive a json object of schema and
- * register all measurements. </p>
- *
- * <p> The format of JSON schema is as follow:
- *
- * <pre>
- * {
- * "schema": [
- * {
- * "measurement_id": "s1",
- * "data_type": "INT32",
- * "encoding": "RLE",
- * "compressor": "SNAPPY"
- * },
- * {
- * "measurement_id": "s3",
- * "data_type": "ENUMS",
- * "encoding": "BITMAP",
- * "compressor": "SNAPPY",
- * "enum_values":["MAN","WOMAN"],
- * "max_error":12,
- * "max_point_number":3
- * },
- * ...
- * ]
- * }
- *
- * </pre>
- * </p>
- *
- * @author kangrong
- * @see TSEncodingBuilder TSEncodingBuilder
- */
-public class JsonConverter {
-
- private static final Logger LOG = LoggerFactory.getLogger(JsonConverter.class);
- private JsonConverter(){}
- /**
- * input a FileSchema and a jsonObject to be converted.
- *
- * @param jsonSchema the whole schema in type of JSONObject
- * @return converted measurement descriptors
- * @throws InvalidJsonSchemaException throw exception when json schema is not valid
- */
-
- public static Map<String, MeasurementSchema> converterJsonToMeasurementSchemas(
- JSONObject jsonSchema)
- throws InvalidJsonSchemaException {
- Map<String, MeasurementSchema> result = new HashMap<>();
- if (!jsonSchema.containsKey(JsonFormatConstant.JSON_SCHEMA)) {
- throw new InvalidJsonSchemaException("missing fields:" + JsonFormatConstant.JSON_SCHEMA);
- }
-
- /**
- * get schema of all measurements in JSONArray from JSONObject.
- * <p></p>
- * "schema": [ { "measurement_id": "s1", "data_type": "INT32", "encoding": "RLE" },
- * { "measurement_id": "s2", "data_type": "INT64", "encoding": "TS_2DIFF" }... ]
- */
- JSONArray schemaArray = jsonSchema.getJSONArray(JsonFormatConstant.JSON_SCHEMA);
- for (int i = 0; i < schemaArray.size(); i++) {
- MeasurementSchema mdescriptor = convertJsonToMeasurementSchema(schemaArray.getJSONObject(i));
- result.put(mdescriptor.getMeasurementId(), mdescriptor);
- }
- return result;
- }
-
- /**
- * convert the input JSONObject to MeasurementSchema.
- *
- * @param measurementObj properties of one measurement
- *
- * an example: { "measurement_id": "s3", "data_type": "ENUMS", "encoding": "BITMAP", // some
- * measurement may have some properties "compressor": "SNAPPY", "enum_values":["MAN","WOMAN"],
- * "max_error":12, "max_point_number":3 }
- * @return converted MeasurementSchema
- */
- public static MeasurementSchema convertJsonToMeasurementSchema(JSONObject measurementObj) {
- if (!measurementObj.containsKey(JsonFormatConstant.MEASUREMENT_UID) && !measurementObj
- .containsKey(JsonFormatConstant.DATA_TYPE)
- && !measurementObj.containsKey(JsonFormatConstant.MEASUREMENT_ENCODING)
- && !measurementObj.containsKey(JsonFormatConstant.COMPRESS_TYPE)) {
- LOG.warn(
- "The format of given json is error. Give up to register this measurement. Given json:{}",
- measurementObj);
- return null;
- }
- // get measurementID
- String measurementId = measurementObj.getString(JsonFormatConstant.MEASUREMENT_UID);
- // get data type information
- TSDataType type = TSDataType.valueOf(measurementObj.getString(JsonFormatConstant.DATA_TYPE));
- // get encoding information
- TSEncoding encoding = TSEncoding
- .valueOf(measurementObj.getString(JsonFormatConstant.MEASUREMENT_ENCODING));
- CompressionType compressionType = measurementObj.containsKey(JsonFormatConstant.COMPRESS_TYPE)
- ? CompressionType.valueOf(measurementObj.getString(JsonFormatConstant.COMPRESS_TYPE))
- : CompressionType.valueOf(TSFileConfig.compressor);
- // all information of one series
- Map<String, String> props = new HashMap<>();
- for (Object key : measurementObj.keySet()) {
- if (key.equals(JsonFormatConstant.MEASUREMENT_ENCODING) || key
- .equals(JsonFormatConstant.COMPRESS_TYPE)
- || key.equals(JsonFormatConstant.MEASUREMENT_UID) || key
- .equals(JsonFormatConstant.DATA_TYPE)) {
- continue;
- }
- String value = measurementObj.get(key.toString()).toString();
- props.put(key.toString(), value);
- }
- return new MeasurementSchema(measurementId, type, encoding, compressionType, props);
- }
-
- /**
- * function for converting chunk group size from jsonSchema.
- */
- public static long convertJsonToChunkGroupSize(JSONObject jsonSchema) {
- if (jsonSchema.containsKey(JsonFormatConstant.ROW_GROUP_SIZE)) {
- return jsonSchema.getLong(JsonFormatConstant.ROW_GROUP_SIZE);
- }
- return 128L * 1024 * 1024;
- }
-
- /**
- * given a FileSchema and convert it into a JSONObject.
- *
- * @param fileSchema the given schema in type of {@linkplain FileSchema FileSchema}
- * @return converted File Schema in type of JSONObject
- */
- public static JSONObject converterFileSchemaToJson(FileSchema fileSchema) {
- /** JSONObject form of FileSchema. **/
- JSONObject ret = new JSONObject();
- /** JSONObject form of all MeasurementSchemas in fileSchema. **/
- JSONArray jsonSchema = new JSONArray();
-
- for (MeasurementSchema measurementSchema : fileSchema.getAllMeasurementSchema().values()) {
- jsonSchema.add(convertMeasurementSchemaToJson(measurementSchema));
- }
-
- ret.put(JsonFormatConstant.JSON_SCHEMA, jsonSchema);
- return ret;
- }
-
- /**
- * given a MeasurementSchema and convert it to a JSONObject.
- *
- * @param measurementSchema the given descriptor in type of {@linkplain MeasurementSchema
- * MeasurementSchema}
- * @return converted MeasurementSchema in form of JSONObject
- *
- * an example: { "measurement_id": "s3", "data_type": "ENUMS", "encoding": "BITMAP", // some
- * measurement may have some properties "compressor": "SNAPPY", "enum_values":["MAN","WOMAN"],
- * "max_error":12, "max_point_number":3 }
- */
- private static JSONObject convertMeasurementSchemaToJson(MeasurementSchema measurementSchema) {
- JSONObject measurementObj = new JSONObject();
- // put measurementID, data type, encoding info and properties into result JSONObject
- measurementObj.put(JsonFormatConstant.MEASUREMENT_UID, measurementSchema.getMeasurementId());
- measurementObj.put(JsonFormatConstant.DATA_TYPE, measurementSchema.getType().toString());
- measurementObj.put(JsonFormatConstant.MEASUREMENT_ENCODING,
- measurementSchema.getEncodingType().toString());
- measurementSchema.getProps().forEach(measurementObj::put);
- return measurementObj;
- }
-
-}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TsFileGeneratorForSeriesReaderByTimestamp.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TsFileGeneratorForSeriesReaderByTimestamp.java
index fa8cb19..ec2eebb 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TsFileGeneratorForSeriesReaderByTimestamp.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TsFileGeneratorForSeriesReaderByTimestamp.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.FileUtils;
@@ -38,6 +39,7 @@ import org.apache.iotdb.tsfile.utils.RecordUtils;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.junit.Assert;
import org.junit.Ignore;
import org.slf4j.Logger;
@@ -53,7 +55,7 @@ public class TsFileGeneratorForSeriesReaderByTimestamp {
public static String inputDataFile;
public static String outputDataFile = "target/testTsFile.tsfile";
public static String errorOutputDataFile;
- public static JSONObject jsonSchema;
+ public static FileSchema schema;
private static int rowCount;
private static int chunkGroupSize;
private static int pageSize;
@@ -72,7 +74,7 @@ public class TsFileGeneratorForSeriesReaderByTimestamp {
public static void prepare() throws IOException {
inputDataFile = "target/perTestInputData";
errorOutputDataFile = "target/perTestErrorOutputData.tsfile";
- jsonSchema = generateTestData();
+ generateTestData();
generateSampleInputDataFile();
}
@@ -157,7 +159,6 @@ public class TsFileGeneratorForSeriesReaderByTimestamp {
}
// LOG.info(jsonSchema.toString());
- FileSchema schema = new FileSchema(jsonSchema);
preChunkGroupSize = TSFileDescriptor.getInstance().getConfig().groupSizeInByte;
prePageSize = TSFileDescriptor.getInstance().getConfig().maxNumberOfPointsInPage;
TSFileDescriptor.getInstance().getConfig().groupSizeInByte = chunkGroupSize;
@@ -173,54 +174,16 @@ public class TsFileGeneratorForSeriesReaderByTimestamp {
LOG.info("write to file successfully!!");
}
- private static JSONObject generateTestData() {
+ private static void generateTestData() {
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
- JSONObject s1 = new JSONObject();
- s1.put(JsonFormatConstant.MEASUREMENT_UID, "s1");
- s1.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT32.toString());
- s1.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- s1.put(JsonFormatConstant.COMPRESS_TYPE, conf.compressor);
- JSONObject s2 = new JSONObject();
- s2.put(JsonFormatConstant.MEASUREMENT_UID, "s2");
- s2.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
- s2.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- s2.put(JsonFormatConstant.COMPRESS_TYPE, "SNAPPY");
- JSONObject s3 = new JSONObject();
- s3.put(JsonFormatConstant.MEASUREMENT_UID, "s3");
- s3.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
- s3.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- s2.put(JsonFormatConstant.COMPRESS_TYPE, "UNCOMPRESSED");
- JSONObject s4 = new JSONObject();
- s4.put(JsonFormatConstant.MEASUREMENT_UID, "s4");
- s4.put(JsonFormatConstant.DATA_TYPE, TSDataType.TEXT.toString());
- s4.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.PLAIN.toString());
- JSONObject s5 = new JSONObject();
- s5.put(JsonFormatConstant.MEASUREMENT_UID, "s5");
- s5.put(JsonFormatConstant.DATA_TYPE, TSDataType.BOOLEAN.toString());
- s5.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.PLAIN.toString());
- JSONObject s6 = new JSONObject();
- s6.put(JsonFormatConstant.MEASUREMENT_UID, "s6");
- s6.put(JsonFormatConstant.DATA_TYPE, TSDataType.FLOAT.toString());
- s6.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.RLE.toString());
- JSONObject s7 = new JSONObject();
- s7.put(JsonFormatConstant.MEASUREMENT_UID, "s7");
- s7.put(JsonFormatConstant.DATA_TYPE, TSDataType.DOUBLE.toString());
- s7.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.RLE.toString());
-
- JSONArray measureGroup1 = new JSONArray();
- measureGroup1.add(s1);
- measureGroup1.add(s2);
- measureGroup1.add(s3);
- measureGroup1.add(s4);
- measureGroup1.add(s5);
- measureGroup1.add(s6);
- measureGroup1.add(s7);
-
- JSONObject jsonSchema = new JSONObject();
- jsonSchema.put(JsonFormatConstant.DELTA_TYPE, "test_type");
- jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, measureGroup1);
- // System.out.println(jsonSchema);
- return jsonSchema;
+ schema = new FileSchema();
+ schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT32, TSEncoding.valueOf(conf.valueEncoder)));
+ schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder), CompressionType.UNCOMPRESSED));
+ schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder), CompressionType.SNAPPY));
+ schema.registerMeasurement(new MeasurementSchema("s4", TSDataType.TEXT, TSEncoding.PLAIN));
+ schema.registerMeasurement(new MeasurementSchema("s5", TSDataType.BOOLEAN, TSEncoding.PLAIN));
+ schema.registerMeasurement(new MeasurementSchema("s6", TSDataType.FLOAT, TSEncoding.RLE));
+ schema.registerMeasurement(new MeasurementSchema("s7", TSDataType.DOUBLE, TSEncoding.RLE));
}
static public void writeToFile(FileSchema schema)
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java
index f35f704..245a663 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,7 +47,7 @@ public class FileGenerator {
public static String inputDataFile;
public static String outputDataFile = "target/perTestOutputData.tsfile";
public static String errorOutputDataFile;
- public static JSONObject jsonSchema;
+ public static FileSchema schema;
public static int oldMaxNumberOfPointsInPage;
public static void generateFile(int rowCount, int maxNumberOfPointsInPage)
@@ -69,7 +70,7 @@ public class FileGenerator {
public static void prepare() throws IOException {
inputDataFile = "target/perTestInputData";
errorOutputDataFile = "target/perTestErrorOutputData.tsfile";
- jsonSchema = generateTestData();
+ generateTestData();
generateSampleInputDataFile();
}
@@ -156,8 +157,6 @@ public class FileGenerator {
errorFile.delete();
}
- // LOG.info(jsonSchema.toString());
- FileSchema schema = new FileSchema(jsonSchema);
// TSFileDescriptor.conf.chunkGroupSize = 2000;
// TSFileDescriptor.conf.pageSizeInByte = 100;
@@ -172,51 +171,16 @@ public class FileGenerator {
LOG.info("write to file successfully!!");
}
- private static JSONObject generateTestData() {
+ private static void generateTestData() {
+ schema = new FileSchema();
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
- JSONObject s1 = new JSONObject();
- s1.put(JsonFormatConstant.MEASUREMENT_UID, "s1");
- s1.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT32.toString());
- s1.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- JSONObject s2 = new JSONObject();
- s2.put(JsonFormatConstant.MEASUREMENT_UID, "s2");
- s2.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
- s2.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- JSONObject s3 = new JSONObject();
- s3.put(JsonFormatConstant.MEASUREMENT_UID, "s3");
- s3.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
- s3.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- JSONObject s4 = new JSONObject();
- s4.put(JsonFormatConstant.MEASUREMENT_UID, "s4");
- s4.put(JsonFormatConstant.DATA_TYPE, TSDataType.TEXT.toString());
- s4.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.PLAIN.toString());
- JSONObject s5 = new JSONObject();
- s5.put(JsonFormatConstant.MEASUREMENT_UID, "s5");
- s5.put(JsonFormatConstant.DATA_TYPE, TSDataType.BOOLEAN.toString());
- s5.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.PLAIN.toString());
- JSONObject s6 = new JSONObject();
- s6.put(JsonFormatConstant.MEASUREMENT_UID, "s6");
- s6.put(JsonFormatConstant.DATA_TYPE, TSDataType.FLOAT.toString());
- s6.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.RLE.toString());
- JSONObject s7 = new JSONObject();
- s7.put(JsonFormatConstant.MEASUREMENT_UID, "s7");
- s7.put(JsonFormatConstant.DATA_TYPE, TSDataType.DOUBLE.toString());
- s7.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.RLE.toString());
-
- JSONArray measureGroup1 = new JSONArray();
- measureGroup1.add(s1);
- measureGroup1.add(s2);
- measureGroup1.add(s3);
- measureGroup1.add(s4);
- measureGroup1.add(s5);
- measureGroup1.add(s6);
- measureGroup1.add(s7);
-
- JSONObject jsonSchema = new JSONObject();
- jsonSchema.put(JsonFormatConstant.DELTA_TYPE, "test_type");
- jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, measureGroup1);
- // System.out.println(jsonSchema);
- return jsonSchema;
+ schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT32, TSEncoding.valueOf(conf.valueEncoder)));
+ schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder)));
+ schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder)));
+ schema.registerMeasurement(new MeasurementSchema("s4", TSDataType.TEXT, TSEncoding.PLAIN));
+ schema.registerMeasurement(new MeasurementSchema("s5", TSDataType.BOOLEAN, TSEncoding.PLAIN));
+ schema.registerMeasurement(new MeasurementSchema("s6", TSDataType.FLOAT, TSEncoding.RLE));
+ schema.registerMeasurement(new MeasurementSchema("s7", TSDataType.DOUBLE, TSEncoding.RLE));
}
static public void writeToFile(FileSchema schema)
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtilsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtilsTest.java
index a2b3dc2..16f2e46 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtilsTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtilsTest.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.junit.Before;
import org.junit.Test;
@@ -44,51 +45,24 @@ import org.junit.Test;
public class RecordUtilsTest {
FileSchema schema;
- JSONObject jsonSchema = generateTestData();
- private static JSONObject generateTestData() {
+ private static FileSchema generateTestData() {
+ FileSchema fileSchema = new FileSchema();
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
- JSONObject s1 = new JSONObject();
- s1.put(JsonFormatConstant.MEASUREMENT_UID, "s1");
- s1.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT32.toString());
- s1.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- JSONObject s2 = new JSONObject();
- s2.put(JsonFormatConstant.MEASUREMENT_UID, "s2");
- s2.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
- s2.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- JSONObject s3 = new JSONObject();
- s3.put(JsonFormatConstant.MEASUREMENT_UID, "s3");
- s3.put(JsonFormatConstant.DATA_TYPE, TSDataType.FLOAT.toString());
- s3.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- JSONObject s4 = new JSONObject();
- s4.put(JsonFormatConstant.MEASUREMENT_UID, "s4");
- s4.put(JsonFormatConstant.DATA_TYPE, TSDataType.DOUBLE.toString());
- s4.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- JSONObject s5 = new JSONObject();
- s5.put(JsonFormatConstant.MEASUREMENT_UID, "s5");
- s5.put(JsonFormatConstant.DATA_TYPE, TSDataType.BOOLEAN.toString());
- s5.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.PLAIN.toString());
- JSONObject s6 = new JSONObject();
- s6.put(JsonFormatConstant.MEASUREMENT_UID, "s6");
- s6.put(JsonFormatConstant.DATA_TYPE, TSDataType.TEXT.toString());
- s6.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.PLAIN.toString());
- JSONArray columnGroup1 = new JSONArray();
- columnGroup1.add(s1);
- columnGroup1.add(s2);
- columnGroup1.add(s3);
- columnGroup1.add(s4);
- columnGroup1.add(s5);
- columnGroup1.add(s6);
-
- JSONObject jsonSchema = new JSONObject();
- jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, columnGroup1);
- jsonSchema.put(JsonFormatConstant.DELTA_TYPE, "1");
- return jsonSchema;
+ fileSchema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT32, TSEncoding.valueOf(conf.valueEncoder)));
+ fileSchema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder)));
+ fileSchema.registerMeasurement(new MeasurementSchema("s3", TSDataType.FLOAT, TSEncoding.valueOf(conf.valueEncoder)));
+ fileSchema.registerMeasurement(new MeasurementSchema("s4", TSDataType.DOUBLE, TSEncoding.valueOf(conf.valueEncoder)));
+ fileSchema.registerMeasurement(new MeasurementSchema("s5", TSDataType.BOOLEAN, TSEncoding.PLAIN));
+ fileSchema.registerMeasurement(new MeasurementSchema("s6", TSDataType.TEXT, TSEncoding.PLAIN));
+ return fileSchema;
}
+
@Before
public void prepare() throws WriteProcessException {
- schema = new FileSchema(jsonSchema);
+ schema = new FileSchema();
+ schema = generateTestData();
}
@Test
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java
index 2caf2a4..e8f3398 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.tsfile.utils.FileUtils.Unit;
import org.apache.iotdb.tsfile.utils.RecordUtils;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -59,7 +60,7 @@ public class PerfTest {
static public String inputDataFile;
static public String outputDataFile;
static public String errorOutputDataFile;
- static public JSONObject jsonSchema;
+ static public FileSchema schema;
static public Random r = new Random();
static private void generateSampleInputDataFile() throws IOException {
@@ -116,9 +117,6 @@ public class PerfTest {
errorFile.delete();
}
- // LOG.info(jsonSchema.toString());
- FileSchema schema = new FileSchema(jsonSchema);
-
// TSFileDescriptor.conf.chunkGroupSize = 2000;
// TSFileDescriptor.conf.pageSizeInByte = 100;
innerWriter = new TsFileWriter(file, schema, TSFileDescriptor.getInstance().getConfig());
@@ -173,34 +171,18 @@ public class PerfTest {
LOG.info("tsfile size:{} B", FileUtils.getLocalFileByte(outputDataFile, Unit.B));
}
- private static JSONObject generateTestData() {
+ private static FileSchema generateTestData() {
+ FileSchema fileSchema = new FileSchema();
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
- JSONObject s1 = new JSONObject();
- s1.put(JsonFormatConstant.MEASUREMENT_UID, "s1");
- s1.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
- s1.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- JSONObject s2 = new JSONObject();
- s2.put(JsonFormatConstant.MEASUREMENT_UID, "s2");
- s2.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
- s2.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
- JSONObject s3 = new JSONObject();
- s3.put(JsonFormatConstant.MEASUREMENT_UID, "s3");
- s3.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
- s3.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
+ fileSchema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder)));
+ fileSchema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder)));
+ fileSchema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder)));
+ fileSchema.registerMeasurement(new MeasurementSchema("s4", TSDataType.TEXT, TSEncoding.PLAIN));
JSONObject s4 = new JSONObject();
s4.put(JsonFormatConstant.MEASUREMENT_UID, "s4");
s4.put(JsonFormatConstant.DATA_TYPE, TSDataType.TEXT.toString());
s4.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.PLAIN.toString());
- JSONArray measureGroup1 = new JSONArray();
- measureGroup1.add(s1);
- measureGroup1.add(s2);
- measureGroup1.add(s3);
- measureGroup1.add(s4);
-
- JSONObject jsonSchema = new JSONObject();
- jsonSchema.put(JsonFormatConstant.DELTA_TYPE, "test_type");
- jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, measureGroup1);
- return jsonSchema;
+ return fileSchema;
}
@Before
@@ -213,7 +195,7 @@ public class PerfTest {
inputDataFile = "target/perTestInputData";
outputDataFile = "target/perTestOutputData.tsfile";
errorOutputDataFile = "target/perTestErrorOutputData.tsfile";
- jsonSchema = generateTestData();
+ schema = generateTestData();
generateSampleInputDataFile();
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/ReadPageInMemTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/ReadPageInMemTest.java
index b8cbf8e..d0d68c7 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/ReadPageInMemTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/ReadPageInMemTest.java
@@ -30,9 +30,11 @@ import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.RecordUtils;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -43,46 +45,21 @@ public class ReadPageInMemTest {
private File file = new File(filePath);
private TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
private TsFileWriter innerWriter;
- private FileSchema fileSchema = null;
+ private FileSchema schema = null;
private int pageSize;
private int ChunkGroupSize;
private int pageCheckSizeThreshold;
private int defaultMaxStringLength;
- private static JSONObject getJsonSchema() {
-
+ private static FileSchema getFileSchema() {
+ FileSchema fileSchema = new FileSchema();
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
- JSONObject s1 = new JSONObject();
- s1.put(JsonFormatConstant.MEASUREMENT_UID, "s1");
- s1.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT32.toString());
- s1.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
-
- JSONObject s2 = new JSONObject();
- s2.put(JsonFormatConstant.MEASUREMENT_UID, "s2");
- s2.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
- s2.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
-
- JSONObject s3 = new JSONObject();
- s3.put(JsonFormatConstant.MEASUREMENT_UID, "s3");
- s3.put(JsonFormatConstant.DATA_TYPE, TSDataType.FLOAT.toString());
- s3.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
-
- JSONObject s4 = new JSONObject();
- s4.put(JsonFormatConstant.MEASUREMENT_UID, "s4");
- s4.put(JsonFormatConstant.DATA_TYPE, TSDataType.DOUBLE.toString());
- s4.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
-
- JSONArray measureGroup = new JSONArray();
- measureGroup.add(s1);
- measureGroup.add(s2);
- measureGroup.add(s3);
- measureGroup.add(s4);
-
- JSONObject jsonSchema = new JSONObject();
- jsonSchema.put(JsonFormatConstant.DELTA_TYPE, "test_type");
- jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, measureGroup);
- return jsonSchema;
+ fileSchema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT32, TSEncoding.valueOf(conf.valueEncoder)));
+ fileSchema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder)));
+ fileSchema.registerMeasurement(new MeasurementSchema("s3", TSDataType.FLOAT, TSEncoding.valueOf(conf.valueEncoder)));
+ fileSchema.registerMeasurement(new MeasurementSchema("s4", TSDataType.DOUBLE, TSEncoding.valueOf(conf.valueEncoder)));
+ return fileSchema;
}
@Before
@@ -96,8 +73,8 @@ public class ReadPageInMemTest {
conf.pageCheckSizeThreshold = 1;
defaultMaxStringLength = conf.maxStringLength;
conf.maxStringLength = 2;
- fileSchema = new FileSchema(getJsonSchema());
- innerWriter = new TsFileWriter(new File(filePath), fileSchema, conf);
+ schema = getFileSchema();
+ innerWriter = new TsFileWriter(new File(filePath), schema, conf);
}
@After
@@ -114,7 +91,7 @@ public class ReadPageInMemTest {
String line = "";
for (int i = 1; i <= 3; i++) {
line = "root.car.d1," + i + ",s1,1,s2,1,s3,0.1,s4,0.1";
- TSRecord record = RecordUtils.parseSimpleTupleRecord(line, fileSchema);
+ TSRecord record = RecordUtils.parseSimpleTupleRecord(line, schema);
try {
innerWriter.write(record);
} catch (IOException | WriteProcessException e) {
@@ -124,7 +101,7 @@ public class ReadPageInMemTest {
}
for (int i = 4; i < 100; i++) {
line = "root.car.d1," + i + ",s1,1,s2,1,s3,0.1,s4,0.1";
- TSRecord record = RecordUtils.parseSimpleTupleRecord(line, fileSchema);
+ TSRecord record = RecordUtils.parseSimpleTupleRecord(line, schema);
try {
innerWriter.write(record);
} catch (IOException | WriteProcessException e) {
@@ -146,7 +123,7 @@ public class ReadPageInMemTest {
String line = "";
for (int i = 1; i <= 3; i++) {
line = "root.car.d1," + i + ",s1,1,s2,1,s3,0.1,s4,0.1";
- TSRecord record = RecordUtils.parseSimpleTupleRecord(line, fileSchema);
+ TSRecord record = RecordUtils.parseSimpleTupleRecord(line, schema);
try {
innerWriter.write(record);
} catch (IOException | WriteProcessException e) {
@@ -156,7 +133,7 @@ public class ReadPageInMemTest {
}
for (int i = 1; i <= 3; i++) {
line = "root.car.d2," + i + ",s1,1,s2,1,s3,0.1,s4,0.1";
- TSRecord record = RecordUtils.parseSimpleTupleRecord(line, fileSchema);
+ TSRecord record = RecordUtils.parseSimpleTupleRecord(line, schema);
try {
innerWriter.write(record);
} catch (IOException | WriteProcessException e) {
@@ -167,7 +144,7 @@ public class ReadPageInMemTest {
for (int i = 4; i < 100; i++) {
line = "root.car.d1," + i + ",s1,1,s2,1,s3,0.1,s4,0.1";
- TSRecord record = RecordUtils.parseSimpleTupleRecord(line, fileSchema);
+ TSRecord record = RecordUtils.parseSimpleTupleRecord(line, schema);
try {
innerWriter.write(record);
} catch (IOException | WriteProcessException e) {
@@ -178,7 +155,7 @@ public class ReadPageInMemTest {
for (int i = 4; i < 100; i++) {
line = "root.car.d2," + i + ",s1,1,s2,1,s3,0.1,s4,0.1";
- TSRecord record = RecordUtils.parseSimpleTupleRecord(line, fileSchema);
+ TSRecord record = RecordUtils.parseSimpleTupleRecord(line, schema);
try {
innerWriter.write(record);
} catch (IOException | WriteProcessException e) {
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java
index cf35fb2..7e85ebc 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java
@@ -22,22 +22,24 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.*;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Random;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
-import org.apache.commons.io.IOUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.utils.RecordUtils;
import org.apache.iotdb.tsfile.utils.StringContainer;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -58,15 +60,15 @@ public class WriteTest {
private String inputDataFile;
private String outputDataFile;
private String errorOutputDataFile;
- private String schemaFile;
private Random rm = new Random();
+ private ArrayList<MeasurementSchema> measurementArray;
private FileSchema schema;
private int stageSize = 4;
private int stageState = -1;
private int prePageSize;
private int prePageCheckThres;
private TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
- private JSONArray measurementArray;
+
private String[][] stageDeviceIds = {{"d1", "d2", "d3"}, {"d1"}, {"d2", "d3"}};
private String[] measurementIds = {"s0", "s1", "s2", "s3", "s4", "s5"};
private long longBase = System.currentTimeMillis() * 1000;
@@ -77,7 +79,6 @@ public class WriteTest {
inputDataFile = "target/writeTestInputData";
outputDataFile = "target/writeTestOutputData.tsfile";
errorOutputDataFile = "target/writeTestErrorOutputData.tsfile";
- schemaFile = "src/test/resources/test_write_schema.json";
// for each row, flush page forcely
prePageSize = conf.pageSizeInByte;
conf.pageSizeInByte = 0;
@@ -97,14 +98,19 @@ public class WriteTest {
if (errorFile.exists()) {
errorFile.delete();
}
-
- JSONObject emptySchema = JSON.parseObject("{\"delta_type\": \"test_type\",\"properties\": {\n"
- + "\"key1\": \"value1\",\n" + "\"key2\": \"value2\"\n" + "},\"schema\": [],}");
- try (InputStream inputStream = new FileInputStream(schemaFile)){
- String jsonStr = IOUtils.toString(inputStream, "utf8");
- measurementArray = JSON.parseObject(jsonStr).getJSONArray(JsonFormatConstant.JSON_SCHEMA);
- }
- schema = new FileSchema(emptySchema);
+ measurementArray = new ArrayList<>();
+ measurementArray.add(new MeasurementSchema("s0", TSDataType.INT32, TSEncoding.RLE));
+ measurementArray.add(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.TS_2DIFF));
+ HashMap<String,String> props = new HashMap<>();
+ props.put("max_point_number", "2");
+ measurementArray.add(new MeasurementSchema("s2", TSDataType.FLOAT, TSEncoding.RLE,
+ CompressionType.valueOf(TSFileConfig.compressor), props));
+ props = new HashMap<>();
+ props.put("max_point_number", "3");
+ measurementArray.add(new MeasurementSchema("s3", TSDataType.DOUBLE, TSEncoding.TS_2DIFF,
+ CompressionType.valueOf(TSFileConfig.compressor), props));
+ measurementArray.add(new MeasurementSchema("s4", TSDataType.BOOLEAN, TSEncoding.PLAIN));
+ schema = new FileSchema();
LOG.info(schema.toString());
tsFileWriter = new TsFileWriter(file, schema, conf);
}
@@ -197,7 +203,7 @@ public class WriteTest {
String[] strings;
// add all measurement except the last one at before writing
for (int i = 0; i < measurementArray.size() - 1; i++) {
- tsFileWriter.addMeasurementByJson((JSONObject) measurementArray.get(i));
+ tsFileWriter.addMeasurement(measurementArray.get(i));
}
while (true) {
if (lineCount % stageSize == 0) {
@@ -211,7 +217,7 @@ public class WriteTest {
}
if (lineCount == ROW_COUNT / 2) {
tsFileWriter
- .addMeasurementByJson((JSONObject) measurementArray.get(measurementArray.size() - 1));
+ .addMeasurement(measurementArray.get(measurementArray.size() - 1));
}
strings = getNextRecord(lineCount, stageState);
for (String str : strings) {
@@ -222,12 +228,12 @@ public class WriteTest {
lineCount++;
}
// test duplicate measurement adding
- JSONObject dupMeasure = (JSONObject) measurementArray.get(measurementArray.size() - 1);
+ MeasurementSchema dupMeasure = measurementArray.get(measurementArray.size() - 1);
try {
- tsFileWriter.addMeasurementByJson(dupMeasure);
+ tsFileWriter.addMeasurement(dupMeasure);
} catch (WriteProcessException e) {
assertEquals("given measurement has exists! " + dupMeasure
- .getString(JsonFormatConstant.MEASUREMENT_UID),
+ .getMeasurementId(),
e.getMessage());
}
try {
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/schema/converter/JsonConverterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/schema/converter/JsonConverterTest.java
deleted file mode 100755
index 4d0ad73..0000000
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/schema/converter/JsonConverterTest.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.tsfile.write.schema.converter;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.*;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONException;
-import com.alibaba.fastjson.JSONObject;
-import org.apache.commons.io.IOUtils;
-import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant;
-import org.apache.iotdb.tsfile.exception.write.InvalidJsonSchemaException;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
-import org.apache.iotdb.tsfile.write.schema.JsonConverter;
-import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.junit.Test;
-
-/**
- * @author kangrong
- */
-public class JsonConverterTest {
-
- @Test
- public void testJsonConverter() throws WriteProcessException {
- String path = "src/test/resources/test_schema.json";
- JSONObject obj = null;
- try (InputStream inputStream = new FileInputStream(path)) {
- String jsonStr = IOUtils.toString(inputStream, "utf8");
- obj = JSON.parseObject(jsonStr);
- } catch (JSONException | IOException e) {
- e.printStackTrace();
- fail();
- }
-
- FileSchema fileSchema = new FileSchema(obj);
- Collection<MeasurementSchema> measurements = fileSchema.getAllMeasurementSchema().values();
- String[] measureDesStrings = {"[s4,DOUBLE,RLE,{max_error=12},UNCOMPRESSED]",
- "[s5,INT32,TS_2DIFF,{},UNCOMPRESSED]", "[s1,INT32,RLE,{},UNCOMPRESSED]",
- "[s2,INT64,TS_2DIFF,{},UNCOMPRESSED]",
-
- };
- int i = 0;
- for (MeasurementSchema desc : measurements) {
- assertEquals(measureDesStrings[i++], desc.toString());
- }
-
- }
-
- @Test
- public void testConvertInJsonAndFileSchema() throws InvalidJsonSchemaException {
- String path = "src/test/resources/test_schema.json";
- JSONObject srcObj = null;
- try (InputStream inputStream = new FileInputStream(path)){
- String jsonStr = IOUtils.toString(inputStream, "utf8");
- srcObj = JSON.parseObject(jsonStr);
- } catch (JSONException | IOException e) {
- e.printStackTrace();
- fail();
- }
- FileSchema fileSchema = new FileSchema(srcObj);
- JSONObject descObj = JsonConverter.converterFileSchemaToJson(fileSchema);
- // check schema
- assertTrue(descObj.containsKey(JsonFormatConstant.JSON_SCHEMA));
- JSONArray srcSchemaArray = srcObj.getJSONArray(JsonFormatConstant.JSON_SCHEMA);
- JSONArray descSchemaArray = descObj.getJSONArray(JsonFormatConstant.JSON_SCHEMA);
- assertEquals(srcSchemaArray.size(), descSchemaArray.size());
- Map<String, JSONObject> descSchemaMap = new HashMap<>();
- for (int i = 0; i < descSchemaArray.size(); i++) {
- JSONObject descMeasureObj = descSchemaArray.getJSONObject(i);
- assertTrue(descMeasureObj.containsKey(JsonFormatConstant.MEASUREMENT_UID));
- descSchemaMap
- .put(descMeasureObj.getString(JsonFormatConstant.MEASUREMENT_UID), descMeasureObj);
- }
- for (int i = 0; i < srcSchemaArray.size(); i++) {
- JSONObject srcMeasureObj = srcSchemaArray.getJSONObject(i);
- assertTrue(srcMeasureObj.containsKey(JsonFormatConstant.MEASUREMENT_UID));
- String measureUID = srcMeasureObj.getString(JsonFormatConstant.MEASUREMENT_UID);
- assertTrue(descSchemaMap.containsKey(measureUID));
- checkJsonObjectEqual(srcMeasureObj, descSchemaMap.get(measureUID));
- }
- }
-
- /**
- * check whether two given JSONObjects are equal.
- *
- * @param obj1
- * the first JSONObject
- * @param obj2
- * the second JSONObject
- */
- private void checkJsonObjectEqual(JSONObject obj1, JSONObject obj2) {
- assertEquals(obj1.keySet().size(), obj2.keySet().size());
- obj1.keySet().forEach(k -> {
- String key = k;
- assertTrue(obj2.containsKey(key));
- assertTrue(obj2.containsKey(key));
- assertEquals(obj1.get(k).toString(), obj2.get(k).toString());
- });
- }
-
-}
diff --git a/tsfile/src/test/resources/test_schema.json b/tsfile/src/test/resources/test_schema.json
deleted file mode 100644
index 1a4d6b9..0000000
--- a/tsfile/src/test/resources/test_schema.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "schema": [
- {
- "measurement_id": "s1",
- "data_type": "INT32",
- "encoding": "RLE"
- },
- {
- "measurement_id": "s2",
- "data_type": "INT64",
- "encoding": "TS_2DIFF"
- },
- {
- "measurement_id": "s4",
- "data_type": "DOUBLE",
- "encoding": "RLE",
- "max_error": 12
- },
- {
- "measurement_id": "s5",
- "data_type": "INT32",
- "encoding": "TS_2DIFF"
- }
- ]
-}
\ No newline at end of file
diff --git a/tsfile/src/test/resources/test_write_schema.json b/tsfile/src/test/resources/test_write_schema.json
deleted file mode 100644
index 4f9142c..0000000
--- a/tsfile/src/test/resources/test_write_schema.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
- "schema": [
- {
- "measurement_id": "s0",
- "data_type": "INT32",
- "encoding": "RLE"
- },
- {
- "measurement_id": "s1",
- "data_type": "INT64",
- "encoding": "TS_2DIFF"
- },
- {
- "measurement_id": "s2",
- "data_type": "FLOAT",
- "encoding": "RLE",
- "max_point_number": 2
- },
- {
- "measurement_id": "s3",
- "data_type": "DOUBLE",
- "encoding": "TS_2DIFF",
- "max_point_number": 3
- },
- {
- "measurement_id": "s4",
- "data_type": "BOOLEAN",
- "encoding": "PLAIN"
- }
- ]
-}
\ No newline at end of file