You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/12/02 08:04:29 UTC
[iotdb] 05/11: change encode and decode way
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch NewTsFile
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5470d7c3c1c8bb9b846fd9d755dd81fb0e1e3766
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Thu Nov 26 11:46:46 2020 +0800
change encode and decode way
---
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 2 +-
.../iotdb/db/engine/upgrade/UpgradeTask.java | 308 ++---
.../iotdb/db/query/control/FileReaderManager.java | 5 -
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 +-
.../org/apache/iotdb/db/service/UpgradeSevice.java | 244 ++--
.../apache/iotdb/db/tools/TsFileSketchTool.java | 2 +-
.../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 1233 ++++++++++----------
.../iotdb/tsfile/encoding/common/EndianType.java | 26 -
.../iotdb/tsfile/encoding/decoder/Decoder.java | 8 +-
.../tsfile/encoding/decoder/FloatDecoder.java | 11 +-
.../tsfile/encoding/decoder/IntRleDecoder.java | 11 +-
.../tsfile/encoding/decoder/LongRleDecoder.java | 11 +-
.../tsfile/encoding/decoder/PlainDecoder.java | 21 +-
.../iotdb/tsfile/encoding/decoder/RleDecoder.java | 15 +-
.../tsfile/encoding/encoder/FloatEncoder.java | 6 +-
.../tsfile/encoding/encoder/IntRleEncoder.java | 8 +-
.../tsfile/encoding/encoder/LongRleEncoder.java | 6 +-
.../tsfile/encoding/encoder/PlainEncoder.java | 52 +-
.../iotdb/tsfile/encoding/encoder/RleEncoder.java | 20 +-
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 13 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 6 +-
.../tsfile/read/reader/chunk/ChunkReader.java | 4 +-
.../write/writer/RestorableTsFileIOWriter.java | 1 -
.../tsfile/encoding/decoder/IntRleDecoderTest.java | 83 +-
.../encoding/decoder/LongRleDecoderTest.java | 17 +-
.../metadata/statistics/DoubleStatisticsTest.java | 10 +-
.../metadata/statistics/FloatStatisticsTest.java | 11 +-
.../metadata/statistics/IntegerStatisticsTest.java | 10 +-
.../metadata/statistics/LongStatisticsTest.java | 15 +-
.../iotdb/tsfile/file/metadata/utils/Utils.java | 36 +-
.../tsfile/read/TsFileSequenceReaderTest.java | 2 +-
.../iotdb/tsfile/read/reader/PageReaderTest.java | 54 +-
.../iotdb/tsfile/write/TsFileIOWriterTest.java | 19 +-
.../iotdb/tsfile/write/writer/PageWriterTest.java | 50 +-
34 files changed, 1067 insertions(+), 1255 deletions(-)
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
index f837b4b..e9314fa 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
@@ -94,7 +94,7 @@ public class TsFileSequenceRead {
break;
case MetaMarker.CHUNK_GROUP_HEADER:
System.out.println("Chunk Group Footer position: " + reader.position());
- ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupFooter();
+ ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader();
System.out.println("device: " + chunkGroupHeader.getDeviceID());
break;
case MetaMarker.VERSION:
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java
index 015cc01..b170797 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java
@@ -1,154 +1,154 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.upgrade;
-
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.iotdb.db.concurrent.WrappedRunnable;
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.service.UpgradeSevice;
-import org.apache.iotdb.db.tools.upgrade.TsFileOnlineUpgradeTool;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
-import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
-import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class UpgradeTask extends WrappedRunnable {
-
- private TsFileResource upgradeResource;
- private static final Logger logger = LoggerFactory.getLogger(UpgradeTask.class);
- private static final String COMMA_SEPERATOR = ",";
- private static final int maxLevelNum = IoTDBDescriptor.getInstance().getConfig().getSeqLevelNum();
-
- private FSFactory fsFactory = FSFactoryProducer.getFSFactory();
-
- public UpgradeTask(TsFileResource upgradeResource) {
- this.upgradeResource = upgradeResource;
- }
-
- @Override
- public void runMayThrow() {
- try {
- List<TsFileResource> upgradedResources = generateUpgradedFiles();
- upgradeResource.writeLock();
- String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath();
- String oldModificationFilePath = oldTsfilePath + ModificationFile.FILE_SUFFIX;
- try {
- // delete old TsFile and resource
- upgradeResource.delete();
- File modificationFile = FSFactoryProducer.getFSFactory().getFile(oldModificationFilePath);
- // move upgraded TsFiles and modificationFile to their own partition directories
- for (TsFileResource upgradedResource : upgradedResources) {
- File upgradedFile = upgradedResource.getTsFile();
- long partition = upgradedResource.getTimePartition();
- String storageGroupPath = upgradedFile.getParentFile().getParentFile().getParent();
- File partitionDir = FSFactoryProducer.getFSFactory()
- .getFile(storageGroupPath, partition + "");
- if (!partitionDir.exists()) {
- partitionDir.mkdir();
- }
- FSFactoryProducer.getFSFactory().moveFile(upgradedFile,
- FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName()));
- upgradedResource.setFile(
- FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName()));
- // copy mods file to partition directories
- if (modificationFile.exists()) {
- Files.copy(modificationFile.toPath(),
- FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName()
- + ModificationFile.FILE_SUFFIX).toPath());
- }
- upgradedResource.serialize();
- // delete tmp partition folder when it is empty
- if (upgradedFile.getParentFile().isDirectory()
- && upgradedFile.getParentFile().listFiles().length == 0) {
- Files.delete(upgradedFile.getParentFile().toPath());
- }
- // rename all files to 0 level
- upgradedFile = upgradedResource.getTsFile();
- File zeroMergeVersionFile = getMaxMergeVersionFile(upgradedFile);
- fsFactory.moveFile(upgradedFile, zeroMergeVersionFile);
- fsFactory.moveFile(
- fsFactory.getFile(upgradedFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX),
- fsFactory
- .getFile(
- zeroMergeVersionFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX));
- upgradedResource.setFile(upgradedFile);
- }
- // delete old modificationFile
- if (modificationFile.exists()) {
- Files.delete(modificationFile.toPath());
- }
- // delete upgrade folder when it is empty
- if (upgradeResource.getTsFile().getParentFile().isDirectory()
- && upgradeResource.getTsFile().getParentFile().listFiles().length == 0) {
- Files.delete(upgradeResource.getTsFile().getParentFile().toPath());
- }
- upgradeResource.setUpgradedResources(upgradedResources);
- UpgradeLog.writeUpgradeLogFile(
- oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.UPGRADE_SUCCESS);
- upgradeResource.getUpgradeTsFileResourceCallBack().call(upgradeResource);
- } finally {
- upgradeResource.writeUnlock();
- }
- UpgradeSevice.setCntUpgradeFileNum(UpgradeSevice.getCntUpgradeFileNum() - 1);
- logger.info("Upgrade completes, file path:{} , the remaining upgraded file num: {}",
- oldTsfilePath, UpgradeSevice.getCntUpgradeFileNum());
- } catch (Exception e) {
- logger.error("meet error when upgrade file:{}", upgradeResource.getTsFile().getAbsolutePath(),
- e);
- }
- }
-
- private List<TsFileResource> generateUpgradedFiles() throws WriteProcessException {
- upgradeResource.readLock();
- String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath();
- List<TsFileResource> upgradedResources = new ArrayList<>();
- UpgradeLog.writeUpgradeLogFile(
- oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.BEGIN_UPGRADE_FILE);
- try {
- TsFileOnlineUpgradeTool.upgradeOneTsfile(oldTsfilePath, upgradedResources);
- UpgradeLog.writeUpgradeLogFile(
- oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.AFTER_UPGRADE_FILE);
- } catch (IOException e) {
- logger
- .error("generate upgrade file failed, the file to be upgraded:{}", oldTsfilePath, e);
- } finally {
- upgradeResource.readUnlock();
- }
- return upgradedResources;
- }
-
- private File getMaxMergeVersionFile(File seqFile) {
- String[] splits = seqFile.getName().replace(TSFILE_SUFFIX, "")
- .split(IoTDBConstant.FILE_NAME_SEPARATOR);
- return fsFactory.getFile(seqFile.getParentFile(),
- splits[0] + IoTDBConstant.FILE_NAME_SEPARATOR + splits[1]
- + IoTDBConstant.FILE_NAME_SEPARATOR + (maxLevelNum - 1) + TSFILE_SUFFIX);
- }
-
-}
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// */
+//package org.apache.iotdb.db.engine.upgrade;
+//
+//import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
+//
+//import java.io.File;
+//import java.io.IOException;
+//import java.nio.file.Files;
+//import java.util.ArrayList;
+//import java.util.List;
+//import org.apache.iotdb.db.concurrent.WrappedRunnable;
+//import org.apache.iotdb.db.conf.IoTDBConstant;
+//import org.apache.iotdb.db.conf.IoTDBDescriptor;
+//import org.apache.iotdb.db.engine.modification.ModificationFile;
+//import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+//import org.apache.iotdb.db.service.UpgradeSevice;
+//import org.apache.iotdb.db.tools.upgrade.TsFileOnlineUpgradeTool;
+//import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+//import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+//import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
+//import org.slf4j.Logger;
+//import org.slf4j.LoggerFactory;
+//
+//public class UpgradeTask extends WrappedRunnable {
+//
+// private TsFileResource upgradeResource;
+// private static final Logger logger = LoggerFactory.getLogger(UpgradeTask.class);
+// private static final String COMMA_SEPERATOR = ",";
+// private static final int maxLevelNum = IoTDBDescriptor.getInstance().getConfig().getSeqLevelNum();
+//
+// private FSFactory fsFactory = FSFactoryProducer.getFSFactory();
+//
+// public UpgradeTask(TsFileResource upgradeResource) {
+// this.upgradeResource = upgradeResource;
+// }
+//
+// @Override
+// public void runMayThrow() {
+// try {
+// List<TsFileResource> upgradedResources = generateUpgradedFiles();
+// upgradeResource.writeLock();
+// String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath();
+// String oldModificationFilePath = oldTsfilePath + ModificationFile.FILE_SUFFIX;
+// try {
+// // delete old TsFile and resource
+// upgradeResource.delete();
+// File modificationFile = FSFactoryProducer.getFSFactory().getFile(oldModificationFilePath);
+// // move upgraded TsFiles and modificationFile to their own partition directories
+// for (TsFileResource upgradedResource : upgradedResources) {
+// File upgradedFile = upgradedResource.getTsFile();
+// long partition = upgradedResource.getTimePartition();
+// String storageGroupPath = upgradedFile.getParentFile().getParentFile().getParent();
+// File partitionDir = FSFactoryProducer.getFSFactory()
+// .getFile(storageGroupPath, partition + "");
+// if (!partitionDir.exists()) {
+// partitionDir.mkdir();
+// }
+// FSFactoryProducer.getFSFactory().moveFile(upgradedFile,
+// FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName()));
+// upgradedResource.setFile(
+// FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName()));
+// // copy mods file to partition directories
+// if (modificationFile.exists()) {
+// Files.copy(modificationFile.toPath(),
+// FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName()
+// + ModificationFile.FILE_SUFFIX).toPath());
+// }
+// upgradedResource.serialize();
+// // delete tmp partition folder when it is empty
+// if (upgradedFile.getParentFile().isDirectory()
+// && upgradedFile.getParentFile().listFiles().length == 0) {
+// Files.delete(upgradedFile.getParentFile().toPath());
+// }
+// // rename all files to 0 level
+// upgradedFile = upgradedResource.getTsFile();
+// File zeroMergeVersionFile = getMaxMergeVersionFile(upgradedFile);
+// fsFactory.moveFile(upgradedFile, zeroMergeVersionFile);
+// fsFactory.moveFile(
+// fsFactory.getFile(upgradedFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX),
+// fsFactory
+// .getFile(
+// zeroMergeVersionFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX));
+// upgradedResource.setFile(upgradedFile);
+// }
+// // delete old modificationFile
+// if (modificationFile.exists()) {
+// Files.delete(modificationFile.toPath());
+// }
+// // delete upgrade folder when it is empty
+// if (upgradeResource.getTsFile().getParentFile().isDirectory()
+// && upgradeResource.getTsFile().getParentFile().listFiles().length == 0) {
+// Files.delete(upgradeResource.getTsFile().getParentFile().toPath());
+// }
+// upgradeResource.setUpgradedResources(upgradedResources);
+// UpgradeLog.writeUpgradeLogFile(
+// oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.UPGRADE_SUCCESS);
+// upgradeResource.getUpgradeTsFileResourceCallBack().call(upgradeResource);
+// } finally {
+// upgradeResource.writeUnlock();
+// }
+// UpgradeSevice.setCntUpgradeFileNum(UpgradeSevice.getCntUpgradeFileNum() - 1);
+// logger.info("Upgrade completes, file path:{} , the remaining upgraded file num: {}",
+// oldTsfilePath, UpgradeSevice.getCntUpgradeFileNum());
+// } catch (Exception e) {
+// logger.error("meet error when upgrade file:{}", upgradeResource.getTsFile().getAbsolutePath(),
+// e);
+// }
+// }
+//
+// private List<TsFileResource> generateUpgradedFiles() throws WriteProcessException {
+// upgradeResource.readLock();
+// String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath();
+// List<TsFileResource> upgradedResources = new ArrayList<>();
+// UpgradeLog.writeUpgradeLogFile(
+// oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.BEGIN_UPGRADE_FILE);
+// try {
+// TsFileOnlineUpgradeTool.upgradeOneTsfile(oldTsfilePath, upgradedResources);
+// UpgradeLog.writeUpgradeLogFile(
+// oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.AFTER_UPGRADE_FILE);
+// } catch (IOException e) {
+// logger
+// .error("generate upgrade file failed, the file to be upgraded:{}", oldTsfilePath, e);
+// } finally {
+// upgradeResource.readUnlock();
+// }
+// return upgradedResources;
+// }
+//
+// private File getMaxMergeVersionFile(File seqFile) {
+// String[] splits = seqFile.getName().replace(TSFILE_SUFFIX, "")
+// .split(IoTDBConstant.FILE_NAME_SEPARATOR);
+// return fsFactory.getFile(seqFile.getParentFile(),
+// splits[0] + IoTDBConstant.FILE_NAME_SEPARATOR + splits[1]
+// + IoTDBConstant.FILE_NAME_SEPARATOR + (maxLevelNum - 1) + TSFILE_SUFFIX);
+// }
+//
+//}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
index 776fcdb..9a9eb18 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
@@ -33,7 +33,6 @@ import org.apache.iotdb.db.service.ServiceType;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.UnClosedTsFileReader;
-import org.apache.iotdb.tsfile.v1.read.TsFileSequenceReaderForV1;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -168,10 +167,6 @@ public class FileReaderManager implements IService {
else {
tsFileReader = new TsFileSequenceReader(filePath);
switch (tsFileReader.readVersionNumber()) {
- case TSFileConfig.VERSION_NUMBER_V1:
- tsFileReader.close();
- tsFileReader = new TsFileSequenceReaderForV1(filePath);
- break;
case TSFileConfig.VERSION_NUMBER_V2:
break;
default:
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index f56e7ca..60d7095 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -137,7 +137,7 @@ public class IoTDB implements IoTDBMBean {
}
registerManager.register(SyncServerManager.getInstance());
- registerManager.register(UpgradeSevice.getINSTANCE());
+// registerManager.register(UpgradeSevice.getINSTANCE());
registerManager.register(MergeManager.getINSTANCE());
registerManager.register(CompactionMergeTaskPoolManager.getInstance());
diff --git a/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java b/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java
index b7d1b89..d3260c7 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java
@@ -1,122 +1,122 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.service;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.StorageEngine;
-import org.apache.iotdb.db.engine.upgrade.UpgradeLog;
-import org.apache.iotdb.db.engine.upgrade.UpgradeTask;
-import org.apache.iotdb.db.exception.StartupException;
-import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.utils.UpgradeUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class UpgradeSevice implements IService {
-
- private static final Logger logger = LoggerFactory.getLogger(UpgradeSevice.class);
-
- private static final UpgradeSevice INSTANCE = new UpgradeSevice();
- private ExecutorService upgradeThreadPool;
- private AtomicInteger threadCnt = new AtomicInteger();
- private static int cntUpgradeFileNum;
-
-
- private UpgradeSevice() {
- }
-
- public static UpgradeSevice getINSTANCE() {
- return INSTANCE;
- }
-
- @Override
- public void start() throws StartupException {
- int updateThreadNum = IoTDBDescriptor.getInstance().getConfig().getUpgradeThreadNum();
- if (updateThreadNum <= 0) {
- updateThreadNum = 1;
- }
- upgradeThreadPool = Executors.newFixedThreadPool(updateThreadNum,
- r -> new Thread(r, "UpgradeThread-" + threadCnt.getAndIncrement()));
- UpgradeLog.createUpgradeLog();
- countUpgradeFiles();
- if (cntUpgradeFileNum == 0) {
- stop();
- return;
- }
- upgradeAll();
- }
-
- @Override
- public void stop() {
- UpgradeLog.closeLogWriter();
- if (upgradeThreadPool != null) {
- upgradeThreadPool.shutdownNow();
- logger.info("Waiting for upgrade task pool to shut down");
- while (!upgradeThreadPool.isTerminated()) {
- // wait
- }
- upgradeThreadPool = null;
- logger.info("Upgrade service stopped");
- }
- }
-
- @Override
- public ServiceType getID() {
- return ServiceType.UPGRADE_SERVICE;
- }
-
-
- public static void setCntUpgradeFileNum(int cntUpgradeFileNum) {
- UpgradeUtils.getCntUpgradeFileLock().writeLock().lock();
- try {
- UpgradeSevice.cntUpgradeFileNum = cntUpgradeFileNum;
- } finally {
- UpgradeUtils.getCntUpgradeFileLock().writeLock().unlock();
- }
- }
-
- public static int getCntUpgradeFileNum() {
- UpgradeUtils.getCntUpgradeFileLock().readLock().lock();
- try {
- return cntUpgradeFileNum;
- } finally {
- UpgradeUtils.getCntUpgradeFileLock().readLock().unlock();
- }
- }
-
- public void submitUpgradeTask(UpgradeTask upgradeTask) {
- upgradeThreadPool.submit(upgradeTask);
- }
-
- private static void countUpgradeFiles() {
- cntUpgradeFileNum = StorageEngine.getInstance().countUpgradeFiles();
- logger.info("finish counting upgrading files, total num:{}", cntUpgradeFileNum);
- }
-
- private static void upgradeAll() {
- try {
- StorageEngine.getInstance().upgradeAll();
- } catch (StorageEngineException e) {
- logger.error("Cannot perform a global upgrade because", e);
- }
- }
-}
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// */
+//package org.apache.iotdb.db.service;
+//
+//import java.util.concurrent.ExecutorService;
+//import java.util.concurrent.Executors;
+//import java.util.concurrent.atomic.AtomicInteger;
+//import org.apache.iotdb.db.conf.IoTDBDescriptor;
+//import org.apache.iotdb.db.engine.StorageEngine;
+//import org.apache.iotdb.db.engine.upgrade.UpgradeLog;
+//import org.apache.iotdb.db.engine.upgrade.UpgradeTask;
+//import org.apache.iotdb.db.exception.StartupException;
+//import org.apache.iotdb.db.exception.StorageEngineException;
+//import org.apache.iotdb.db.utils.UpgradeUtils;
+//import org.slf4j.Logger;
+//import org.slf4j.LoggerFactory;
+//
+//public class UpgradeSevice implements IService {
+//
+// private static final Logger logger = LoggerFactory.getLogger(UpgradeSevice.class);
+//
+// private static final UpgradeSevice INSTANCE = new UpgradeSevice();
+// private ExecutorService upgradeThreadPool;
+// private AtomicInteger threadCnt = new AtomicInteger();
+// private static int cntUpgradeFileNum;
+//
+//
+// private UpgradeSevice() {
+// }
+//
+// public static UpgradeSevice getINSTANCE() {
+// return INSTANCE;
+// }
+//
+// @Override
+// public void start() throws StartupException {
+// int updateThreadNum = IoTDBDescriptor.getInstance().getConfig().getUpgradeThreadNum();
+// if (updateThreadNum <= 0) {
+// updateThreadNum = 1;
+// }
+// upgradeThreadPool = Executors.newFixedThreadPool(updateThreadNum,
+// r -> new Thread(r, "UpgradeThread-" + threadCnt.getAndIncrement()));
+// UpgradeLog.createUpgradeLog();
+// countUpgradeFiles();
+// if (cntUpgradeFileNum == 0) {
+// stop();
+// return;
+// }
+// upgradeAll();
+// }
+//
+// @Override
+// public void stop() {
+// UpgradeLog.closeLogWriter();
+// if (upgradeThreadPool != null) {
+// upgradeThreadPool.shutdownNow();
+// logger.info("Waiting for upgrade task pool to shut down");
+// while (!upgradeThreadPool.isTerminated()) {
+// // wait
+// }
+// upgradeThreadPool = null;
+// logger.info("Upgrade service stopped");
+// }
+// }
+//
+// @Override
+// public ServiceType getID() {
+// return ServiceType.UPGRADE_SERVICE;
+// }
+//
+//
+// public static void setCntUpgradeFileNum(int cntUpgradeFileNum) {
+// UpgradeUtils.getCntUpgradeFileLock().writeLock().lock();
+// try {
+// UpgradeSevice.cntUpgradeFileNum = cntUpgradeFileNum;
+// } finally {
+// UpgradeUtils.getCntUpgradeFileLock().writeLock().unlock();
+// }
+// }
+//
+// public static int getCntUpgradeFileNum() {
+// UpgradeUtils.getCntUpgradeFileLock().readLock().lock();
+// try {
+// return cntUpgradeFileNum;
+// } finally {
+// UpgradeUtils.getCntUpgradeFileLock().readLock().unlock();
+// }
+// }
+//
+// public void submitUpgradeTask(UpgradeTask upgradeTask) {
+// upgradeThreadPool.submit(upgradeTask);
+// }
+//
+// private static void countUpgradeFiles() {
+// cntUpgradeFileNum = StorageEngine.getInstance().countUpgradeFiles();
+// logger.info("finish counting upgrading files, total num:{}", cntUpgradeFileNum);
+// }
+//
+// private static void upgradeAll() {
+// try {
+// StorageEngine.getInstance().upgradeAll();
+// } catch (StorageEngineException e) {
+// logger.error("Cannot perform a global upgrade because", e);
+// }
+// }
+//}
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
index 779fb56..b72d99a 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -105,7 +105,7 @@ public class TsFileSketchTool {
}
// chunkGroupFooter begins
printlnBoth(pw, String.format("%20s", chunkEndPos) + "|\t[Chunk Group Footer]");
- ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupFooter(chunkEndPos, false);
+ ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader(chunkEndPos, false);
printlnBoth(pw, String.format("%20s", "") + "|\t\t[marker] 0");
printlnBoth(pw,
String.format("%20s", "") + "|\t\t[deviceID] " + chunkGroupHeader.getDeviceID());
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
index 52b7eb3..7f75ce8 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
@@ -1,619 +1,614 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.tools.upgrade;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.iotdb.db.engine.StorageEngine;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.compress.IUnCompressor;
-import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
-import org.apache.iotdb.tsfile.exception.write.PageException;
-import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
-import org.apache.iotdb.tsfile.file.MetaMarker;
-import org.apache.iotdb.tsfile.file.footer.ChunkGroupHeader;
-import org.apache.iotdb.tsfile.file.header.ChunkHeader;
-import org.apache.iotdb.tsfile.file.header.PageHeader;
-import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
-import org.apache.iotdb.tsfile.read.common.BatchData;
-import org.apache.iotdb.tsfile.read.reader.TsFileInput;
-import org.apache.iotdb.tsfile.read.reader.page.PageReader;
-import org.apache.iotdb.tsfile.utils.Binary;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-import org.apache.iotdb.tsfile.v1.file.metadata.ChunkGroupMetaDataV1;
-import org.apache.iotdb.tsfile.v1.file.metadata.TsDeviceMetadataIndexV1;
-import org.apache.iotdb.tsfile.v1.file.metadata.TsDeviceMetadataV1;
-import org.apache.iotdb.tsfile.v1.file.metadata.TsFileMetadataV1;
-import org.apache.iotdb.tsfile.v1.file.utils.HeaderUtils;
-import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
-import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
-import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TsFileOnlineUpgradeTool implements AutoCloseable {
-
- private static final Logger logger = LoggerFactory.getLogger(TsFileOnlineUpgradeTool.class);
-
- private TsFileInput tsFileInput;
- private long fileMetadataPos;
- private int fileMetadataSize;
- private ByteBuffer markerBuffer = ByteBuffer.allocate(Byte.BYTES);
- private Decoder defaultTimeDecoder = Decoder.getDecoderByType(
- TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()),
- TSDataType.INT64);
- private Decoder valueDecoder;
- protected String file;
-
- // PartitionId -> TsFileIOWriter
- private Map<Long, TsFileIOWriter> partitionWriterMap;
-
- /**
- * Create a file reader of the given file. The reader will read the tail of the file to get the
- * file metadata size.Then the reader will skip the first TSFileConfig.OLD_MAGIC_STRING.length()
- * bytes of the file for preparing reading real data.
- *
- * @param file the data file
- * @throws IOException If some I/O error occurs
- */
- public TsFileOnlineUpgradeTool(String file) throws IOException {
- this(file, true);
- }
-
- /**
- * construct function for TsfileOnlineUpgradeTool.
- *
- * @param file -given file name
- * @param loadMetadataSize -load meta data size
- */
- public TsFileOnlineUpgradeTool(String file, boolean loadMetadataSize) throws IOException {
- this.file = file;
- tsFileInput = FSFactoryProducer.getFileInputFactory().getTsFileInput(file);
- partitionWriterMap = new HashMap<>();
- try {
- if (loadMetadataSize) {
- loadMetadataSize();
- }
- } catch (Exception e) {
- tsFileInput.close();
- throw e;
- }
- }
-
- /**
- * upgrade a single tsfile
- *
- * @param tsFileName old version tsFile's absolute path
- * @param upgradedResources new version tsFiles' resources
- */
- public static void upgradeOneTsfile(String tsFileName, List<TsFileResource> upgradedResources)
- throws IOException, WriteProcessException {
- try (TsFileOnlineUpgradeTool updater = new TsFileOnlineUpgradeTool(tsFileName)) {
- updater.upgradeFile(upgradedResources);
- }
- }
-
- /**
- *
- */
- public void loadMetadataSize() throws IOException {
- ByteBuffer metadataSize = ByteBuffer.allocate(Integer.BYTES);
- tsFileInput.read(metadataSize,
- tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES);
- metadataSize.flip();
- // read file metadata size and position
- fileMetadataSize = ReadWriteIOUtils.readInt(metadataSize);
- fileMetadataPos =
- tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES
- - fileMetadataSize;
- // skip the magic header
- position(TSFileConfig.MAGIC_STRING.length());
- }
-
- public String readTailMagic() throws IOException {
- long totalSize = tsFileInput.size();
-
- ByteBuffer magicStringBytes = ByteBuffer.allocate(TSFileConfig.MAGIC_STRING.length());
- tsFileInput.read(magicStringBytes, totalSize - TSFileConfig.MAGIC_STRING.length());
- magicStringBytes.flip();
- return new String(magicStringBytes.array());
- }
-
- /**
- * whether the file is a complete TsFile: only if the head magic and tail magic string exists.
- */
- public boolean isComplete() throws IOException {
- return tsFileInput.size() >= TSFileConfig.MAGIC_STRING.length() * 2 && readTailMagic()
- .equals(readHeadMagic());
- }
-
- /**
- * this function does not modify the position of the file reader.
- */
- public String readHeadMagic() throws IOException {
- return readHeadMagic(false);
- }
-
- /**
- * @param movePosition whether move the position of the file reader after reading the magic header
- * to the end of the magic head string.
- */
- public String readHeadMagic(boolean movePosition) throws IOException {
- ByteBuffer magicStringBytes = ByteBuffer.allocate(TSFileConfig.MAGIC_STRING.length());
- if (movePosition) {
- tsFileInput.position(0);
- tsFileInput.read(magicStringBytes);
- } else {
- tsFileInput.read(magicStringBytes, 0);
- }
- magicStringBytes.flip();
- return new String(magicStringBytes.array());
- }
-
- /**
- * this function reads version number and checks compatibility of TsFile.
- */
- public String readVersionNumber() throws IOException {
- ByteBuffer versionNumberBytes = ByteBuffer
- .allocate(TSFileConfig.VERSION_NUMBER_V2.getBytes().length);
- tsFileInput.position(TSFileConfig.MAGIC_STRING.getBytes().length);
- tsFileInput.read(versionNumberBytes);
- versionNumberBytes.flip();
- return new String(versionNumberBytes.array());
- }
-
- /**
- * this function does not modify the position of the file reader.
- */
- public TsFileMetadataV1 readFileMetadata() throws IOException {
- return TsFileMetadataV1.deserializeFrom(readData(fileMetadataPos, fileMetadataSize));
- }
-
- /**
- * this function does not modify the position of the file reader.
- */
- public TsDeviceMetadataV1 readTsDeviceMetaData(TsDeviceMetadataIndexV1 index) throws IOException {
- return TsDeviceMetadataV1.deserializeFrom(readData(index.getOffset(), index.getLen()));
- }
-
- /**
- * read data from current position of the input, and deserialize it to a CHUNK_GROUP_FOOTER. <br>
- * This method is not threadsafe.
- *
- * @return a CHUNK_GROUP_FOOTER
- * @throws IOException io error
- */
- public ChunkGroupHeader readChunkGroupFooter() throws IOException {
- return ChunkGroupHeader.deserializeFrom(tsFileInput.wrapAsInputStream(), true);
- }
-
- /**
- * read data from current position of the input, and deserialize it to a CHUNK_HEADER. <br> This
- * method is not threadsafe.
- *
- * @return a CHUNK_HEADER
- * @throws IOException io error
- */
- public ChunkHeader readChunkHeader() throws IOException {
- return HeaderUtils.deserializeChunkHeaderV1(tsFileInput.wrapAsInputStream(), true);
- }
-
- /**
- * not thread safe.
- *
- * @param type given tsfile data type
- */
- public PageHeader readPageHeader(TSDataType type) throws IOException {
- return HeaderUtils.deserializePageHeaderV1(tsFileInput.wrapAsInputStream(), type);
- }
-
- public ByteBuffer readPage(PageHeader header, CompressionType type)
- throws IOException {
- ByteBuffer buffer = readData(-1, header.getCompressedSize());
- IUnCompressor unCompressor = IUnCompressor.getUnCompressor(type);
- ByteBuffer uncompressedBuffer = ByteBuffer.allocate(header.getUncompressedSize());
- if (type == CompressionType.UNCOMPRESSED) {
- return buffer;
- }
- unCompressor.uncompress(buffer.array(), buffer.position(), buffer.remaining(),
- uncompressedBuffer.array(),
- 0);
- return uncompressedBuffer;
- }
-
- public ByteBuffer readCompressedPage(PageHeader header) throws IOException {
- return readData(-1, header.getCompressedSize());
- }
-
- public long position() throws IOException {
- return tsFileInput.position();
- }
-
- public void position(long offset) throws IOException {
- tsFileInput.position(offset);
- }
-
- /**
- * read one byte from the input. <br> this method is not thread safe
- */
- public byte readMarker() throws IOException {
- markerBuffer.clear();
- if (ReadWriteIOUtils.readAsPossible(tsFileInput, markerBuffer) == 0) {
- throw new IOException("reach the end of the file.");
- }
- markerBuffer.flip();
- return markerBuffer.get();
- }
-
- public byte readMarker(long position) throws IOException {
- return readData(position, Byte.BYTES).get();
- }
-
- public void close() throws IOException {
- this.tsFileInput.close();
- }
-
- public String getFileName() {
- return this.file;
- }
-
- /**
- * read data from tsFileInput, from the current position (if position = -1), or the given
- * position. <br> if position = -1, the tsFileInput's position will be changed to the current
- * position + real data size that been read. Other wise, the tsFileInput's position is not
- * changed.
- *
- * @param position the start position of data in the tsFileInput, or the current position if
- * position = -1
- * @param size the size of data that want to read
- * @return data that been read.
- */
- private ByteBuffer readData(long position, int size) throws IOException {
- ByteBuffer buffer = ByteBuffer.allocate(size);
- if (position == -1) {
- if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != size) {
- throw new IOException("reach the end of the data");
- }
- } else {
- if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, size) != size) {
- throw new IOException("reach the end of the data");
- }
- }
- buffer.flip();
- return buffer;
- }
-
- /**
- * upgrade file and resource
- *
- * @throws IOException, WriteProcessException
- */
- @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- public void upgradeFile(List<TsFileResource> upgradedResources)
- throws IOException, WriteProcessException {
- File oldTsFile = FSFactoryProducer.getFSFactory().getFile(this.file);
-
- // check if the old TsFile has correct header
- if (!fileCheck(oldTsFile)) {
- return;
- }
-
- // ChunkGroupOffset -> version
- Map<Long, Long> oldVersionInfo = getVersionInfo();
-
- // start to scan chunks and chunkGroups
- long startOffsetOfChunkGroup = 0;
- boolean newChunkGroup = true;
- long versionOfChunkGroup = 0;
- int chunkGroupCount = 0;
- List<List<PageHeader>> pageHeadersInChunkGroup = new ArrayList<>();
- List<List<ByteBuffer>> pageDataInChunkGroup = new ArrayList<>();
- List<List<Boolean>> pagePartitionInfoInChunkGroup = new ArrayList<>();
- byte marker;
- List<MeasurementSchema> measurementSchemaList = new ArrayList<>();
- try {
- while ((marker = this.readMarker()) != MetaMarker.SEPARATOR) {
- switch (marker) {
- case MetaMarker.CHUNK_HEADER:
- // this is the first chunk of a new ChunkGroup.
- if (newChunkGroup) {
- newChunkGroup = false;
- startOffsetOfChunkGroup = this.position() - 1;
- versionOfChunkGroup = oldVersionInfo.get(startOffsetOfChunkGroup);
- }
- ChunkHeader header = this.readChunkHeader();
- MeasurementSchema measurementSchema = new MeasurementSchema(header.getMeasurementID(),
- header.getDataType(),
- header.getEncodingType(),
- header.getCompressionType());
- measurementSchemaList.add(measurementSchema);
- List<PageHeader> pageHeadersInChunk = new ArrayList<>();
- List<ByteBuffer> dataInChunk = new ArrayList<>();
- List<Boolean> pagePartitionInfo = new ArrayList<>();
- for (int j = 0; j < header.getNumOfPages(); j++) {
- PageHeader pageHeader = readPageHeader(header.getDataType());
- boolean pageInSamePartition = checkIfPageInSameTimePartition(pageHeader);
- pagePartitionInfo.add(pageInSamePartition);
- ByteBuffer pageData = pageInSamePartition ?
- readCompressedPage(pageHeader)
- : readPage(pageHeader, header.getCompressionType());
- pageHeadersInChunk.add(pageHeader);
- dataInChunk.add(pageData);
- }
- pageHeadersInChunkGroup.add(pageHeadersInChunk);
- pageDataInChunkGroup.add(dataInChunk);
- pagePartitionInfoInChunkGroup.add(pagePartitionInfo);
- break;
- case MetaMarker.CHUNK_GROUP_HEADER:
- // this is the footer of a ChunkGroup.
- ChunkGroupHeader chunkGroupHeader = this.readChunkGroupFooter();
- String deviceID = chunkGroupHeader.getDeviceID();
- rewrite(oldTsFile, deviceID, measurementSchemaList, pageHeadersInChunkGroup,
- pageDataInChunkGroup, versionOfChunkGroup, pagePartitionInfoInChunkGroup);
-
- pageHeadersInChunkGroup.clear();
- pageDataInChunkGroup.clear();
- measurementSchemaList.clear();
- pagePartitionInfoInChunkGroup.clear();
- newChunkGroup = true;
- chunkGroupCount++;
- break;
-
- default:
- // the disk file is corrupted, using this file may be dangerous
- logger.error("Unrecognized marker detected, this file may be corrupted");
- return;
- }
- }
- // close upgraded tsFiles and generate resources for them
- for (TsFileIOWriter tsFileIOWriter : partitionWriterMap.values()) {
- upgradedResources.add(endFileAndGenerateResource(tsFileIOWriter));
- }
- } catch (IOException e2) {
- logger.info("TsFile upgrade process cannot proceed at position {} after {} chunk groups "
- + "recovered, because : {}", this.position(), chunkGroupCount, e2.getMessage());
- } finally {
- if (tsFileInput != null) {
- tsFileInput.close();
- }
- }
- }
-
- private boolean checkIfPageInSameTimePartition(PageHeader pageHeader) {
- return StorageEngine.getTimePartition(pageHeader.getStartTime())
- == StorageEngine.getTimePartition(pageHeader.getEndTime());
- }
-
- /**
- * This method is for rewriting the ChunkGroup which data is in the different time partitions. In
- * this case, we have to decode the data to points, and then rewrite the data points to different
- * chunkWriters, finally write chunks to their own upgraded TsFiles
- */
- private void rewrite(File oldTsFile, String deviceId, List<MeasurementSchema> schemas,
- List<List<PageHeader>> pageHeadersInChunkGroup, List<List<ByteBuffer>> dataInChunkGroup,
- long versionOfChunkGroup, List<List<Boolean>> pagePartitionInfoInChunkGroup)
- throws IOException, PageException {
- Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup = new HashMap<>();
- for (int i = 0; i < schemas.size(); i++) {
- MeasurementSchema schema = schemas.get(i);
- List<ByteBuffer> pageDataInChunk = dataInChunkGroup.get(i);
- List<PageHeader> pageHeadersInChunk = pageHeadersInChunkGroup.get(i);
- List<Boolean> pagePartitionInfo = pagePartitionInfoInChunkGroup.get(i);
- valueDecoder = Decoder
- .getDecoderByType(schema.getEncodingType(), schema.getType());
- for (int j = 0; j < pageDataInChunk.size(); j++) {
- if (Boolean.TRUE.equals(pagePartitionInfo.get(j))) {
- writePageInSamePartitionToFile(oldTsFile, schema, pageHeadersInChunk.get(j),
- pageDataInChunk.get(j), chunkWritersInChunkGroup);
- } else {
- writePageInDifferentPartitionsToFiles(oldTsFile, schema, pageDataInChunk.get(j),
- chunkWritersInChunkGroup);
- }
- }
- }
-
- for (Entry<Long, Map<MeasurementSchema, ChunkWriterImpl>> entry : chunkWritersInChunkGroup
- .entrySet()) {
- long partitionId = entry.getKey();
- TsFileIOWriter tsFileIOWriter = partitionWriterMap.get(partitionId);
- tsFileIOWriter.startChunkGroup(deviceId);
- // write chunks to their own upgraded tsFiles
- for (IChunkWriter chunkWriter : entry.getValue().values()) {
- chunkWriter.writeToFileWriter(tsFileIOWriter);
- }
- tsFileIOWriter.endChunkGroup();
- tsFileIOWriter.writeVersion(versionOfChunkGroup);
- }
- }
-
- private TsFileIOWriter getOrDefaultTsFileIOWriter(File oldTsFile, long partition) {
- return partitionWriterMap.computeIfAbsent(partition, k ->
- {
- File partitionDir = FSFactoryProducer.getFSFactory().getFile(oldTsFile.getParent()
- + File.separator + partition);
- if (!partitionDir.exists()) {
- partitionDir.mkdirs();
- }
- File newFile = FSFactoryProducer.getFSFactory().getFile(oldTsFile.getParent()
- + File.separator + partition + File.separator + oldTsFile.getName());
- try {
- if (!newFile.createNewFile()) {
- logger.error("The TsFile {} has been created ", newFile);
- return null;
- }
- return new TsFileIOWriter(newFile);
- } catch (IOException e) {
- logger.error("Create new TsFile {} failed ", newFile);
- return null;
- }
- }
- );
- }
-
- private void writePageInSamePartitionToFile(File oldTsFile, MeasurementSchema schema,
- PageHeader pageHeader,
- ByteBuffer pageData,
- Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup)
- throws PageException {
- long partitionId = StorageEngine.getTimePartition(pageHeader.getStartTime());
- getOrDefaultTsFileIOWriter(oldTsFile, partitionId);
- Map<MeasurementSchema, ChunkWriterImpl> chunkWriters = chunkWritersInChunkGroup
- .getOrDefault(partitionId, new HashMap<>());
- ChunkWriterImpl chunkWriter = chunkWriters
- .getOrDefault(schema, new ChunkWriterImpl(schema));
- chunkWriter.writePageHeaderAndDataIntoBuff(pageData, pageHeader);
- chunkWriters.put(schema, chunkWriter);
- chunkWritersInChunkGroup.put(partitionId, chunkWriters);
- }
-
- private void writePageInDifferentPartitionsToFiles(File oldTsFile, MeasurementSchema schema,
- ByteBuffer pageData,
- Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup)
- throws IOException {
- valueDecoder.reset();
- PageReader pageReader = new PageReader(pageData, schema.getType(), valueDecoder,
- defaultTimeDecoder, null);
- BatchData batchData = pageReader.getAllSatisfiedPageData();
- while (batchData.hasCurrent()) {
- long time = batchData.currentTime();
- Object value = batchData.currentValue();
- long partitionId = StorageEngine.getTimePartition(time);
-
- Map<MeasurementSchema, ChunkWriterImpl> chunkWriters = chunkWritersInChunkGroup
- .getOrDefault(partitionId, new HashMap<>());
- ChunkWriterImpl chunkWriter = chunkWriters
- .getOrDefault(schema, new ChunkWriterImpl(schema));
- getOrDefaultTsFileIOWriter(oldTsFile, partitionId);
- switch (schema.getType()) {
- case INT32:
- chunkWriter.write(time, (int) value);
- break;
- case INT64:
- chunkWriter.write(time, (long) value);
- break;
- case FLOAT:
- chunkWriter.write(time, (float) value);
- break;
- case DOUBLE:
- chunkWriter.write(time, (double) value);
- break;
- case BOOLEAN:
- chunkWriter.write(time, (boolean) value);
- break;
- case TEXT:
- chunkWriter.write(time, (Binary) value);
- break;
- default:
- throw new UnSupportedDataTypeException(
- String.format("Data type %s is not supported.", schema.getType()));
- }
- batchData.next();
- chunkWriters.put(schema, chunkWriter);
- chunkWritersInChunkGroup.put(partitionId, chunkWriters);
- }
- }
-
- /**
- * check if the file to be upgraded has correct magic strings and version number
- */
- private boolean fileCheck(File oldTsFile) throws IOException {
- long fileSize;
- if (!oldTsFile.exists()) {
- logger.error("the file to be updated does not exist, file path: {}", oldTsFile.getPath());
- return false;
- } else {
- fileSize = oldTsFile.length();
- }
-
- String magic = readHeadMagic(true);
- if (!magic.equals(TSFileConfig.MAGIC_STRING)) {
- logger.error("the file's MAGIC STRING is incorrect, file path: {}", oldTsFile.getPath());
- return false;
- }
-
- String versionNumber = readVersionNumber();
- if (!versionNumber.equals(TSFileConfig.VERSION_NUMBER_V1)) {
- logger.error("the file's Version Number is incorrect, file path: {}", oldTsFile.getPath());
- return false;
- }
-
- if (fileSize == TSFileConfig.MAGIC_STRING.length()) {
- logger.error("the file only contains magic string, file path: {}", oldTsFile.getPath());
- return false;
- } else if (!readTailMagic().equals(TSFileConfig.MAGIC_STRING)) {
- logger.error("the file cannot upgrade, file path: {}", oldTsFile.getPath());
- return false;
- }
- return true;
- }
-
- private Map<Long, Long> getVersionInfo() throws IOException {
- Map<Long, Long> versionInfo = new HashMap<>();
- TsFileMetadataV1 fileMetadata = readFileMetadata();
- List<TsDeviceMetadataV1> oldDeviceMetadataList = new ArrayList<>();
- for (TsDeviceMetadataIndexV1 index : fileMetadata.getDeviceMap().values()) {
- TsDeviceMetadataV1 oldDeviceMetadata = readTsDeviceMetaData(index);
- oldDeviceMetadataList.add(oldDeviceMetadata);
- }
-
- for (TsDeviceMetadataV1 oldTsDeviceMetadata : oldDeviceMetadataList) {
- for (ChunkGroupMetaDataV1 oldChunkGroupMetadata : oldTsDeviceMetadata
- .getChunkGroupMetaDataList()) {
- long version = oldChunkGroupMetadata.getVersion();
- long offsetOfChunkGroup = oldChunkGroupMetadata.getStartOffsetOfChunkGroup();
- // get version informations
- versionInfo.put(offsetOfChunkGroup, version);
- }
- }
- return versionInfo;
- }
-
- private TsFileResource endFileAndGenerateResource(TsFileIOWriter tsFileIOWriter)
- throws IOException {
- tsFileIOWriter.endFile();
- TsFileResource tsFileResource = new TsFileResource(tsFileIOWriter.getFile());
- Map<String, List<TimeseriesMetadata>> deviceTimeseriesMetadataMap = tsFileIOWriter
- .getDeviceTimeseriesMetadataMap();
- for (Map.Entry<String, List<TimeseriesMetadata>> entry : deviceTimeseriesMetadataMap
- .entrySet()) {
- String device = entry.getKey();
- for (TimeseriesMetadata timeseriesMetaData : entry.getValue()) {
- tsFileResource.updateStartTime(device, timeseriesMetaData.getStatistics().getStartTime());
- tsFileResource.updateEndTime(device, timeseriesMetaData.getStatistics().getEndTime());
- }
- }
- tsFileResource.setClosed(true);
- return tsFileResource;
- }
-
-}
\ No newline at end of file
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// */
+//package org.apache.iotdb.db.tools.upgrade;
+//
+//import java.io.File;
+//import java.io.IOException;
+//import java.nio.ByteBuffer;
+//import java.util.ArrayList;
+//import java.util.HashMap;
+//import java.util.List;
+//import java.util.Map;
+//import java.util.Map.Entry;
+//import org.apache.iotdb.db.engine.StorageEngine;
+//import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+//import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+//import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+//import org.apache.iotdb.tsfile.compress.IUnCompressor;
+//import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+//import org.apache.iotdb.tsfile.exception.write.PageException;
+//import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
+//import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+//import org.apache.iotdb.tsfile.file.MetaMarker;
+//import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
+//import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+//import org.apache.iotdb.tsfile.file.header.PageHeader;
+//import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
+//import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+//import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+//import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+//import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+//import org.apache.iotdb.tsfile.read.common.BatchData;
+//import org.apache.iotdb.tsfile.read.reader.TsFileInput;
+//import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+//import org.apache.iotdb.tsfile.utils.Binary;
+//import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+//import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
+//import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
+//import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+//import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
+//import org.slf4j.Logger;
+//import org.slf4j.LoggerFactory;
+//
+//public class TsFileOnlineUpgradeTool implements AutoCloseable {
+//
+// private static final Logger logger = LoggerFactory.getLogger(TsFileOnlineUpgradeTool.class);
+//
+// private TsFileInput tsFileInput;
+// private long fileMetadataPos;
+// private int fileMetadataSize;
+// private ByteBuffer markerBuffer = ByteBuffer.allocate(Byte.BYTES);
+// private Decoder defaultTimeDecoder = Decoder.getDecoderByType(
+// TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()),
+// TSDataType.INT64);
+// private Decoder valueDecoder;
+// protected String file;
+//
+// // PartitionId -> TsFileIOWriter
+// private Map<Long, TsFileIOWriter> partitionWriterMap;
+//
+// /**
+// * Create a file reader of the given file. The reader will read the tail of the file to get the
+// * file metadata size.Then the reader will skip the first TSFileConfig.OLD_MAGIC_STRING.length()
+// * bytes of the file for preparing reading real data.
+// *
+// * @param file the data file
+// * @throws IOException If some I/O error occurs
+// */
+// public TsFileOnlineUpgradeTool(String file) throws IOException {
+// this(file, true);
+// }
+//
+// /**
+// * construct function for TsfileOnlineUpgradeTool.
+// *
+// * @param file -given file name
+// * @param loadMetadataSize -load meta data size
+// */
+// public TsFileOnlineUpgradeTool(String file, boolean loadMetadataSize) throws IOException {
+// this.file = file;
+// tsFileInput = FSFactoryProducer.getFileInputFactory().getTsFileInput(file);
+// partitionWriterMap = new HashMap<>();
+// try {
+// if (loadMetadataSize) {
+// loadMetadataSize();
+// }
+// } catch (Exception e) {
+// tsFileInput.close();
+// throw e;
+// }
+// }
+//
+// /**
+// * upgrade a single tsfile
+// *
+// * @param tsFileName old version tsFile's absolute path
+// * @param upgradedResources new version tsFiles' resources
+// */
+// public static void upgradeOneTsfile(String tsFileName, List<TsFileResource> upgradedResources)
+// throws IOException, WriteProcessException {
+// try (TsFileOnlineUpgradeTool updater = new TsFileOnlineUpgradeTool(tsFileName)) {
+// updater.upgradeFile(upgradedResources);
+// }
+// }
+//
+// /**
+// *
+// */
+// public void loadMetadataSize() throws IOException {
+// ByteBuffer metadataSize = ByteBuffer.allocate(Integer.BYTES);
+// tsFileInput.read(metadataSize,
+// tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES);
+// metadataSize.flip();
+// // read file metadata size and position
+// fileMetadataSize = ReadWriteIOUtils.readInt(metadataSize);
+// fileMetadataPos =
+// tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES
+// - fileMetadataSize;
+// // skip the magic header
+// position(TSFileConfig.MAGIC_STRING.length());
+// }
+//
+// public String readTailMagic() throws IOException {
+// long totalSize = tsFileInput.size();
+//
+// ByteBuffer magicStringBytes = ByteBuffer.allocate(TSFileConfig.MAGIC_STRING.length());
+// tsFileInput.read(magicStringBytes, totalSize - TSFileConfig.MAGIC_STRING.length());
+// magicStringBytes.flip();
+// return new String(magicStringBytes.array());
+// }
+//
+// /**
+// * whether the file is a complete TsFile: only if the head magic and tail magic string exists.
+// */
+// public boolean isComplete() throws IOException {
+// return tsFileInput.size() >= TSFileConfig.MAGIC_STRING.length() * 2 && readTailMagic()
+// .equals(readHeadMagic());
+// }
+//
+// /**
+// * this function does not modify the position of the file reader.
+// */
+// public String readHeadMagic() throws IOException {
+// return readHeadMagic(false);
+// }
+//
+// /**
+// * @param movePosition whether move the position of the file reader after reading the magic header
+// * to the end of the magic head string.
+// */
+// public String readHeadMagic(boolean movePosition) throws IOException {
+// ByteBuffer magicStringBytes = ByteBuffer.allocate(TSFileConfig.MAGIC_STRING.length());
+// if (movePosition) {
+// tsFileInput.position(0);
+// tsFileInput.read(magicStringBytes);
+// } else {
+// tsFileInput.read(magicStringBytes, 0);
+// }
+// magicStringBytes.flip();
+// return new String(magicStringBytes.array());
+// }
+//
+// /**
+// * this function reads version number and checks compatibility of TsFile.
+// */
+// public String readVersionNumber() throws IOException {
+// ByteBuffer versionNumberBytes = ByteBuffer
+// .allocate(TSFileConfig.VERSION_NUMBER_V2.getBytes().length);
+// tsFileInput.position(TSFileConfig.MAGIC_STRING.getBytes().length);
+// tsFileInput.read(versionNumberBytes);
+// versionNumberBytes.flip();
+// return new String(versionNumberBytes.array());
+// }
+//
+// /**
+// * this function does not modify the position of the file reader.
+// */
+// public TsFileMetadataV1 readFileMetadata() throws IOException {
+// return TsFileMetadataV1.deserializeFrom(readData(fileMetadataPos, fileMetadataSize));
+// }
+//
+// /**
+// * this function does not modify the position of the file reader.
+// */
+// public TsDeviceMetadataV1 readTsDeviceMetaData(TsDeviceMetadataIndexV1 index) throws IOException {
+// return TsDeviceMetadataV1.deserializeFrom(readData(index.getOffset(), index.getLen()));
+// }
+//
+// /**
+// * read data from current position of the input, and deserialize it to a CHUNK_GROUP_FOOTER. <br>
+// * This method is not threadsafe.
+// *
+// * @return a CHUNK_GROUP_FOOTER
+// * @throws IOException io error
+// */
+// public ChunkGroupHeader readChunkGroupFooter() throws IOException {
+// return ChunkGroupHeader.deserializeFrom(tsFileInput.wrapAsInputStream(), true);
+// }
+//
+// /**
+// * read data from current position of the input, and deserialize it to a CHUNK_HEADER. <br> This
+// * method is not threadsafe.
+// *
+// * @return a CHUNK_HEADER
+// * @throws IOException io error
+// */
+// public ChunkHeader readChunkHeader() throws IOException {
+// return HeaderUtils.deserializeChunkHeaderV1(tsFileInput.wrapAsInputStream(), true);
+// }
+//
+// /**
+// * not thread safe.
+// *
+// * @param type given tsfile data type
+// */
+// public PageHeader readPageHeader(TSDataType type) throws IOException {
+// return HeaderUtils.deserializePageHeaderV1(tsFileInput.wrapAsInputStream(), type);
+// }
+//
+// public ByteBuffer readPage(PageHeader header, CompressionType type)
+// throws IOException {
+// ByteBuffer buffer = readData(-1, header.getCompressedSize());
+// IUnCompressor unCompressor = IUnCompressor.getUnCompressor(type);
+// ByteBuffer uncompressedBuffer = ByteBuffer.allocate(header.getUncompressedSize());
+// if (type == CompressionType.UNCOMPRESSED) {
+// return buffer;
+// }
+// unCompressor.uncompress(buffer.array(), buffer.position(), buffer.remaining(),
+// uncompressedBuffer.array(),
+// 0);
+// return uncompressedBuffer;
+// }
+//
+// public ByteBuffer readCompressedPage(PageHeader header) throws IOException {
+// return readData(-1, header.getCompressedSize());
+// }
+//
+// public long position() throws IOException {
+// return tsFileInput.position();
+// }
+//
+// public void position(long offset) throws IOException {
+// tsFileInput.position(offset);
+// }
+//
+// /**
+// * read one byte from the input. <br> this method is not thread safe
+// */
+// public byte readMarker() throws IOException {
+// markerBuffer.clear();
+// if (ReadWriteIOUtils.readAsPossible(tsFileInput, markerBuffer) == 0) {
+// throw new IOException("reach the end of the file.");
+// }
+// markerBuffer.flip();
+// return markerBuffer.get();
+// }
+//
+// public byte readMarker(long position) throws IOException {
+// return readData(position, Byte.BYTES).get();
+// }
+//
+// public void close() throws IOException {
+// this.tsFileInput.close();
+// }
+//
+// public String getFileName() {
+// return this.file;
+// }
+//
+// /**
+// * read data from tsFileInput, from the current position (if position = -1), or the given
+// * position. <br> if position = -1, the tsFileInput's position will be changed to the current
+// * position + real data size that been read. Other wise, the tsFileInput's position is not
+// * changed.
+// *
+// * @param position the start position of data in the tsFileInput, or the current position if
+// * position = -1
+// * @param size the size of data that want to read
+// * @return data that been read.
+// */
+// private ByteBuffer readData(long position, int size) throws IOException {
+// ByteBuffer buffer = ByteBuffer.allocate(size);
+// if (position == -1) {
+// if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != size) {
+// throw new IOException("reach the end of the data");
+// }
+// } else {
+// if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, size) != size) {
+// throw new IOException("reach the end of the data");
+// }
+// }
+// buffer.flip();
+// return buffer;
+// }
+//
+// /**
+// * upgrade file and resource
+// *
+// * @throws IOException, WriteProcessException
+// */
+// @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
+// public void upgradeFile(List<TsFileResource> upgradedResources)
+// throws IOException, WriteProcessException {
+// File oldTsFile = FSFactoryProducer.getFSFactory().getFile(this.file);
+//
+// // check if the old TsFile has correct header
+// if (!fileCheck(oldTsFile)) {
+// return;
+// }
+//
+// // ChunkGroupOffset -> version
+// Map<Long, Long> oldVersionInfo = getVersionInfo();
+//
+// // start to scan chunks and chunkGroups
+// long startOffsetOfChunkGroup = 0;
+// boolean newChunkGroup = true;
+// long versionOfChunkGroup = 0;
+// int chunkGroupCount = 0;
+// List<List<PageHeader>> pageHeadersInChunkGroup = new ArrayList<>();
+// List<List<ByteBuffer>> pageDataInChunkGroup = new ArrayList<>();
+// List<List<Boolean>> pagePartitionInfoInChunkGroup = new ArrayList<>();
+// byte marker;
+// List<MeasurementSchema> measurementSchemaList = new ArrayList<>();
+// try {
+// while ((marker = this.readMarker()) != MetaMarker.SEPARATOR) {
+// switch (marker) {
+// case MetaMarker.CHUNK_HEADER:
+// // this is the first chunk of a new ChunkGroup.
+// if (newChunkGroup) {
+// newChunkGroup = false;
+// startOffsetOfChunkGroup = this.position() - 1;
+// versionOfChunkGroup = oldVersionInfo.get(startOffsetOfChunkGroup);
+// }
+// ChunkHeader header = this.readChunkHeader();
+// MeasurementSchema measurementSchema = new MeasurementSchema(header.getMeasurementID(),
+// header.getDataType(),
+// header.getEncodingType(),
+// header.getCompressionType());
+// measurementSchemaList.add(measurementSchema);
+// List<PageHeader> pageHeadersInChunk = new ArrayList<>();
+// List<ByteBuffer> dataInChunk = new ArrayList<>();
+// List<Boolean> pagePartitionInfo = new ArrayList<>();
+// for (int j = 0; j < header.getNumOfPages(); j++) {
+// PageHeader pageHeader = readPageHeader(header.getDataType());
+// boolean pageInSamePartition = checkIfPageInSameTimePartition(pageHeader);
+// pagePartitionInfo.add(pageInSamePartition);
+// ByteBuffer pageData = pageInSamePartition ?
+// readCompressedPage(pageHeader)
+// : readPage(pageHeader, header.getCompressionType());
+// pageHeadersInChunk.add(pageHeader);
+// dataInChunk.add(pageData);
+// }
+// pageHeadersInChunkGroup.add(pageHeadersInChunk);
+// pageDataInChunkGroup.add(dataInChunk);
+// pagePartitionInfoInChunkGroup.add(pagePartitionInfo);
+// break;
+// case MetaMarker.CHUNK_GROUP_HEADER:
+// // this is the footer of a ChunkGroup.
+// ChunkGroupHeader chunkGroupHeader = this.readChunkGroupFooter();
+// String deviceID = chunkGroupHeader.getDeviceID();
+// rewrite(oldTsFile, deviceID, measurementSchemaList, pageHeadersInChunkGroup,
+// pageDataInChunkGroup, versionOfChunkGroup, pagePartitionInfoInChunkGroup);
+//
+// pageHeadersInChunkGroup.clear();
+// pageDataInChunkGroup.clear();
+// measurementSchemaList.clear();
+// pagePartitionInfoInChunkGroup.clear();
+// newChunkGroup = true;
+// chunkGroupCount++;
+// break;
+//
+// default:
+// // the disk file is corrupted, using this file may be dangerous
+// logger.error("Unrecognized marker detected, this file may be corrupted");
+// return;
+// }
+// }
+// // close upgraded tsFiles and generate resources for them
+// for (TsFileIOWriter tsFileIOWriter : partitionWriterMap.values()) {
+// upgradedResources.add(endFileAndGenerateResource(tsFileIOWriter));
+// }
+// } catch (IOException e2) {
+// logger.info("TsFile upgrade process cannot proceed at position {} after {} chunk groups "
+// + "recovered, because : {}", this.position(), chunkGroupCount, e2.getMessage());
+// } finally {
+// if (tsFileInput != null) {
+// tsFileInput.close();
+// }
+// }
+// }
+//
+// private boolean checkIfPageInSameTimePartition(PageHeader pageHeader) {
+// return StorageEngine.getTimePartition(pageHeader.getStartTime())
+// == StorageEngine.getTimePartition(pageHeader.getEndTime());
+// }
+//
+// /**
+// * This method is for rewriting the ChunkGroup which data is in the different time partitions. In
+// * this case, we have to decode the data to points, and then rewrite the data points to different
+// * chunkWriters, finally write chunks to their own upgraded TsFiles
+// */
+// private void rewrite(File oldTsFile, String deviceId, List<MeasurementSchema> schemas,
+// List<List<PageHeader>> pageHeadersInChunkGroup, List<List<ByteBuffer>> dataInChunkGroup,
+// long versionOfChunkGroup, List<List<Boolean>> pagePartitionInfoInChunkGroup)
+// throws IOException, PageException {
+// Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup = new HashMap<>();
+// for (int i = 0; i < schemas.size(); i++) {
+// MeasurementSchema schema = schemas.get(i);
+// List<ByteBuffer> pageDataInChunk = dataInChunkGroup.get(i);
+// List<PageHeader> pageHeadersInChunk = pageHeadersInChunkGroup.get(i);
+// List<Boolean> pagePartitionInfo = pagePartitionInfoInChunkGroup.get(i);
+// valueDecoder = Decoder
+// .getDecoderByType(schema.getEncodingType(), schema.getType());
+// for (int j = 0; j < pageDataInChunk.size(); j++) {
+// if (Boolean.TRUE.equals(pagePartitionInfo.get(j))) {
+// writePageInSamePartitionToFile(oldTsFile, schema, pageHeadersInChunk.get(j),
+// pageDataInChunk.get(j), chunkWritersInChunkGroup);
+// } else {
+// writePageInDifferentPartitionsToFiles(oldTsFile, schema, pageDataInChunk.get(j),
+// chunkWritersInChunkGroup);
+// }
+// }
+// }
+//
+// for (Entry<Long, Map<MeasurementSchema, ChunkWriterImpl>> entry : chunkWritersInChunkGroup
+// .entrySet()) {
+// long partitionId = entry.getKey();
+// TsFileIOWriter tsFileIOWriter = partitionWriterMap.get(partitionId);
+// tsFileIOWriter.startChunkGroup(deviceId);
+// // write chunks to their own upgraded tsFiles
+// for (IChunkWriter chunkWriter : entry.getValue().values()) {
+// chunkWriter.writeToFileWriter(tsFileIOWriter);
+// }
+// tsFileIOWriter.endChunkGroup();
+// tsFileIOWriter.writeVersion(versionOfChunkGroup);
+// }
+// }
+//
+// private TsFileIOWriter getOrDefaultTsFileIOWriter(File oldTsFile, long partition) {
+// return partitionWriterMap.computeIfAbsent(partition, k ->
+// {
+// File partitionDir = FSFactoryProducer.getFSFactory().getFile(oldTsFile.getParent()
+// + File.separator + partition);
+// if (!partitionDir.exists()) {
+// partitionDir.mkdirs();
+// }
+// File newFile = FSFactoryProducer.getFSFactory().getFile(oldTsFile.getParent()
+// + File.separator + partition + File.separator + oldTsFile.getName());
+// try {
+// if (!newFile.createNewFile()) {
+// logger.error("The TsFile {} has been created ", newFile);
+// return null;
+// }
+// return new TsFileIOWriter(newFile);
+// } catch (IOException e) {
+// logger.error("Create new TsFile {} failed ", newFile);
+// return null;
+// }
+// }
+// );
+// }
+//
+// private void writePageInSamePartitionToFile(File oldTsFile, MeasurementSchema schema,
+// PageHeader pageHeader,
+// ByteBuffer pageData,
+// Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup)
+// throws PageException {
+// long partitionId = StorageEngine.getTimePartition(pageHeader.getStartTime());
+// getOrDefaultTsFileIOWriter(oldTsFile, partitionId);
+// Map<MeasurementSchema, ChunkWriterImpl> chunkWriters = chunkWritersInChunkGroup
+// .getOrDefault(partitionId, new HashMap<>());
+// ChunkWriterImpl chunkWriter = chunkWriters
+// .getOrDefault(schema, new ChunkWriterImpl(schema));
+// chunkWriter.writePageHeaderAndDataIntoBuff(pageData, pageHeader);
+// chunkWriters.put(schema, chunkWriter);
+// chunkWritersInChunkGroup.put(partitionId, chunkWriters);
+// }
+//
+// private void writePageInDifferentPartitionsToFiles(File oldTsFile, MeasurementSchema schema,
+// ByteBuffer pageData,
+// Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup)
+// throws IOException {
+// valueDecoder.reset();
+// PageReader pageReader = new PageReader(pageData, schema.getType(), valueDecoder,
+// defaultTimeDecoder, null);
+// BatchData batchData = pageReader.getAllSatisfiedPageData();
+// while (batchData.hasCurrent()) {
+// long time = batchData.currentTime();
+// Object value = batchData.currentValue();
+// long partitionId = StorageEngine.getTimePartition(time);
+//
+// Map<MeasurementSchema, ChunkWriterImpl> chunkWriters = chunkWritersInChunkGroup
+// .getOrDefault(partitionId, new HashMap<>());
+// ChunkWriterImpl chunkWriter = chunkWriters
+// .getOrDefault(schema, new ChunkWriterImpl(schema));
+// getOrDefaultTsFileIOWriter(oldTsFile, partitionId);
+// switch (schema.getType()) {
+// case INT32:
+// chunkWriter.write(time, (int) value);
+// break;
+// case INT64:
+// chunkWriter.write(time, (long) value);
+// break;
+// case FLOAT:
+// chunkWriter.write(time, (float) value);
+// break;
+// case DOUBLE:
+// chunkWriter.write(time, (double) value);
+// break;
+// case BOOLEAN:
+// chunkWriter.write(time, (boolean) value);
+// break;
+// case TEXT:
+// chunkWriter.write(time, (Binary) value);
+// break;
+// default:
+// throw new UnSupportedDataTypeException(
+// String.format("Data type %s is not supported.", schema.getType()));
+// }
+// batchData.next();
+// chunkWriters.put(schema, chunkWriter);
+// chunkWritersInChunkGroup.put(partitionId, chunkWriters);
+// }
+// }
+//
+// /**
+// * check if the file to be upgraded has correct magic strings and version number
+// */
+// private boolean fileCheck(File oldTsFile) throws IOException {
+// long fileSize;
+// if (!oldTsFile.exists()) {
+// logger.error("the file to be updated does not exist, file path: {}", oldTsFile.getPath());
+// return false;
+// } else {
+// fileSize = oldTsFile.length();
+// }
+//
+// String magic = readHeadMagic(true);
+// if (!magic.equals(TSFileConfig.MAGIC_STRING)) {
+// logger.error("the file's MAGIC STRING is incorrect, file path: {}", oldTsFile.getPath());
+// return false;
+// }
+//
+// String versionNumber = readVersionNumber();
+// if (!versionNumber.equals(TSFileConfig.VERSION_NUMBER_V1)) {
+// logger.error("the file's Version Number is incorrect, file path: {}", oldTsFile.getPath());
+// return false;
+// }
+//
+// if (fileSize == TSFileConfig.MAGIC_STRING.length()) {
+// logger.error("the file only contains magic string, file path: {}", oldTsFile.getPath());
+// return false;
+// } else if (!readTailMagic().equals(TSFileConfig.MAGIC_STRING)) {
+// logger.error("the file cannot upgrade, file path: {}", oldTsFile.getPath());
+// return false;
+// }
+// return true;
+// }
+//
+// private Map<Long, Long> getVersionInfo() throws IOException {
+// Map<Long, Long> versionInfo = new HashMap<>();
+// TsFileMetadataV1 fileMetadata = readFileMetadata();
+// List<TsDeviceMetadataV1> oldDeviceMetadataList = new ArrayList<>();
+// for (TsDeviceMetadataIndexV1 index : fileMetadata.getDeviceMap().values()) {
+// TsDeviceMetadataV1 oldDeviceMetadata = readTsDeviceMetaData(index);
+// oldDeviceMetadataList.add(oldDeviceMetadata);
+// }
+//
+// for (TsDeviceMetadataV1 oldTsDeviceMetadata : oldDeviceMetadataList) {
+// for (ChunkGroupMetaDataV1 oldChunkGroupMetadata : oldTsDeviceMetadata
+// .getChunkGroupMetaDataList()) {
+// long version = oldChunkGroupMetadata.getVersion();
+// long offsetOfChunkGroup = oldChunkGroupMetadata.getStartOffsetOfChunkGroup();
+// // get version informations
+// versionInfo.put(offsetOfChunkGroup, version);
+// }
+// }
+// return versionInfo;
+// }
+//
+// private TsFileResource endFileAndGenerateResource(TsFileIOWriter tsFileIOWriter)
+// throws IOException {
+// tsFileIOWriter.endFile();
+// TsFileResource tsFileResource = new TsFileResource(tsFileIOWriter.getFile());
+// Map<String, List<TimeseriesMetadata>> deviceTimeseriesMetadataMap = tsFileIOWriter
+// .getDeviceTimeseriesMetadataMap();
+// for (Map.Entry<String, List<TimeseriesMetadata>> entry : deviceTimeseriesMetadataMap
+// .entrySet()) {
+// String device = entry.getKey();
+// for (TimeseriesMetadata timeseriesMetaData : entry.getValue()) {
+// tsFileResource.updateStartTime(device, timeseriesMetaData.getStatistics().getStartTime());
+// tsFileResource.updateEndTime(device, timeseriesMetaData.getStatistics().getEndTime());
+// }
+// }
+// tsFileResource.setClosed(true);
+// return tsFileResource;
+// }
+//
+//}
\ No newline at end of file
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EndianType.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EndianType.java
deleted file mode 100644
index 1506d27..0000000
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EndianType.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.tsfile.encoding.common;
-
-/**
- * In current version, we only support BIG_ENDIAN mode.
- */
-public enum EndianType {
- BIG_ENDIAN, LITTLE_ENDIAN
-}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
index 6bffedd..0e43cf6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
@@ -22,8 +22,6 @@ package org.apache.iotdb.tsfile.encoding.decoder;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
-
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
@@ -50,14 +48,14 @@ public abstract class Decoder {
public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) {
switch (encoding) {
case PLAIN:
- return new PlainDecoder(EndianType.BIG_ENDIAN);
+ return new PlainDecoder();
case RLE:
switch (dataType) {
case BOOLEAN:
case INT32:
- return new IntRleDecoder(EndianType.BIG_ENDIAN);
+ return new IntRleDecoder();
case INT64:
- return new LongRleDecoder(EndianType.BIG_ENDIAN);
+ return new LongRleDecoder();
case FLOAT:
case DOUBLE:
return new FloatDecoder(TSEncoding.valueOf(encoding.toString()), dataType);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java
index eaaf729..ce0bc2a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java
@@ -21,17 +21,14 @@ package org.apache.iotdb.tsfile.encoding.decoder;
import java.io.IOException;
import java.nio.ByteBuffer;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.encoding.encoder.FloatEncoder;
import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Decoder for float or double value using rle or two diff. For more info about
@@ -58,10 +55,10 @@ public class FloatDecoder extends Decoder {
super(encodingType);
if (encodingType == TSEncoding.RLE) {
if (dataType == TSDataType.FLOAT) {
- decoder = new IntRleDecoder(EndianType.BIG_ENDIAN);
+ decoder = new IntRleDecoder();
logger.debug("tsfile-encoding FloatDecoder: init decoder using int-rle and float");
} else if (dataType == TSDataType.DOUBLE) {
- decoder = new LongRleDecoder(EndianType.BIG_ENDIAN);
+ decoder = new LongRleDecoder();
logger.debug("tsfile-encoding FloatDecoder: init decoder using long-rle and double");
} else {
throw new TsFileDecodingException(String.format("data type %s is not supported by FloatDecoder", dataType));
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java
index a41512f..0166446 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java
@@ -21,15 +21,12 @@ package org.apache.iotdb.tsfile.encoding.decoder;
import java.io.IOException;
import java.nio.ByteBuffer;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.encoding.bitpacking.IntPacker;
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Decoder for int value using rle or bit-packing.
@@ -53,8 +50,8 @@ public class IntRleDecoder extends RleDecoder {
*/
private IntPacker packer;
- public IntRleDecoder(EndianType endianType) {
- super(endianType);
+ public IntRleDecoder() {
+ super();
currentValue = 0;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoder.java
index 3f8da99..de27c0c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoder.java
@@ -21,15 +21,12 @@ package org.apache.iotdb.tsfile.encoding.decoder;
import java.io.IOException;
import java.nio.ByteBuffer;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.encoding.bitpacking.LongPacker;
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Decoder for long value using rle or bit-packing.
@@ -53,8 +50,8 @@ public class LongRleDecoder extends RleDecoder {
*/
private LongPacker packer;
- public LongRleDecoder(EndianType endianType) {
- super(endianType);
+ public LongRleDecoder() {
+ super();
currentValue = 0;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java
index 1fd15c1..8c31cff 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java
@@ -22,31 +22,16 @@ package org.apache.iotdb.tsfile.encoding.decoder;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.Binary;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
public class PlainDecoder extends Decoder {
- private static final Logger logger = LoggerFactory.getLogger(PlainDecoder.class);
- private EndianType endianType;
-
- public EndianType getEndianType() {
- return endianType;
- }
-
- public void setEndianType(EndianType endianType) {
- this.endianType = endianType;
- }
- public PlainDecoder(EndianType endianType) {
+ public PlainDecoder() {
super(TSEncoding.PLAIN);
- this.endianType = endianType;
}
@Override
@@ -61,7 +46,7 @@ public class PlainDecoder extends Decoder {
@Override
public int readInt(ByteBuffer buffer) {
- return buffer.getInt();
+ return ReadWriteForEncodingUtils.readVarInt(buffer);
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RleDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RleDecoder.java
index 8606537..bb5e703 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RleDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RleDecoder.java
@@ -22,10 +22,8 @@ package org.apache.iotdb.tsfile.encoding.decoder;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
-
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.Binary;
@@ -40,16 +38,6 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
*/
public abstract class RleDecoder extends Decoder {
- private EndianType endianType;
-
- public EndianType getEndianType() {
- return endianType;
- }
-
- public void setEndianType(EndianType endianType) {
- this.endianType = endianType;
- }
-
protected TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
/**
* mode to indicate current encoding type 0 - RLE 1 - BIT_PACKED.
@@ -87,9 +75,8 @@ public abstract class RleDecoder extends Decoder {
* a constructor, init with endianType, default encoding is
* <code>TSEncoding.RLE</code>.
*/
- public RleDecoder(EndianType endianType) {
+ public RleDecoder() {
super(TSEncoding.RLE);
- this.endianType = endianType;
reset();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
index 873a4ba..1b4b558 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java
@@ -21,8 +21,6 @@ package org.apache.iotdb.tsfile.encoding.encoder;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.exception.encoding.TsFileEncodingException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
@@ -66,9 +64,9 @@ public class FloatEncoder extends Encoder {
isMaxPointNumberSaved = false;
if (encodingType == TSEncoding.RLE) {
if (dataType == TSDataType.FLOAT) {
- encoder = new IntRleEncoder(EndianType.BIG_ENDIAN);
+ encoder = new IntRleEncoder();
} else if (dataType == TSDataType.DOUBLE) {
- encoder = new LongRleEncoder(EndianType.BIG_ENDIAN);
+ encoder = new LongRleEncoder();
} else {
throw new TsFileEncodingException(String.format("data type %s is not supported by FloatEncoder", dataType));
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRleEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRleEncoder.java
index f3bb89a..9ed819e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRleEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRleEncoder.java
@@ -22,10 +22,8 @@ package org.apache.iotdb.tsfile.encoding.encoder;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
-
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.encoding.bitpacking.IntPacker;
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
/**
@@ -38,11 +36,11 @@ public class IntRleEncoder extends RleEncoder<Integer> {
*/
private IntPacker packer;
- public IntRleEncoder(EndianType endianType) {
- super(endianType);
+ public IntRleEncoder() {
+ super();
bufferedValues = new Integer[TSFileConfig.RLE_MIN_REPEATED_NUM];
preValue = 0;
- values = new ArrayList<Integer>();
+ values = new ArrayList<>();
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongRleEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongRleEncoder.java
index dbbbfbb..f8a33f3 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongRleEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongRleEncoder.java
@@ -22,10 +22,8 @@ package org.apache.iotdb.tsfile.encoding.encoder;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
-
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.encoding.bitpacking.LongPacker;
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
/**
@@ -41,8 +39,8 @@ public class LongRleEncoder extends RleEncoder<Long> {
/**
* Constructor of LongRleEncoder.
*/
- public LongRleEncoder(EndianType endianType) {
- super(endianType);
+ public LongRleEncoder() {
+ super();
bufferedValues = new Long[TSFileConfig.RLE_MIN_REPEATED_NUM];
preValue = (long) 0;
values = new ArrayList<Long>();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java
index d30bcc8..50bff74 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java
@@ -22,40 +22,27 @@ package org.apache.iotdb.tsfile.encoding.encoder;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
-
-import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.exception.encoding.TsFileEncodingException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.Binary;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class PlainEncoder extends Encoder {
private static final Logger logger = LoggerFactory.getLogger(PlainEncoder.class);
- private EndianType endianType;
private TSDataType dataType;
private int maxStringLength;
- public PlainEncoder(EndianType endianType, TSDataType dataType, int maxStringLength) {
+ public PlainEncoder(TSDataType dataType, int maxStringLength) {
super(TSEncoding.PLAIN);
- this.endianType = endianType;
this.dataType = dataType;
this.maxStringLength = maxStringLength;
}
- public void setEndianType(EndianType endianType) {
- this.endianType = endianType;
- }
-
- public EndianType getEndianType() {
- return endianType;
- }
-
@Override
public void encode(boolean value, ByteArrayOutputStream out) {
if (value) {
@@ -67,40 +54,19 @@ public class PlainEncoder extends Encoder {
@Override
public void encode(short value, ByteArrayOutputStream out) {
- if (this.endianType == EndianType.LITTLE_ENDIAN) {
- out.write(value & 0xFF);
- out.write((value >> 8) & 0xFF);
- } else if (this.endianType == EndianType.BIG_ENDIAN) {
- out.write((value >> 8) & 0xFF);
- out.write(value & 0xFF);
- }
+ out.write((value >> 8) & 0xFF);
+ out.write(value & 0xFF);
}
@Override
public void encode(int value, ByteArrayOutputStream out) {
- if (this.endianType == EndianType.LITTLE_ENDIAN) {
- out.write(value & 0xFF);
- out.write((value >> 8) & 0xFF);
- out.write((value >> 16) & 0xFF);
- out.write((value >> 24) & 0xFF);
- } else if (this.endianType == EndianType.BIG_ENDIAN) {
- out.write((value >> 24) & 0xFF);
- out.write((value >> 16) & 0xFF);
- out.write((value >> 8) & 0xFF);
- out.write(value & 0xFF);
- }
+ ReadWriteForEncodingUtils.writeVarInt(value, out);
}
@Override
public void encode(long value, ByteArrayOutputStream out) {
- if (this.endianType == EndianType.LITTLE_ENDIAN) {
- for (int i = 0; i < 8; i++) {
- out.write((byte) (((value) >> (i * 8)) & 0xFF));
- }
- } else if (this.endianType == EndianType.BIG_ENDIAN) {
- for (int i = 7; i >= 0; i--) {
- out.write((byte) (((value) >> (i * 8)) & 0xFF));
- }
+ for (int i = 7; i >= 0; i--) {
+ out.write((byte) (((value) >> (i * 8)) & 0xFF));
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java
index 51003ce..51ad831 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java
@@ -24,17 +24,14 @@ import java.io.IOException;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.exception.encoding.TsFileEncodingException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Encodes values using a combination of run length encoding and bit packing,
@@ -66,16 +63,6 @@ public abstract class RleEncoder<T extends Comparable<T>> extends Encoder {
private static final Logger logger = LoggerFactory.getLogger(RleEncoder.class);
- private EndianType endianType;
-
- public EndianType getEndianType() {
- return endianType;
- }
-
- public void setEndianType(EndianType endianType) {
- this.endianType = endianType;
- }
-
/**
* we save all value in a list and calculate its bitwidth.
*/
@@ -134,9 +121,8 @@ public abstract class RleEncoder<T extends Comparable<T>> extends Encoder {
/**
* constructor.
*/
- public RleEncoder(EndianType endianType) {
+ public RleEncoder() {
super(TSEncoding.RLE);
- this.endianType = endianType;
bytesBuffer = new ArrayList<>();
isBitPackRun = false;
isBitWidthSaved = false;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
index 0f6cd7c..8fd3cfa 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
@@ -20,17 +20,14 @@
package org.apache.iotdb.tsfile.encoding.encoder;
import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant;
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Each subclass of TSEncodingBuilder responds a enumerate value in {@linkplain TSEncoding
@@ -105,7 +102,7 @@ public abstract class TSEncodingBuilder {
@Override
public Encoder getEncoder(TSDataType type) {
- return new PlainEncoder(EndianType.BIG_ENDIAN, type, maxStringLength);
+ return new PlainEncoder(type, maxStringLength);
}
@Override
@@ -137,9 +134,9 @@ public abstract class TSEncodingBuilder {
switch (type) {
case INT32:
case BOOLEAN:
- return new IntRleEncoder(EndianType.BIG_ENDIAN);
+ return new IntRleEncoder();
case INT64:
- return new LongRleEncoder(EndianType.BIG_ENDIAN);
+ return new LongRleEncoder();
case FLOAT:
case DOUBLE:
return new FloatEncoder(TSEncoding.RLE, type, maxPointNumber);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index fd3e50a..a966ba4 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -659,7 +659,7 @@ public class TsFileSequenceReader implements AutoCloseable {
* @return a CHUNK_GROUP_FOOTER
* @throws IOException io error
*/
- public ChunkGroupHeader readChunkGroupFooter() throws IOException {
+ public ChunkGroupHeader readChunkGroupHeader() throws IOException {
return ChunkGroupHeader.deserializeFrom(tsFileInput.wrapAsInputStream(), true);
}
@@ -671,7 +671,7 @@ public class TsFileSequenceReader implements AutoCloseable {
* @return a CHUNK_GROUP_FOOTER
* @throws IOException io error
*/
- public ChunkGroupHeader readChunkGroupFooter(long position, boolean markerRead)
+ public ChunkGroupHeader readChunkGroupHeader(long position, boolean markerRead)
throws IOException {
return ChunkGroupHeader.deserializeFrom(tsFileInput, position, markerRead);
}
@@ -968,7 +968,7 @@ public class TsFileSequenceReader implements AutoCloseable {
// this is a chunk group
// if there is something wrong with the ChunkGroup Footer, we will drop this ChunkGroup
// because we can not guarantee the correctness of the deviceId.
- ChunkGroupHeader chunkGroupHeader = this.readChunkGroupFooter();
+ ChunkGroupHeader chunkGroupHeader = this.readChunkGroupHeader();
deviceID = chunkGroupHeader.getDeviceID();
if (newSchema != null) {
for (MeasurementSchema tsSchema : measurementSchemaList) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
index 57549a4..f747926 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
@@ -33,7 +33,6 @@ import org.apache.iotdb.tsfile.read.reader.IPageReader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.IChunkReader;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
-import org.apache.iotdb.tsfile.v1.file.utils.HeaderUtils;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -92,8 +91,7 @@ public class ChunkReader implements IChunkReader {
// construct next satisfied page header
while (chunkDataBuffer.remaining() > 0) {
// deserialize a PageHeader from chunkDataBuffer
- PageHeader pageHeader = isFromOldTsFile ? HeaderUtils.deserializePageHeaderV1(chunkDataBuffer, chunkHeader.getDataType()) :
- PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType());
+ PageHeader pageHeader = PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType());
// if the current page satisfies
if (pageSatisfied(pageHeader)) {
pageReaderList.add(constructPageReaderForNextPage(pageHeader));
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
index 52052d4..5dbd171 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
@@ -91,7 +91,6 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), false)) {
truncatedSize = reader.selfCheck(knownSchemas, chunkGroupMetadataList, versionInfo, true);
- totalChunkNum = reader.getTotalChunkNum();
if (truncatedSize == TsFileCheckStatus.COMPLETE_FILE) {
crashed = false;
canWrite = false;
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java
index bed4ee9..95b413d 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java
@@ -26,30 +26,22 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
-import org.apache.iotdb.tsfile.encoding.decoder.IntRleDecoder;
-import org.apache.iotdb.tsfile.encoding.decoder.RleDecoder;
import org.apache.iotdb.tsfile.encoding.encoder.IntRleEncoder;
import org.apache.iotdb.tsfile.encoding.encoder.RleEncoder;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
public class IntRleDecoderTest {
private List<Integer> rleList;
private List<Integer> bpList;
private List<Integer> hybridList;
- private int rleBitWidth;
- private int bpBitWidth;
- private int hybridWidth;
@Before
public void setUp() throws Exception {
- rleList = new ArrayList<Integer>();
+ rleList = new ArrayList<>();
int rleCount = 11;
int rleNum = 18;
int rleStart = 11;
@@ -63,9 +55,8 @@ public class IntRleDecoderTest {
rleCount += 2;
rleStart *= -3;
}
- rleBitWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(rleList);
- bpList = new ArrayList<Integer>();
+ bpList = new ArrayList<>();
int bpCount = 100000;
int bpStart = 11;
for (int i = 0; i < bpCount; i++) {
@@ -76,9 +67,8 @@ public class IntRleDecoderTest {
bpList.add(bpStart);
}
}
- bpBitWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(bpList);
- hybridList = new ArrayList<Integer>();
+ hybridList = new ArrayList<>();
int hybridCount = 11;
int hybridNum = 1000;
int hybridStart = 20;
@@ -101,7 +91,6 @@ public class IntRleDecoderTest {
}
hybridCount += 2;
}
- hybridWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(hybridList);
}
@After
@@ -114,17 +103,16 @@ public class IntRleDecoderTest {
for (int i = 7000000; i < 10000000; i++) {
list.add(i);
}
- int width = ReadWriteForEncodingUtils.getIntMaxBitWidth(list);
- testLength(list, width, false, 1);
+ testLength(list, false, 1);
for (int i = 1; i < 10; i++) {
- testLength(list, width, false, i);
+ testLength(list, false, i);
}
}
@Test
public void testRleReadInt() throws IOException {
for (int i = 1; i < 10; i++) {
- testLength(rleList, rleBitWidth, false, i);
+ testLength(rleList, false, i);
}
}
@@ -144,30 +132,29 @@ public class IntRleDecoderTest {
rleCount *= 7;
rleStart *= -3;
}
- int bitWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(repeatList);
for (int i = 1; i < 10; i++) {
- testLength(repeatList, bitWidth, false, i);
+ testLength(repeatList, false, i);
}
}
@Test
public void testBitPackingReadInt() throws IOException {
for (int i = 1; i < 10; i++) {
- testLength(bpList, bpBitWidth, false, i);
+ testLength(bpList, false, i);
}
}
@Test
public void testHybridReadInt() throws IOException {
for (int i = 1; i < 3; i++) {
- testLength(hybridList, hybridWidth, false, i);
+ testLength(hybridList, false, i);
}
}
@Test
public void testHybridReadBoolean() throws IOException {
for (int i = 1; i < 10; i++) {
- testLength(hybridList, hybridWidth, false, i);
+ testLength(hybridList, false, i);
}
}
@@ -178,44 +165,10 @@ public class IntRleDecoderTest {
}
}
- public void testBooleanLength(List<Integer> list, int bitWidth, boolean isDebug, int repeatCount)
+ public void testLength(List<Integer> list, boolean isDebug, int repeatCount)
throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
- RleEncoder<Integer> encoder = new IntRleEncoder(EndianType.BIG_ENDIAN);
- for (int i = 0; i < repeatCount; i++) {
- for (int value : list) {
- if (value % 2 == 0) {
- encoder.encode(false, baos);
- } else {
- encoder.encode(true, baos);
- }
-
- }
- encoder.flush(baos);
- }
-
- ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
- RleDecoder decoder = new IntRleDecoder(EndianType.BIG_ENDIAN);
- for (int i = 0; i < repeatCount; i++) {
- for (int value : list) {
- boolean value_ = decoder.readBoolean(buffer);
- if (isDebug) {
- System.out.println(value_ + "/" + value);
- }
- if (value % 2 == 0) {
- assertEquals(false, value_);
- } else {
- assertEquals(true, value_);
- }
-
- }
- }
- }
-
- public void testLength(List<Integer> list, int bitWidth, boolean isDebug, int repeatCount)
- throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- RleEncoder<Integer> encoder = new IntRleEncoder(EndianType.BIG_ENDIAN);
+ RleEncoder<Integer> encoder = new IntRleEncoder();
for (int i = 0; i < repeatCount; i++) {
for (int value : list) {
encoder.encode(value, baos);
@@ -224,7 +177,7 @@ public class IntRleDecoderTest {
}
ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
- RleDecoder decoder = new IntRleDecoder(EndianType.BIG_ENDIAN);
+ RleDecoder decoder = new IntRleDecoder();
for (int i = 0; i < repeatCount; i++) {
for (int value : list) {
int value_ = decoder.readInt(buffer);
@@ -237,14 +190,14 @@ public class IntRleDecoderTest {
}
private void testBitPackedReadHeader(int num) throws IOException {
- List<Integer> list = new ArrayList<Integer>();
+ List<Integer> list = new ArrayList<>();
for (int i = 0; i < num; i++) {
list.add(i);
}
ByteArrayOutputStream baos = new ByteArrayOutputStream();
int bitWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(list);
- RleEncoder<Integer> encoder = new IntRleEncoder(EndianType.BIG_ENDIAN);
+ RleEncoder<Integer> encoder = new IntRleEncoder();
for (int value : list) {
encoder.encode(value, baos);
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java
index ad41ac3..153112d 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java
@@ -26,17 +26,12 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
-import org.apache.iotdb.tsfile.encoding.decoder.LongRleDecoder;
-import org.apache.iotdb.tsfile.encoding.decoder.RleDecoder;
import org.apache.iotdb.tsfile.encoding.encoder.LongRleEncoder;
import org.apache.iotdb.tsfile.encoding.encoder.RleEncoder;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
public class LongRleDecoderTest {
@@ -184,7 +179,7 @@ public class LongRleDecoderTest {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
int bitWidth = ReadWriteForEncodingUtils.getLongMaxBitWidth(list);
- RleEncoder<Long> encoder = new LongRleEncoder(EndianType.BIG_ENDIAN);
+ RleEncoder<Long> encoder = new LongRleEncoder();
for (long value : list) {
encoder.encode(value, baos);
}
@@ -206,7 +201,7 @@ public class LongRleDecoderTest {
public void testLength(List<Long> list, int bitWidth, boolean isDebug, int repeatCount)
throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
- RleEncoder<Long> encoder = new LongRleEncoder(EndianType.BIG_ENDIAN);
+ RleEncoder<Long> encoder = new LongRleEncoder();
for (int i = 0; i < repeatCount; i++) {
for (long value : list) {
encoder.encode(value, baos);
@@ -215,7 +210,7 @@ public class LongRleDecoderTest {
}
ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
- RleDecoder decoder = new LongRleDecoder(EndianType.BIG_ENDIAN);
+ RleDecoder decoder = new LongRleDecoder();
for (int i = 0; i < repeatCount; i++) {
for (long value : list) {
long value_ = decoder.readLong(buffer);
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java
index 0a0131c..4bcbe5b 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java
@@ -18,11 +18,11 @@
*/
package org.apache.iotdb.tsfile.file.metadata.statistics;
-import org.junit.Test;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import org.junit.Test;
+
public class DoubleStatisticsTest {
private static final double maxError = 0.0001d;
@@ -36,7 +36,7 @@ public class DoubleStatisticsTest {
assertFalse(doubleStats.isEmpty());
assertEquals(2.32d, doubleStats.getMaxValue(), maxError);
assertEquals(1.34d, doubleStats.getMinValue(), maxError);
- assertEquals(2.32d + 1.34d, doubleStats.getSumValue(), maxError);
+ assertEquals(2.32d + 1.34d, doubleStats.getSumDoubleValue(), maxError);
assertEquals(1.34d, doubleStats.getFirstValue(), maxError);
assertEquals(2.32d, doubleStats.getLastValue(), maxError);
}
@@ -60,14 +60,14 @@ public class DoubleStatisticsTest {
assertFalse(doubleStats3.isEmpty());
assertEquals(100.13453d, doubleStats3.getMaxValue(), maxError);
assertEquals(1.34d, doubleStats3.getMinValue(), maxError);
- assertEquals(100.13453d + 1.34d, doubleStats3.getSumValue(), maxError);
+ assertEquals(100.13453d + 1.34d, doubleStats3.getSumDoubleValue(), maxError);
assertEquals(1.34d, doubleStats3.getFirstValue(), maxError);
assertEquals(100.13453d, doubleStats3.getLastValue(), maxError);
doubleStats3.mergeStatistics(doubleStats2);
assertEquals(200.435d, doubleStats3.getMaxValue(), maxError);
assertEquals(1.34d, doubleStats3.getMinValue(), maxError);
- assertEquals(100.13453d + 1.34d + 200.435d, doubleStats3.getSumValue(), maxError);
+ assertEquals(100.13453d + 1.34d + 200.435d, doubleStats3.getSumDoubleValue(), maxError);
assertEquals(1.34d, doubleStats3.getFirstValue(), maxError);
assertEquals(200.435d, doubleStats3.getLastValue(), maxError);
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java
index 8bd49c3..2867a23 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java
@@ -18,11 +18,11 @@
*/
package org.apache.iotdb.tsfile.file.metadata.statistics;
-import org.junit.Test;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import org.junit.Test;
+
public class FloatStatisticsTest {
private static final float maxError = 0.0001f;
@@ -36,7 +36,7 @@ public class FloatStatisticsTest {
assertFalse(floatStats.isEmpty());
assertEquals(2.32f, (double) floatStats.getMaxValue(), maxError);
assertEquals(1.34f, (double) floatStats.getMinValue(), maxError);
- assertEquals(2.32f + 1.34f, (double) floatStats.getSumValue(), maxError);
+ assertEquals(2.32f + 1.34f, (double) floatStats.getSumDoubleValue(), maxError);
assertEquals(1.34f, (double) floatStats.getFirstValue(), maxError);
assertEquals(2.32f, (double) floatStats.getLastValue(), maxError);
}
@@ -60,14 +60,14 @@ public class FloatStatisticsTest {
assertFalse(floatStats3.isEmpty());
assertEquals(100.13453f, floatStats3.getMaxValue(), maxError);
assertEquals(1.34f, floatStats3.getMinValue(), maxError);
- assertEquals(100.13453f + 1.34f, (float) floatStats3.getSumValue(), maxError);
+ assertEquals(100.13453f + 1.34f, (float) floatStats3.getSumDoubleValue(), maxError);
assertEquals(1.34f, floatStats3.getFirstValue(), maxError);
assertEquals(100.13453f, floatStats3.getLastValue(), maxError);
floatStats3.mergeStatistics(floatStats2);
assertEquals(200.435f, floatStats3.getMaxValue(), maxError);
assertEquals(1.34f, floatStats3.getMinValue(), maxError);
- assertEquals(100.13453f + 1.34f + 200.435f, (float) floatStats3.getSumValue(), maxError);
+ assertEquals(100.13453f + 1.34f + 200.435f, (float) floatStats3.getSumDoubleValue(), maxError);
assertEquals(1.34f, floatStats3.getFirstValue(), maxError);
assertEquals(200.435f, floatStats3.getLastValue(), maxError);
@@ -87,7 +87,6 @@ public class FloatStatisticsTest {
assertEquals(122.34f, floatStats3.getFirstValue(), maxError);
assertEquals(125.34f, floatStats3.getLastValue(), maxError);
-
floatStats3.mergeStatistics(floatStats5);
assertEquals(122.34f, floatStats3.getFirstValue(), maxError);
assertEquals(125.34f, floatStats3.getLastValue(), maxError);
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java
index 9de231e..469f7b1 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java
@@ -18,11 +18,11 @@
*/
package org.apache.iotdb.tsfile.file.metadata.statistics;
-import org.junit.Test;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import org.junit.Test;
+
public class IntegerStatisticsTest {
@Test
@@ -35,7 +35,7 @@ public class IntegerStatisticsTest {
assertEquals(2, (int) intStats.getMaxValue());
assertEquals(1, (int) intStats.getMinValue());
assertEquals(1, (int) intStats.getFirstValue());
- assertEquals(3, (int) intStats.getSumValue());
+ assertEquals(3, (int) intStats.getSumLongValue());
assertEquals(2, (int) intStats.getLastValue());
}
@@ -59,14 +59,14 @@ public class IntegerStatisticsTest {
assertEquals(100, (int) intStats3.getMaxValue());
assertEquals(1, (int) intStats3.getMinValue());
assertEquals(1, (int) intStats3.getFirstValue());
- assertEquals(1 + 100, (int) intStats3.getSumValue());
+ assertEquals(1 + 100, (int) intStats3.getSumLongValue());
assertEquals(100, (int) intStats3.getLastValue());
intStats3.mergeStatistics(intStats2);
assertEquals(200, (int) intStats3.getMaxValue());
assertEquals(1, (int) intStats3.getMinValue());
assertEquals(1, (int) intStats3.getFirstValue());
- assertEquals(101 + 200, (int) intStats3.getSumValue());
+ assertEquals(101 + 200, (int) intStats3.getSumLongValue());
assertEquals(200, (int) intStats3.getLastValue());
// Unseq merge
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java
index 914d37f..4aa0c6f 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java
@@ -18,11 +18,14 @@
*/
package org.apache.iotdb.tsfile.file.metadata.statistics;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException;
import org.junit.Test;
-import static org.junit.Assert.*;
-
public class LongStatisticsTest {
@Test
@@ -38,7 +41,7 @@ public class LongStatisticsTest {
assertEquals(secondValue, (long) longStats.getMaxValue());
assertEquals(firstValue, (long) longStats.getMinValue());
assertEquals(firstValue, (long) longStats.getFirstValue());
- assertEquals(firstValue + secondValue, (long) longStats.getSumValue());
+ assertEquals(firstValue + secondValue, (long) longStats.getSumDoubleValue());
assertEquals(secondValue, (long) longStats.getLastValue());
}
@@ -63,14 +66,14 @@ public class LongStatisticsTest {
assertFalse(longStats3.isEmpty());
assertEquals(max1, (long) longStats3.getMaxValue());
assertEquals(1, (long) longStats3.getMinValue());
- assertEquals(max1 + 1, (long) longStats3.getSumValue());
+ assertEquals(max1 + 1, (long) longStats3.getSumDoubleValue());
assertEquals(1, (long) longStats3.getFirstValue());
assertEquals(max1, (long) longStats3.getLastValue());
longStats3.mergeStatistics(longStats2);
assertEquals(max2, (long) longStats3.getMaxValue());
assertEquals(1, (long) longStats3.getMinValue());
- assertEquals(max2 + max1 + 1, (long) longStats3.getSumValue());
+ assertEquals(max2 + max1 + 1, (long) longStats3.getSumDoubleValue());
assertEquals(1, (long) longStats3.getFirstValue());
assertEquals(max2, (long) longStats3.getLastValue());
@@ -89,7 +92,7 @@ public class LongStatisticsTest {
// if not merge, the min value will not be changed by smaller value in
// intStats5
assertEquals(1, (long) longStats3.getMinValue());
- assertEquals(max2 + max1 + 1, (long) longStats3.getSumValue());
+ assertEquals(max2 + max1 + 1, (long) longStats3.getSumDoubleValue());
assertEquals(1, (long) longStats3.getFirstValue());
assertEquals(max2, (long) longStats3.getLastValue());
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java
index 4dff2b2..cdbd50f 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java
@@ -29,6 +29,8 @@ import org.apache.iotdb.tsfile.file.header.PageHeader;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.MetadataIndexNode;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
+import org.apache.iotdb.tsfile.file.metadata.statistics.BooleanStatistics;
+import org.apache.iotdb.tsfile.file.metadata.statistics.IntegerStatistics;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.junit.Assert;
@@ -131,11 +133,11 @@ public class Utils {
public static void isPageHeaderEqual(PageHeader header1, PageHeader header2) {
if (Utils.isTwoObjectsNotNULL(header1, header2, "PageHeader")) {
- assertTrue(header1.getUncompressedSize() == header2.getUncompressedSize());
- assertTrue(header1.getCompressedSize() == header2.getCompressedSize());
- assertTrue(header1.getNumOfValues() == header2.getNumOfValues());
- assertTrue(header1.getEndTime() == header2.getEndTime());
- assertTrue(header1.getStartTime() == header2.getStartTime());
+ assertEquals(header1.getUncompressedSize(), header2.getUncompressedSize());
+ assertEquals(header1.getCompressedSize(), header2.getCompressedSize());
+ assertEquals(header1.getNumOfValues(), header2.getNumOfValues());
+ assertEquals(header1.getEndTime(), header2.getEndTime());
+ assertEquals(header1.getStartTime(), header2.getStartTime());
if (Utils
.isTwoObjectsNotNULL(header1.getStatistics(), header2.getStatistics(), "statistics")) {
Utils.isStatisticsEqual(header1.getStatistics(), header2.getStatistics());
@@ -144,21 +146,23 @@ public class Utils {
}
public static void isStatisticsEqual(Statistics statistics1, Statistics statistics2) {
- if ((statistics1 == null) ^ (statistics2 == null)) {
+ if ((statistics1 == null) || (statistics2 == null)) {
System.out.println("error");
fail("one of statistics is null");
}
- if ((statistics1 != null) && (statistics2 != null)) {
- if (statistics1.isEmpty() ^ statistics2.isEmpty()) {
- fail("one of statistics is empty while the other one is not");
- }
- if (!statistics1.isEmpty() && !statistics2.isEmpty()) {
- assertEquals(statistics1.getMinValue(), statistics2.getMinValue());
- assertEquals(statistics1.getMaxValue(), statistics2.getMaxValue());
- assertEquals(statistics1.getFirstValue(), statistics2.getFirstValue());
- assertEquals(statistics1.getSumValue(), statistics2.getSumValue(), maxError);
- assertEquals(statistics1.getLastValue(), statistics2.getLastValue());
+ if (statistics1.isEmpty() || statistics2.isEmpty()) {
+ fail("one of statistics is empty while the other one is not");
+ }
+ if (!statistics1.isEmpty() && !statistics2.isEmpty()) {
+ assertEquals(statistics1.getMinValue(), statistics2.getMinValue());
+ assertEquals(statistics1.getMaxValue(), statistics2.getMaxValue());
+ assertEquals(statistics1.getFirstValue(), statistics2.getFirstValue());
+ if (statistics1 instanceof IntegerStatistics || statistics1 instanceof BooleanStatistics) {
+ assertEquals(statistics1.getSumLongValue(), statistics2.getSumLongValue());
+ } else {
+ assertEquals(statistics1.getSumDoubleValue(), statistics2.getSumDoubleValue(), maxError);
}
+ assertEquals(statistics1.getLastValue(), statistics2.getLastValue());
}
}
}
\ No newline at end of file
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java
index bb86168..c9931fc 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java
@@ -80,7 +80,7 @@ public class TsFileSequenceReaderTest {
}
break;
case MetaMarker.CHUNK_GROUP_HEADER:
- ChunkGroupHeader footer = reader.readChunkGroupFooter();
+ ChunkGroupHeader footer = reader.readChunkGroupHeader();
long endOffset = reader.position();
Pair<Long, Long> pair = new Pair<>(startOffset, endOffset);
deviceChunkGroupMetadataOffsets.putIfAbsent(footer.getDeviceID(), new ArrayList<>());
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java
index 65d86e6..4f4a681 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
import org.apache.iotdb.tsfile.encoding.decoder.DeltaBinaryDecoder;
import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionDecoderV1;
@@ -54,11 +53,11 @@ public class PageReaderTest {
public void testLong() {
LoopWriteReadTest test = new LoopWriteReadTest("Test INT64",
- new LongRleEncoder(EndianType.BIG_ENDIAN),
- new LongRleDecoder(EndianType.BIG_ENDIAN), TSDataType.INT64, POINTS_COUNT_IN_ONE_PAGE) {
+ new LongRleEncoder(),
+ new LongRleDecoder(), TSDataType.INT64, POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
- return Long.valueOf(Long.MAX_VALUE - i);
+ return Long.MAX_VALUE - i;
}
};
test.test(TSDataType.INT64);
@@ -67,11 +66,11 @@ public class PageReaderTest {
@Test
public void testBoolean() {
LoopWriteReadTest test = new LoopWriteReadTest("Test Boolean",
- new IntRleEncoder(EndianType.BIG_ENDIAN),
- new IntRleDecoder(EndianType.BIG_ENDIAN), TSDataType.BOOLEAN, POINTS_COUNT_IN_ONE_PAGE) {
+ new IntRleEncoder(),
+ new IntRleDecoder(), TSDataType.BOOLEAN, POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
- return i % 3 == 0 ? true : false;
+ return i % 3 == 0;
}
};
test.test(TSDataType.BOOLEAN);
@@ -79,12 +78,11 @@ public class PageReaderTest {
@Test
public void testInt() {
- LoopWriteReadTest test = new LoopWriteReadTest("Test INT32",
- new IntRleEncoder(EndianType.BIG_ENDIAN),
- new IntRleDecoder(EndianType.BIG_ENDIAN), TSDataType.INT32, POINTS_COUNT_IN_ONE_PAGE) {
+ LoopWriteReadTest test = new LoopWriteReadTest("Test INT32", new IntRleEncoder(),
+ new IntRleDecoder(), TSDataType.INT32, POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
- return Integer.valueOf(i);
+ return i;
}
};
test.test(TSDataType.INT32);
@@ -96,7 +94,7 @@ public class PageReaderTest {
new SinglePrecisionDecoderV1(), TSDataType.FLOAT, POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
- return Float.valueOf(i) / 10 - Float.valueOf(i) / 100;
+ return (float) i / 10 - (float) i / 100;
}
};
test.test(TSDataType.FLOAT);
@@ -105,7 +103,7 @@ public class PageReaderTest {
new SinglePrecisionDecoderV1(), TSDataType.FLOAT, POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
- return Float.valueOf(i) / 100 - Float.valueOf(i) / 10;
+ return (float) i / 100 - (float) i / 10;
}
};
test2.test(TSDataType.FLOAT);
@@ -117,7 +115,7 @@ public class PageReaderTest {
new DoublePrecisionDecoderV1(), TSDataType.DOUBLE, POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
- return Double.valueOf(i) / 10 - Double.valueOf(i) / 100;
+ return (double) i / 10 - (double) i / 100;
}
};
test.test(TSDataType.DOUBLE);
@@ -126,7 +124,7 @@ public class PageReaderTest {
new DoublePrecisionDecoderV1(), TSDataType.DOUBLE, POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
- return Double.valueOf(i) / 1000 - Double.valueOf(i) / 100;
+ return (double) i / 1000 - (double) i / 100;
}
};
test2.test(TSDataType.DOUBLE);
@@ -135,11 +133,11 @@ public class PageReaderTest {
@Test
public void testBinary() {
LoopWriteReadTest test = new LoopWriteReadTest("Test Double",
- new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.TEXT, 1000),
- new PlainDecoder(EndianType.BIG_ENDIAN), TSDataType.TEXT, POINTS_COUNT_IN_ONE_PAGE) {
+ new PlainEncoder(TSDataType.TEXT, 1000),
+ new PlainDecoder(), TSDataType.TEXT, POINTS_COUNT_IN_ONE_PAGE) {
@Override
public Object generateValueByIndex(int i) {
- return new Binary(new StringBuilder("TEST TEXT").append(i).toString());
+ return new Binary("TEST TEXT" + i);
}
};
test.test(TSDataType.TEXT);
@@ -238,26 +236,26 @@ public class PageReaderTest {
}
}
- private void writeData() throws IOException {
+ private void writeData() {
for (int i = 0; i < count; i++) {
switch (dataType) {
case BOOLEAN:
- pageWriter.write(Long.valueOf(i), (Boolean) generateValueByIndex(i));
+ pageWriter.write(i, (Boolean) generateValueByIndex(i));
break;
case INT32:
- pageWriter.write(Long.valueOf(i), (Integer) generateValueByIndex(i));
+ pageWriter.write(i, (Integer) generateValueByIndex(i));
break;
case INT64:
- pageWriter.write(Long.valueOf(i), (Long) generateValueByIndex(i));
+ pageWriter.write(i, (Long) generateValueByIndex(i));
break;
case FLOAT:
- pageWriter.write(Long.valueOf(i), (Float) generateValueByIndex(i));
+ pageWriter.write(i, (Float) generateValueByIndex(i));
break;
case DOUBLE:
- pageWriter.write(Long.valueOf(i), (Double) generateValueByIndex(i));
+ pageWriter.write(i, (Double) generateValueByIndex(i));
break;
case TEXT:
- pageWriter.write(Long.valueOf(i), (Binary) generateValueByIndex(i));
+ pageWriter.write(i, (Binary) generateValueByIndex(i));
break;
}
@@ -270,11 +268,11 @@ public class PageReaderTest {
@Test
public void testPageDelete() {
LoopWriteReadTest test = new LoopWriteReadTest("Test INT64",
- new LongRleEncoder(EndianType.BIG_ENDIAN),
- new LongRleDecoder(EndianType.BIG_ENDIAN), TSDataType.INT64, 100) {
+ new LongRleEncoder(),
+ new LongRleDecoder(), TSDataType.INT64, 100) {
@Override
public Object generateValueByIndex(int i) {
- return Long.valueOf(Long.MAX_VALUE - i);
+ return Long.MAX_VALUE - i;
}
};
test.testDelete(TSDataType.INT64);
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java
index 61e4786..675a77c 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.constant.TestConstant;
import org.apache.iotdb.tsfile.file.MetaMarker;
-import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
+import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
import org.apache.iotdb.tsfile.file.metadata.TimeSeriesMetadataTest;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
@@ -87,18 +87,17 @@ public class TsFileIOWriterTest {
Assert.assertEquals(TSFileConfig.VERSION_NUMBER, reader.readVersionNumber());
Assert.assertEquals(TSFileConfig.MAGIC_STRING, reader.readTailMagic());
+ // chunk group header
+ Assert.assertEquals(MetaMarker.CHUNK_GROUP_HEADER, reader.readMarker());
+ ChunkGroupHeader footer = reader.readChunkGroupHeader();
+ Assert.assertEquals(deviceId, footer.getDeviceID());
+
// chunk header
- reader.position(TSFileConfig.MAGIC_STRING.getBytes().length + TSFileConfig.VERSION_NUMBER
- .getBytes().length);
- Assert.assertEquals(MetaMarker.CHUNK_HEADER, reader.readMarker());
- ChunkHeader header = reader.readChunkHeader();
+ reader.position(TSFileConfig.MAGIC_STRING.getBytes().length + 1);
+ Assert.assertEquals(MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, reader.readMarker());
+ ChunkHeader header = reader.readChunkHeader(MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER);
Assert.assertEquals(TimeSeriesMetadataTest.measurementUID, header.getMeasurementID());
- // chunk group footer
- Assert.assertEquals(MetaMarker.CHUNK_GROUP_FOOTER, reader.readMarker());
- ChunkGroupFooter footer = reader.readChunkGroupFooter();
- Assert.assertEquals(deviceId, footer.getDeviceID());
-
// separator
Assert.assertEquals(MetaMarker.VERSION, reader.readMarker());
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/PageWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/PageWriterTest.java
index 05cfc24..3253495 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/PageWriterTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/PageWriterTest.java
@@ -18,8 +18,12 @@
*/
package org.apache.iotdb.tsfile.write.writer;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
import org.apache.iotdb.tsfile.constant.TestConstant;
-import org.apache.iotdb.tsfile.encoding.common.EndianType;
import org.apache.iotdb.tsfile.encoding.decoder.PlainDecoder;
import org.apache.iotdb.tsfile.encoding.encoder.PlainEncoder;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -28,13 +32,6 @@ import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.iotdb.tsfile.write.page.PageWriter;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.apache.iotdb.tsfile.constant.TestConstant;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
import org.junit.Test;
public class PageWriterTest {
@@ -42,8 +39,8 @@ public class PageWriterTest {
@Test
public void testWriteInt() {
PageWriter writer = new PageWriter();
- writer.setTimeEncoder(new PlainEncoder(EndianType.LITTLE_ENDIAN, TSDataType.INT64, 0));
- writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT32, 0));
+ writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0));
+ writer.setValueEncoder(new PlainEncoder(TSDataType.INT32, 0));
writer.initStatistics(TSDataType.INT32);
int value = 1;
int timeCount = 0;
@@ -58,12 +55,11 @@ public class PageWriterTest {
byte[] timeBytes = new byte[timeSize];
buffer.get(timeBytes);
ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes);
- PlainDecoder decoder = new PlainDecoder(EndianType.LITTLE_ENDIAN);
+ PlainDecoder decoder = new PlainDecoder();
for (int i = 0; i < timeCount; i++) {
assertEquals(i, decoder.readLong(buffer2));
}
decoder.reset();
- decoder.setEndianType(EndianType.BIG_ENDIAN);
assertEquals(value, decoder.readInt(buffer));
} catch (IOException e) {
fail();
@@ -73,8 +69,8 @@ public class PageWriterTest {
@Test
public void testWriteLong() {
PageWriter writer = new PageWriter();
- writer.setTimeEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0));
- writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0));
+ writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0));
+ writer.setValueEncoder(new PlainEncoder(TSDataType.INT64, 0));
writer.initStatistics(TSDataType.INT64);
long value = 123142120391L;
int timeCount = 0;
@@ -89,7 +85,7 @@ public class PageWriterTest {
byte[] timeBytes = new byte[timeSize];
buffer.get(timeBytes);
ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes);
- PlainDecoder decoder = new PlainDecoder(EndianType.BIG_ENDIAN);
+ PlainDecoder decoder = new PlainDecoder();
for (int i = 0; i < timeCount; i++) {
assertEquals(i, decoder.readLong(buffer2));
}
@@ -103,8 +99,8 @@ public class PageWriterTest {
@Test
public void testWriteFloat() {
PageWriter writer = new PageWriter();
- writer.setTimeEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0));
- writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.FLOAT, 0));
+ writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0));
+ writer.setValueEncoder(new PlainEncoder(TSDataType.FLOAT, 0));
writer.initStatistics(TSDataType.FLOAT);
float value = 2.2f;
int timeCount = 0;
@@ -119,7 +115,7 @@ public class PageWriterTest {
byte[] timeBytes = new byte[timeSize];
buffer.get(timeBytes);
ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes);
- PlainDecoder decoder = new PlainDecoder(EndianType.BIG_ENDIAN);
+ PlainDecoder decoder = new PlainDecoder();
for (int i = 0; i < timeCount; i++) {
assertEquals(i, decoder.readLong(buffer2));
}
@@ -133,8 +129,8 @@ public class PageWriterTest {
@Test
public void testWriteBoolean() {
PageWriter writer = new PageWriter();
- writer.setTimeEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0));
- writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.BOOLEAN, 0));
+ writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0));
+ writer.setValueEncoder(new PlainEncoder( TSDataType.BOOLEAN, 0));
writer.initStatistics(TSDataType.BOOLEAN);
boolean value = false;
int timeCount = 0;
@@ -149,7 +145,7 @@ public class PageWriterTest {
byte[] timeBytes = new byte[timeSize];
buffer.get(timeBytes);
ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes);
- PlainDecoder decoder = new PlainDecoder(EndianType.BIG_ENDIAN);
+ PlainDecoder decoder = new PlainDecoder();
for (int i = 0; i < timeCount; i++) {
assertEquals(i, decoder.readLong(buffer2));
}
@@ -162,8 +158,8 @@ public class PageWriterTest {
@Test
public void testWriteBinary() {
PageWriter writer = new PageWriter();
- writer.setTimeEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0));
- writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.TEXT, 0));
+ writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0));
+ writer.setValueEncoder(new PlainEncoder(TSDataType.TEXT, 0));
writer.initStatistics(TSDataType.TEXT);
String value = "I have a dream";
int timeCount = 0;
@@ -178,7 +174,7 @@ public class PageWriterTest {
byte[] timeBytes = new byte[timeSize];
buffer.get(timeBytes);
ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes);
- PlainDecoder decoder = new PlainDecoder(EndianType.BIG_ENDIAN);
+ PlainDecoder decoder = new PlainDecoder();
for (int i = 0; i < timeCount; i++) {
assertEquals(i, decoder.readLong(buffer2));
}
@@ -192,8 +188,8 @@ public class PageWriterTest {
@Test
public void testWriteDouble() {
PageWriter writer = new PageWriter();
- writer.setTimeEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0));
- writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.DOUBLE, 0));
+ writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0));
+ writer.setValueEncoder(new PlainEncoder(TSDataType.DOUBLE, 0));
writer.initStatistics(TSDataType.DOUBLE);
double value = 1d;
int timeCount = 0;
@@ -208,7 +204,7 @@ public class PageWriterTest {
byte[] timeBytes = new byte[timeSize];
buffer.get(timeBytes);
ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes);
- PlainDecoder decoder = new PlainDecoder(EndianType.BIG_ENDIAN);
+ PlainDecoder decoder = new PlainDecoder();
for (int i = 0; i < timeCount; i++) {
assertEquals(i, decoder.readLong(buffer2));
}