You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by in...@apache.org on 2018/07/02 05:26:30 UTC
incubator-hawq git commit: HAWQ-1624. Change libhdfs3 to be ABI
compatible with libhdfs
Repository: incubator-hawq
Updated Branches:
refs/heads/master 5920b817e -> c6146c02e
HAWQ-1624. Change libhdfs3 to be ABI compatible with libhdfs
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c6146c02
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c6146c02
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c6146c02
Branch: refs/heads/master
Commit: c6146c02e0ea24c12f1f29bf230eeeec96b99859
Parents: 5920b81
Author: Sergei Lebedev <s....@criteo.com>
Authored: Tue Jun 12 19:55:02 2018 +0200
Committer: interma <in...@outlook.com>
Committed: Mon Jul 2 13:25:59 2018 +0800
----------------------------------------------------------------------
depends/libhdfs3/src/client/Hdfs.cpp | 29 ++++++++++++--
depends/libhdfs3/src/client/hdfs.h | 34 +++++++++++++++-
.../libhdfs3/test/function/TestCInterface.cpp | 41 +++++++++++++++++---
3 files changed, 93 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c6146c02/depends/libhdfs3/src/client/Hdfs.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/Hdfs.cpp b/depends/libhdfs3/src/client/Hdfs.cpp
index 2840adc..b8b7830 100644
--- a/depends/libhdfs3/src/client/Hdfs.cpp
+++ b/depends/libhdfs3/src/client/Hdfs.cpp
@@ -791,6 +791,23 @@ tSize hdfsRead(hdfsFS fs, hdfsFile file, void * buffer, tSize length) {
return -1;
}
+tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset pos,
+ void * buffer, tSize length) {
+ PARAMETER_ASSERT(fs && file && pos >= 0 && buffer && length > 0, -1, EINVAL);
+ PARAMETER_ASSERT(file->isInput(), -1, EINVAL);
+
+ tOffset oldPos = hdfsTell(fs, file);
+ if (oldPos < 0 || hdfsSeek(fs, file, pos) < 0) {
+ return -1;
+ }
+
+ tSize done = hdfsRead(fs, file, buffer, length);
+ // hdfsRead failure followed by hdfsSeek failure would cause
+ // hdfsGetLastError to only report the latter one, since there is
+ // no exception chaining.
+ return hdfsSeek(fs, file, oldPos) == 0 ? done : -1;
+}
+
tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void * buffer, tSize length) {
PARAMETER_ASSERT(fs && file && buffer && length > 0, -1, EINVAL);
PARAMETER_ASSERT(!file->isInput(), -1, EINVAL);
@@ -811,12 +828,12 @@ tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void * buffer, tSize length) {
}
int hdfsFlush(hdfsFS fs, hdfsFile file) {
- PARAMETER_ASSERT(fs && file && file, -1, EINVAL);
+ PARAMETER_ASSERT(fs && file, -1, EINVAL);
return hdfsHFlush(fs, file);
}
int hdfsHFlush(hdfsFS fs, hdfsFile file) {
- PARAMETER_ASSERT(fs && file && file, -1, EINVAL);
+ PARAMETER_ASSERT(fs && file, -1, EINVAL);
PARAMETER_ASSERT(!file->isInput(), -1, EINVAL);
try {
@@ -834,7 +851,11 @@ int hdfsHFlush(hdfsFS fs, hdfsFile file) {
}
int hdfsSync(hdfsFS fs, hdfsFile file) {
- PARAMETER_ASSERT(fs && file && file, -1, EINVAL);
+ return hdfsHSync(fs, file);
+}
+
+int hdfsHSync(hdfsFS fs, hdfsFile file) {
+ PARAMETER_ASSERT(fs && file, -1, EINVAL);
PARAMETER_ASSERT(!file->isInput(), -1, EINVAL);
try {
@@ -852,7 +873,7 @@ int hdfsSync(hdfsFS fs, hdfsFile file) {
}
int hdfsAvailable(hdfsFS fs, hdfsFile file) {
- PARAMETER_ASSERT(fs && file && file, -1, EINVAL);
+ PARAMETER_ASSERT(fs && file, -1, EINVAL);
PARAMETER_ASSERT(file->isInput(), -1, EINVAL);
try {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c6146c02/depends/libhdfs3/src/client/hdfs.h
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/src/client/hdfs.h b/depends/libhdfs3/src/client/hdfs.h
index f8b61ea..b784367 100644
--- a/depends/libhdfs3/src/client/hdfs.h
+++ b/depends/libhdfs3/src/client/hdfs.h
@@ -39,6 +39,15 @@
#define EINTERNAL 255
#endif
+#if defined(__GNUC__) || defined(__clang__)
+#define DEPRECATED __attribute__((deprecated))
+#elif defined(_MSC_VER)
+#define DEPRECATED __declspec(deprecated)
+#else
+#pragma message("WARNING: DEPRECATED is not supported by the compiler.")
+#define DEPRECATED
+#endif
+
/** All APIs set errno to meaningful values */
#ifdef __cplusplus
@@ -360,6 +369,18 @@ tOffset hdfsTell(hdfsFS fs, hdfsFile file);
tSize hdfsRead(hdfsFS fs, hdfsFile file, void * buffer, tSize length);
/**
+ * hdfsPread - Positional read of data from an open file.
+ * @param fs The configured filesystem handle.
+ * @param file The file handle.
+ * @param offset Position from which to read
+ * @param buffer The buffer to copy read bytes into.
+ * @param length The length of the buffer.
+ * @return See hdfsRead
+ */
+tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset offset,
+ void * buffer, tSize length);
+
+/**
* hdfsWrite - Write data into an open file.
* @param fs The configured filesystem handle.
* @param file The file handle.
@@ -387,13 +408,24 @@ int hdfsFlush(hdfsFS fs, hdfsFile file);
int hdfsHFlush(hdfsFS fs, hdfsFile file);
/**
+ * This function is deprecated. Please use hdfsHSync instead.
+ *
* hdfsSync - Flush out and sync the data in client's user buffer. After the
* return of this call, new readers will see the data.
* @param fs configured filesystem handle
* @param file file handle
* @return 0 on success, -1 on error and sets errno
*/
-int hdfsSync(hdfsFS fs, hdfsFile file);
+DEPRECATED int hdfsSync(hdfsFS fs, hdfsFile file);
+
+/**
+ * hdfsHSync - Flush out and sync the data in client's user buffer. After the
+ * return of this call, new readers will see the data.
+ * @param fs configured filesystem handle
+ * @param file file handle
+ * @return 0 on success, -1 on error and sets errno
+ */
+int hdfsHSync(hdfsFS fs, hdfsFile file);
/**
* hdfsAvailable - Number of bytes that can be read from this
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c6146c02/depends/libhdfs3/test/function/TestCInterface.cpp
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/test/function/TestCInterface.cpp b/depends/libhdfs3/test/function/TestCInterface.cpp
index bca7884..887a52c 100644
--- a/depends/libhdfs3/test/function/TestCInterface.cpp
+++ b/depends/libhdfs3/test/function/TestCInterface.cpp
@@ -29,6 +29,7 @@
#include "TestUtil.h"
#include "XmlConfig.h"
+#include <algorithm>
#include <errno.h>
#include <fcntl.h>
#include <stdlib.h>
@@ -1571,6 +1572,34 @@ TEST_F(TestCInterface, TestRead_Success) {
TestRead(fs, 8 * 1024, 1024 * 1024, 21 * 1024 * 1024);
}
+static void TestPread(hdfsFS fs, int readSize, int64_t blockSize,
+ int64_t fileSize, int64_t offset) {
+ std::vector<char> buf(readSize);
+ hdfsFile in = NULL;
+ ASSERT_TRUE(CreateFile(fs, BASE_DIR"/testPread", blockSize, fileSize));
+ in = hdfsOpenFile(fs, BASE_DIR"/testPread", O_RDONLY, 0, 0, 0);
+ ASSERT_TRUE(in != NULL);
+
+ int64_t todo = fileSize - offset;
+ while (todo > 0) {
+ int64_t done = hdfsPread(fs, in, offset, &buf[0], std::min((int64_t) readSize, todo));
+ ASSERT_GT(done, 0);
+ ASSERT_EQ(hdfsTell(fs, in), 0);
+ EXPECT_TRUE(Hdfs::CheckBuffer(&buf[0], done, offset));
+ offset += done;
+ todo -= done;
+ }
+
+ EXPECT_EQ(0, hdfsPread(fs, in, offset, &buf[0], 1));
+ EXPECT_EQ(0, hdfsPread(fs, in, offset, &buf[0], 1));
+ hdfsCloseFile(fs, in);
+}
+
+TEST_F(TestCInterface, TestPread_Success) {
+ TestPread(fs, 1024, 1024, 21 * 1024, 0);
+ TestPread(fs, 1024, 1024, 21 * 1024, 13);
+}
+
TEST_F(TestCInterface, TestWrite_InvalidInput) {
int err;
char buf[10240];
@@ -1678,11 +1707,11 @@ TEST_F(TestCInterface, TestSync_InvalidInput) {
in = hdfsOpenFile(fs, BASE_DIR"/testFlush1", O_RDONLY, 0, 0, 0);
ASSERT_TRUE(in != NULL);
//test invalid input
- err = hdfsSync(NULL, out);
+ err = hdfsHSync(NULL, out);
EXPECT_TRUE(err != 0 && EINVAL == errno);
- err = hdfsSync(fs, NULL);
+ err = hdfsHSync(fs, NULL);
EXPECT_TRUE(err != 0 && EINVAL == errno);
- err = hdfsSync(fs, in);
+ err = hdfsHSync(fs, in);
EXPECT_TRUE(err != 0 && EINVAL == errno);
EXPECT_EQ(0, hdfsCloseFile(fs, out));
EXPECT_EQ(0, hdfsCloseFile(fs, in));
@@ -1732,7 +1761,7 @@ static void TestHFlushAndSync(hdfsFS fs, hdfsFile file, const char * path, int64
EXPECT_EQ(batch, hdfsWrite(fs, file, &buffer[0], batch));
if (sync) {
- EXPECT_EQ(0, hdfsSync(fs, file));
+ EXPECT_EQ(0, hdfsHSync(fs, file));
} else {
EXPECT_EQ(0, hdfsHFlush(fs, file));
}
@@ -2028,7 +2057,7 @@ TEST_F(TestCInterface, TestGetBlockFileLocations_Success) {
out = hdfsOpenFile(fs, BASE_DIR"/TestGetBlockFileLocations_Failure", O_WRONLY, 0, 0, 1024);
ASSERT_TRUE(NULL != out);
ASSERT_TRUE(buffer.size() == hdfsWrite(fs, out, &buffer[0], buffer.size()));
- ASSERT_TRUE(0 == hdfsSync(fs, out));
+ ASSERT_TRUE(0 == hdfsHSync(fs, out));
EXPECT_TRUE(NULL != (bl = hdfsGetFileBlockLocations(fs, BASE_DIR"/TestGetBlockFileLocations_Failure", 0, buffer.size(), &size)));
EXPECT_EQ(2, size);
EXPECT_FALSE(bl[0].corrupt);
@@ -2074,7 +2103,7 @@ TEST_F(TestCInterface, TestGetHosts_Success) {
1024);
ASSERT_TRUE(NULL != out);
ASSERT_TRUE(buffer.size() == hdfsWrite(fs, out, &buffer[0], buffer.size()));
- ASSERT_TRUE(0 == hdfsSync(fs, out));
+ ASSERT_TRUE(0 == hdfsHSync(fs, out));
hosts =
hdfsGetHosts(fs, BASE_DIR "/TestGetHosts_Success", 0, buffer.size());
EXPECT_TRUE(NULL != hosts);