You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by wa...@apache.org on 2016/02/17 03:26:19 UTC
[1/5] incubator-hawq git commit: HAWQ-407. Fix memory leak in parquet
metadata processing for reading table with data [Forced Update!]
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-414 54a31be92 -> e2de16a70 (forced update)
HAWQ-407. Fix memory leak in parquet metadata processing for reading table with data
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/9defbc12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/9defbc12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/9defbc12
Branch: refs/heads/HAWQ-414
Commit: 9defbc12a818971b31ab402ea0935b1ff266589f
Parents: 95a5da1
Author: Lili Ma <ic...@gmail.com>
Authored: Mon Feb 15 14:59:32 2016 +0800
Committer: Lili Ma <ic...@gmail.com>
Committed: Mon Feb 15 14:59:32 2016 +0800
----------------------------------------------------------------------
src/backend/access/parquet/parquetam.c | 4 ++--
src/backend/cdb/cdbparquetfooterprocessor.c | 3 +++
src/backend/cdb/cdbparquetfooterserializer.c | 4 ++--
3 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9defbc12/src/backend/access/parquet/parquetam.c
----------------------------------------------------------------------
diff --git a/src/backend/access/parquet/parquetam.c b/src/backend/access/parquet/parquetam.c
index 67359ca..6438e32 100644
--- a/src/backend/access/parquet/parquetam.c
+++ b/src/backend/access/parquet/parquetam.c
@@ -583,8 +583,8 @@ ParquetInsertDesc parquet_insert_init(Relation rel, ResultRelSegFileInfo *segfil
parquetInsertDesc->aoEntry = aoentry;
parquetInsertDesc->insertCount = 0;
- parquetInsertDesc->parquetMetadata = (struct ParquetMetadata_4C *)
- palloc0(sizeof(struct ParquetMetadata_4C));
+// parquetInsertDesc->parquetMetadata = (struct ParquetMetadata_4C *)
+// palloc0(sizeof(struct ParquetMetadata_4C));
/*SHOULD CALL OPEN METADATA FILE HERE, AND GET parquetMetadata INFO*/
initStringInfo(&titleBuf);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9defbc12/src/backend/cdb/cdbparquetfooterprocessor.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbparquetfooterprocessor.c b/src/backend/cdb/cdbparquetfooterprocessor.c
index 510a65f..691534a 100644
--- a/src/backend/cdb/cdbparquetfooterprocessor.c
+++ b/src/backend/cdb/cdbparquetfooterprocessor.c
@@ -111,6 +111,9 @@ bool readParquetFooter(File fileHandler, ParquetMetadata *parquetMetadata,
DetectHostEndian();
+ *parquetMetadata = (struct ParquetMetadata_4C *)
+ palloc0(sizeof(struct ParquetMetadata_4C));
+
/* if file size is 0, means there's no data in file, return false*/
if (eof == 0)
return false;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9defbc12/src/backend/cdb/cdbparquetfooterserializer.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbparquetfooterserializer.c b/src/backend/cdb/cdbparquetfooterserializer.c
index 8677078..247bbb1 100644
--- a/src/backend/cdb/cdbparquetfooterserializer.c
+++ b/src/backend/cdb/cdbparquetfooterserializer.c
@@ -167,8 +167,8 @@ void initDeserializeFooter(
initCompactProtocol(*footerProtocol, file, fileName, footerLength, PARQUET_FOOTER_BUFFERMODE_READ);
- *parquetMetadata =
- (struct ParquetMetadata_4C*) palloc0(sizeof(struct ParquetMetadata_4C));
+// *parquetMetadata =
+// (struct ParquetMetadata_4C*) palloc0(sizeof(struct ParquetMetadata_4C));
readParquetFileMetadata(parquetMetadata, *footerProtocol);
}
[2/5] incubator-hawq git commit: HAWQ-407. Remove unnecessary comments
Posted by wa...@apache.org.
HAWQ-407. Remove unnecessary comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/cf59b7b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/cf59b7b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/cf59b7b6
Branch: refs/heads/HAWQ-414
Commit: cf59b7b66beef57eda08f39472a98c43d6b76f18
Parents: 9defbc1
Author: Lili Ma <lm...@pivotal.io>
Authored: Wed Feb 17 09:28:32 2016 +0800
Committer: Lili Ma <lm...@pivotal.io>
Committed: Wed Feb 17 09:28:32 2016 +0800
----------------------------------------------------------------------
src/backend/access/parquet/parquetam.c | 4 ----
src/backend/cdb/cdbparquetfooterserializer.c | 3 ---
2 files changed, 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf59b7b6/src/backend/access/parquet/parquetam.c
----------------------------------------------------------------------
diff --git a/src/backend/access/parquet/parquetam.c b/src/backend/access/parquet/parquetam.c
index 6438e32..3c2b7df 100644
--- a/src/backend/access/parquet/parquetam.c
+++ b/src/backend/access/parquet/parquetam.c
@@ -583,10 +583,6 @@ ParquetInsertDesc parquet_insert_init(Relation rel, ResultRelSegFileInfo *segfil
parquetInsertDesc->aoEntry = aoentry;
parquetInsertDesc->insertCount = 0;
-// parquetInsertDesc->parquetMetadata = (struct ParquetMetadata_4C *)
-// palloc0(sizeof(struct ParquetMetadata_4C));
- /*SHOULD CALL OPEN METADATA FILE HERE, AND GET parquetMetadata INFO*/
-
initStringInfo(&titleBuf);
appendStringInfo(&titleBuf, "Write of Parquet relation '%s'",
RelationGetRelationName(parquetInsertDesc->parquet_rel));
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf59b7b6/src/backend/cdb/cdbparquetfooterserializer.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbparquetfooterserializer.c b/src/backend/cdb/cdbparquetfooterserializer.c
index 247bbb1..802793f 100644
--- a/src/backend/cdb/cdbparquetfooterserializer.c
+++ b/src/backend/cdb/cdbparquetfooterserializer.c
@@ -167,9 +167,6 @@ void initDeserializeFooter(
initCompactProtocol(*footerProtocol, file, fileName, footerLength, PARQUET_FOOTER_BUFFERMODE_READ);
-// *parquetMetadata =
-// (struct ParquetMetadata_4C*) palloc0(sizeof(struct ParquetMetadata_4C));
-
readParquetFileMetadata(parquetMetadata, *footerProtocol);
}
[3/5] incubator-hawq git commit: HAWQ-407. Fix memory leak in parquet
metadata processing for reading table with data
Posted by wa...@apache.org.
HAWQ-407. Fix memory leak in parquet metadata processing for reading table with data
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4c3a0918
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4c3a0918
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4c3a0918
Branch: refs/heads/HAWQ-414
Commit: 4c3a09180e6465567e569706ff4a71117ce6a0b3
Parents: 2968e51 cf59b7b
Author: Lili Ma <lm...@pivotal.io>
Authored: Wed Feb 17 09:49:55 2016 +0800
Committer: Lili Ma <lm...@pivotal.io>
Committed: Wed Feb 17 09:49:55 2016 +0800
----------------------------------------------------------------------
src/backend/access/parquet/parquetam.c | 4 ----
src/backend/cdb/cdbparquetfooterprocessor.c | 3 +++
src/backend/cdb/cdbparquetfooterserializer.c | 3 ---
3 files changed, 3 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
[4/5] incubator-hawq git commit: HAWQ-409. dangling pointer is used
in AORelRemoveHashEntry
Posted by wa...@apache.org.
HAWQ-409. dangling pointer is used in AORelRemoveHashEntry
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6dfef0db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6dfef0db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6dfef0db
Branch: refs/heads/HAWQ-414
Commit: 6dfef0db3e47cc1b5e82f8498941634e5342ef78
Parents: 4c3a091
Author: Zhanwei Wang <wa...@apache.org>
Authored: Mon Feb 15 15:27:02 2016 +0800
Committer: Zhanwei Wang <wa...@apache.org>
Committed: Wed Feb 17 10:19:00 2016 +0800
----------------------------------------------------------------------
.../access/appendonly/appendonlywriter.c | 27 +++++++++-----------
1 file changed, 12 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6dfef0db/src/backend/access/appendonly/appendonlywriter.c
----------------------------------------------------------------------
diff --git a/src/backend/access/appendonly/appendonlywriter.c b/src/backend/access/appendonly/appendonlywriter.c
index 766113c..35f84ee 100644
--- a/src/backend/access/appendonly/appendonlywriter.c
+++ b/src/backend/access/appendonly/appendonlywriter.c
@@ -504,24 +504,16 @@ AORelRemoveHashEntry(Oid relid, bool checkIsStale)
int old_next = 0;
Insist(Gp_role == GP_ROLE_DISPATCH);
- if(checkIsStale)
- {
- aoentry = (AORelHashEntryData *)hash_search(AppendOnlyHash,
- (void *) &relid,
- HASH_FIND,
- &found);
- Insist(NULL!=aoentry && aoentry ->staleTid==GetTopTransactionId());
- }
-
aoentry = (AORelHashEntryData *)hash_search(AppendOnlyHash,
- (void *) &relid,
- HASH_REMOVE,
- &found);
+ (void *) &relid,
+ HASH_FIND,
+ &found);
- if (aoentry == NULL)
- {
+ if(checkIsStale)
+ Insist(NULL!=aoentry && aoentry->staleTid==GetTopTransactionId());
+
+ if (found == false)
return false;
- }
/* Release all segment file statuses used by this relation. */
next = aoentry->head_rel_segfile.next;
@@ -534,6 +526,11 @@ AORelRemoveHashEntry(Oid relid, bool checkIsStale)
}
aoentry->head_rel_segfile.next = NEXT_END_OF_LIST;
+ aoentry = (AORelHashEntryData *)hash_search(AppendOnlyHash,
+ (void *) &relid,
+ HASH_REMOVE,
+ &found);
+
if (Debug_appendonly_print_segfile_choice)
{
ereport(LOG, (errmsg("AORelRemoveHashEntry: Remove hash entry for inactive append-only "
[5/5] incubator-hawq git commit: HAWQ-414. Share memory become
inconsistent if the number of segment files created more than
max_appendonly_segfiles
Posted by wa...@apache.org.
HAWQ-414. Share memory become inconsistent if the number of segment files created more than max_appendonly_segfiles
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e2de16a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e2de16a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e2de16a7
Branch: refs/heads/HAWQ-414
Commit: e2de16a7075958d654e5a7893b483ea56894b096
Parents: 6dfef0d
Author: Zhanwei Wang <wa...@apache.org>
Authored: Tue Feb 16 17:40:48 2016 +0800
Committer: Zhanwei Wang <wa...@apache.org>
Committed: Wed Feb 17 10:25:00 2016 +0800
----------------------------------------------------------------------
.../access/appendonly/appendonlywriter.c | 30 ++++++++++++++------
1 file changed, 22 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e2de16a7/src/backend/access/appendonly/appendonlywriter.c
----------------------------------------------------------------------
diff --git a/src/backend/access/appendonly/appendonlywriter.c b/src/backend/access/appendonly/appendonlywriter.c
index 35f84ee..e73f5a1 100644
--- a/src/backend/access/appendonly/appendonlywriter.c
+++ b/src/backend/access/appendonly/appendonlywriter.c
@@ -77,7 +77,7 @@ static AORelHashEntry AppendOnlyRelHashNew(Oid relid, bool *exists);
static AORelHashEntry AORelGetHashEntry(Oid relid);
static AORelHashEntry AORelLookupHashEntry(Oid relid);
static bool AORelCreateHashEntry(Oid relid);
-static int AORelGetSegfileStatus(void);
+static int AORelGetSegfileStatus(AORelHashEntry currentEntry);
static void AORelPutSegfileStatus(int old_status);
static AOSegfileStatus *AORelLookupSegfileStatus(int segno, AORelHashEntry entry);
static bool CheckSegFileForWriteIfNeeded(AORelHashEntryData *aoentry, AOSegfileStatus * segfilesstatus);
@@ -417,11 +417,17 @@ AORelCreateHashEntry(Oid relid)
for (i = 0 ; i < total_segfiles; i++)
{
AOSegfileStatus *status;
- int id = AORelGetSegfileStatus();
+ int id = AORelGetSegfileStatus(aoHashEntry);
/* TODO: we need to release this aoHashEntry here */
if (id == NEXT_END_OF_LIST)
{
pfree(allfsinfoParquet);
+
+ ereport(ERROR, (errmsg("cannot open more than %d "
+ "append-only table segment "
+ "files cocurrently",
+ MaxAORelSegFileStatus)));
+
return false;
}
status = &AOSegfileStatusPool[id];
@@ -446,11 +452,17 @@ AORelCreateHashEntry(Oid relid)
for (i = 0 ; i < total_segfiles; i++)
{
AOSegfileStatus *status;
- int id = AORelGetSegfileStatus();
+ int id = AORelGetSegfileStatus(aoHashEntry);
/* TODO: we need to release this aoHashEntry here */
if (id == NEXT_END_OF_LIST)
{
pfree(allfsinfo);
+
+ ereport(ERROR, (errmsg("cannot open more than %d "
+ "append-only table segment "
+ "files cocurrently",
+ MaxAORelSegFileStatus)));
+
return false;
}
status = &AOSegfileStatusPool[id];
@@ -540,6 +552,7 @@ AORelRemoveHashEntry(Oid relid, bool checkIsStale)
}
AppendOnlyWriter->num_existing_aorels--;
+ Assert(AppendOnlyWriter->num_existing_aorels >= 0);
return true;
}
@@ -1128,7 +1141,7 @@ List *SetSegnoForWrite(List *existing_segnos, Oid relid, int segment_num,
//generate new segment_num to make sure that in keepHash mode, all segment node has at least one segfile is writable
for(int i= 1; i<= segment_num; i++)
{
- int new_status = AORelGetSegfileStatus();
+ int new_status = AORelGetSegfileStatus(aoentry);
if (new_status == NEXT_END_OF_LIST)
{
LWLockRelease(AOSegFileLock);
@@ -1688,7 +1701,7 @@ void ValidateAppendOnlyMetaDataSnapshot(
}
static bool
-AORelFreeSegfileStatus()
+AORelFreeSegfileStatus(AORelHashEntry currentEntry)
{
HASH_SEQ_STATUS status;
AORelHashEntryData *hentry;
@@ -1700,7 +1713,8 @@ AORelFreeSegfileStatus()
while ((hentry = (AORelHashEntryData *) hash_seq_search(&status)) != NULL)
{
- if(hentry->txns_using_rel == 0 && InvalidTransactionId==hentry->staleTid)
+ if(hentry->txns_using_rel == 0 && InvalidTransactionId==hentry->staleTid
+ && hentry != currentEntry)
{
if (Debug_appendonly_print_segfile_choice)
{
@@ -1721,14 +1735,14 @@ AORelFreeSegfileStatus()
}
static int
-AORelGetSegfileStatus(void)
+AORelGetSegfileStatus(AORelHashEntry currentEntry)
{
int result;
while (AppendOnlyWriter->num_existing_segfilestatus + 1 > MaxAORelSegFileStatus)
{
- bool freedOne = AORelFreeSegfileStatus();
+ bool freedOne = AORelFreeSegfileStatus(currentEntry);
if (!freedOne)
{
return NEXT_END_OF_LIST;