You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by ml...@apache.org on 2016/11/21 10:03:05 UTC
incubator-hawq git commit: HAWQ-1149. Fixed bug
gp_persistent_build_all loses data in
gp_relfile_node&gp_persistent_relfile_node
Repository: incubator-hawq
Updated Branches:
refs/heads/master 833c2f97c -> 38b447d7e
HAWQ-1149. Fixed bug gp_persistent_build_all loses data in gp_relfile_node&gp_persistent_relfile_node
Internally there are 3 bugs:
1) hawq2.0 changed the file path of relation file on hdfs from filespace/db/relfile.filenum to filespace/db/table/relfile/filenum. When we scan relfile on hdfs, we should change scan logic correspondingly.
2) Fetched dummy persistentTid & persistentSerialNum to PersistentRelation_MarkCreatePending() .
3) Need to reset Relation->rd_relationnodeinfo.isPresent, so that next time persistentid and serial# can be refetched during PersistentBuild_BuildDb().
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/38b447d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/38b447d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/38b447d7
Branch: refs/heads/master
Commit: 38b447d7e2a6a176bb41eefa7643ce14605a3f0b
Parents: 833c2f9
Author: Ming LI <ml...@apache.org>
Authored: Mon Nov 14 16:21:46 2016 +0800
Committer: Ming LI <ml...@apache.org>
Committed: Mon Nov 21 16:30:58 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/cdbdatabaseinfo.c | 73 ++++++++++++++++++++++------
src/backend/cdb/cdbpersistentbuild.c | 6 +++
src/backend/cdb/cdbpersistentrelation.c | 17 ++++---
src/include/cdb/cdbpersistentrelation.h | 1 +
4 files changed, 75 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/backend/cdb/cdbdatabaseinfo.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatabaseinfo.c b/src/backend/cdb/cdbdatabaseinfo.c
index 062c4d3..4e0be41 100644
--- a/src/backend/cdb/cdbdatabaseinfo.c
+++ b/src/backend/cdb/cdbdatabaseinfo.c
@@ -49,7 +49,6 @@
#include "cdb/cdbpersistentfilesysobj.h"
#include "commands/dbcommands.h"
-
/*-------------------------------------------------------------------------
* Local static type declarations
*------------------------------------------------------------------------- */
@@ -753,7 +752,9 @@ static void DatabaseInfo_AddFile(
HTAB *dbInfoRelHashTable,
Oid tablespace,
char *dbDirPath,
- char *name)
+ char *name,
+ bool isHdfs,
+ Oid table_oid)
{
int64 eof;
int itemCount;
@@ -787,7 +788,7 @@ static void DatabaseInfo_AddFile(
}
FileClose(file);
- itemCount = sscanf(name, "%u.%u", &relfilenode, &segmentFileNum);
+ itemCount = sscanf(name, isHdfs?"%u/%u":"%u.%u", &relfilenode, &segmentFileNum);
// UNDONE: sscanf is a rather poor scanner.
// UNDONE: For right now, just assume properly named files....
@@ -796,12 +797,16 @@ static void DatabaseInfo_AddFile(
DatabaseInfo_AddMiscEntry(info, tablespace, false, name);
return;
}
- else if (itemCount == 1)
+ else if (itemCount == 1 && !isHdfs)
segmentFileNum = 0;
else
Assert(itemCount == 2);
- DatabaseInfo_AddRelSegFile(info, dbInfoRelHashTable, tablespace, relfilenode, segmentFileNum, eof);
+ if(isHdfs){
+ DatabaseInfo_AddRelSegFile(info, dbInfoRelHashTable, tablespace, table_oid, relfilenode, eof);
+ }else{
+ DatabaseInfo_AddRelSegFile(info, dbInfoRelHashTable, tablespace, relfilenode, segmentFileNum, eof);
+ }
}
/*
@@ -818,9 +823,9 @@ DatabaseInfo_Scan(
Oid database)
{
char *dbDirPath;
- DIR *xldir;
- struct dirent *xlde;
- char fromfile[MAXPGPATH];
+ DIR *xldir, *xldir1;
+ struct dirent *xlde,*xlde1;
+ char fromfile[MAXPGPATH],fromfile1[MAXPGPATH];
/* Lookup the database path and allocate a directory scan structure */
dbDirPath = GetDatabasePath(
@@ -869,7 +874,9 @@ DatabaseInfo_Scan(
dbInfoRelHashTable,
tablespace,
dbDirPath,
- xlde->d_name);
+ xlde->d_name,
+ false,
+ InvalidOid);
}
}else{//on hdfs
int ret = HdfsIsDirOrFile(fromfile);
@@ -878,18 +885,52 @@ DatabaseInfo_Scan(
(errcode_for_file_access(),
errmsg("error to fetch info for path \"%s\": %m", fromfile)));
}else if(ret == 0){//direcotry
- DatabaseInfo_AddMiscEntry(
+ if(strcmp(xlde->d_name, dbDirPath) == 0){ //database level
+ DatabaseInfo_AddMiscEntry(
info,
tablespace,
/* isDir */ true,
xlde->d_name);
+ }else{ //table level, need to recursive list file in this directory
+ xldir1 = AllocateDir(fromfile);
+ if (xldir1 == NULL)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("Could not open database directory \"%s\": %m", fromfile)));
+ while ((xlde1 = ReadDir(xldir1, fromfile)) != NULL)
+ {
+ if (strcmp(xlde1->d_name, ".") == 0 ||
+ strcmp(xlde1->d_name, "..") == 0)
+ continue;
+
+ /* Odd... On snow leopard, we get back "/" as a subdir, which is wrong. Ingore it */
+ if (xlde1->d_name[0] == '/' && xlde1->d_name[1] == '\0')
+ continue;
+
+ snprintf(fromfile1, MAXPGPATH, "%s/%s", fromfile, xlde1->d_name);
+
+ if(!IsLocalPath(fromfile1)){//on hdfs
+ int ret = HdfsIsDirOrFile(fromfile1);
+ if(ret != 1){//fetch info error
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("error to fetch file path \"%s\": %m", fromfile1)));
+ }else{
+ DatabaseInfo_AddFile(
+ info,
+ dbInfoRelHashTable,
+ tablespace,
+ fromfile,
+ xlde1->d_name,
+ true,
+ pg_atoi(xlde->d_name, 4, 0));
+ }
+ }
+ }
+ FreeDir(xldir1);
+ }
}else if(ret == 1){//file
- DatabaseInfo_AddFile(
- info,
- dbInfoRelHashTable,
- tablespace,
- dbDirPath,
- xlde->d_name);
+ //just skip for PG_VERSION file
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/backend/cdb/cdbpersistentbuild.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbpersistentbuild.c b/src/backend/cdb/cdbpersistentbuild.c
index 0033769..a12bc3b 100644
--- a/src/backend/cdb/cdbpersistentbuild.c
+++ b/src/backend/cdb/cdbpersistentbuild.c
@@ -222,6 +222,7 @@ static void PersistentBuild_PopulateGpRelationNode(
ItemPointerData persistentTid;
int64 persistentSerialNum;
+ Relation rd;
if (dbInfoRel->reltablespace == GLOBALTABLESPACE_OID &&
info->database != TemplateDbOid)
@@ -417,6 +418,11 @@ static void PersistentBuild_PopulateGpRelationNode(
persistentSerialNum);
}
}
+ // reset Relation->rd_relationnodeinfo.isPresent, so that next time persistentid and serial# can be refetched
+ rd = RelationIdGetRelation(relFileNode.relNode);
+ rd->rd_relationnodeinfo.isPresent = false;
+ RelationClose(rd);
+
(*count)++;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/backend/cdb/cdbpersistentrelation.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbpersistentrelation.c b/src/backend/cdb/cdbpersistentrelation.c
index e7fd7ed..14a2cbc 100644
--- a/src/backend/cdb/cdbpersistentrelation.c
+++ b/src/backend/cdb/cdbpersistentrelation.c
@@ -629,7 +629,7 @@ void PersistentRelation_MarkCreatePending(
relFileNode->spcNode,
relFileNode->dbNode,
relFileNode->relNode,
- persistentTid, persistentSerialNum);
+ persistentTid, *persistentSerialNum);
#endif
/*
@@ -648,14 +648,16 @@ void PersistentRelation_MarkCreatePending(
void PersistentRelation_AddCreated(
RelFileNode *relFileNode,
ItemPointer persistentTid,
+ int64 *persistentSerialNum,
bool flushToXLog)
{
WRITE_PERSISTENT_STATE_ORDERED_LOCK_DECLARE;
PersistentFileSysObjName fsObjName;
- ItemPointerData previousFreeTid;
- int64 persistentSerialNum;
+
RelationDirEntry relationDirEntry;
+
+ ItemPointerData previousFreeTid;
Datum values[Natts_gp_persistent_relation_node];
if (RelFileNode_IsEmpty(relFileNode))
@@ -711,8 +713,11 @@ void PersistentRelation_AddCreated(
PersistentFsObjType_RelationDir,
values,
flushToXLog,
- persistentTid,
- &persistentSerialNum);
+ &relationDirEntry->persistentTid,
+ &relationDirEntry->persistentSerialNum);
+
+ *persistentTid = relationDirEntry->persistentTid;
+ *persistentSerialNum = relationDirEntry->persistentSerialNum;
WRITE_PERSISTENT_STATE_ORDERED_UNLOCK;
@@ -721,7 +726,7 @@ void PersistentRelation_AddCreated(
elog(Persistent_DebugPrintLevel(),
"Persistent relation: Add '%s', in state 'Created', serial number " INT64_FORMAT " at TID %s",
PersistentFileSysObjName_ObjectName(&fsObjName),
- persistentSerialNum,
+ *persistentSerialNum,
ItemPointerToString(persistentTid));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/include/cdb/cdbpersistentrelation.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbpersistentrelation.h b/src/include/cdb/cdbpersistentrelation.h
index 292aaba..9f4e184 100644
--- a/src/include/cdb/cdbpersistentrelation.h
+++ b/src/include/cdb/cdbpersistentrelation.h
@@ -68,6 +68,7 @@ extern void PersistentRelation_AddCreated(
RelFileNode *relFileNode,
/* The tablespace, database, and relation OIDs for the create. */
ItemPointer persistentTid,
+ int64 *persistentSerialNum,
bool flushToXLog);
// -----------------------------------------------------------------------------