You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hbase.apache.org by GitBox <gi...@apache.org> on 2020/05/20 19:04:16 UTC
[GitHub] [hbase] saintstack commented on a change in pull request #1746: HBASE-24388 Introduce a 'local root region' at master side to store t…
saintstack commented on a change in pull request #1746:
URL: https://github.com/apache/hbase/pull/1746#discussion_r428196462
##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
##########
@@ -866,6 +877,50 @@ protected AssignmentManager createAssignmentManager(MasterServices master) {
return new AssignmentManager(master);
}
+ private void createRootTable() throws IOException, KeeperException {
+ RootTable rootTable = new RootTable(this, cleanerPool);
+ rootTable.initialize();
+ // try migrate data from zookeeper
+ try (RegionScanner scanner =
+ rootTable.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
+ List<Cell> cells = new ArrayList<>();
+ boolean moreRows = scanner.next(cells);
+ if (!cells.isEmpty() || moreRows) {
+ // notice that all replicas for a region are in the same row, so the migration can be
+ // done with in a one row put, which means if we have data in root table then we can make
+ // sure that the migration is done.
Review comment:
Good
##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
##########
@@ -866,6 +877,50 @@ protected AssignmentManager createAssignmentManager(MasterServices master) {
return new AssignmentManager(master);
}
+ private void createRootTable() throws IOException, KeeperException {
Review comment:
Would be cool if this was not inline in HMaster class. It too big as it is.
##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
##########
@@ -866,6 +877,50 @@ protected AssignmentManager createAssignmentManager(MasterServices master) {
return new AssignmentManager(master);
}
+ private void createRootTable() throws IOException, KeeperException {
+ RootTable rootTable = new RootTable(this, cleanerPool);
+ rootTable.initialize();
+ // try migrate data from zookeeper
+ try (RegionScanner scanner =
+ rootTable.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
+ List<Cell> cells = new ArrayList<>();
+ boolean moreRows = scanner.next(cells);
+ if (!cells.isEmpty() || moreRows) {
+ // notice that all replicas for a region are in the same row, so the migration can be
+ // done with in a one row put, which means if we have data in root table then we can make
+ // sure that the migration is done.
+ LOG.info("Root table already has data in it, skip migrating...");
+ this.rootTable = rootTable;
+ return;
+ }
+ }
+ // start migrating
+ byte[] row = MetaTableAccessor.getMetaKeyForRegion(RegionInfoBuilder.FIRST_META_REGIONINFO);
+ Put put = new Put(row);
+ List<String> metaReplicaNodes = zooKeeper.getMetaReplicaNodes();
+ StringBuilder info = new StringBuilder("Migrating meta location:");
+ for (String metaReplicaNode : metaReplicaNodes) {
+ int replicaId = zooKeeper.getZNodePaths().getMetaReplicaIdFromZnode(metaReplicaNode);
+ RegionState state = MetaTableLocator.getMetaRegionState(zooKeeper, replicaId);
+ info.append(" ").append(state);
+ put.setTimestamp(state.getStamp());
+ MetaTableAccessor.addRegionInfo(put, state.getRegion());
+ if (state.getServerName() != null) {
+ MetaTableAccessor.addLocation(put, state.getServerName(), HConstants.NO_SEQNUM, replicaId);
+ }
+ put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(put.getRow())
+ .setFamily(HConstants.CATALOG_FAMILY)
+ .setQualifier(RegionStateStore.getStateColumn(replicaId)).setTimestamp(put.getTimestamp())
+ .setType(Cell.Type.Put).setValue(Bytes.toBytes(state.getState().name())).build());
+ }
+ if (!put.isEmpty()) {
+ LOG.info(info.toString());
+ } else {
+ LOG.info("No meta location avaiable on zookeeper, skip migrating...");
+ }
+ this.rootTable = rootTable;
+ }
+
Review comment:
good
##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/region/LocalRegion.java
##########
@@ -0,0 +1,328 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.region;
+
+import static org.apache.hadoop.hbase.HConstants.HREGION_LOGDIR_NAME;
+
+import java.io.IOException;
+import java.util.Collections;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegion.FlushResult;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.util.RecoverLeaseFSUtils;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hbase.thirdparty.com.google.common.math.IntMath;
+
+/**
+ * A region that stores data in a separated directory on WAL file system.
Review comment:
We still need to do this snowflaking?
##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
##########
@@ -866,6 +877,50 @@ protected AssignmentManager createAssignmentManager(MasterServices master) {
return new AssignmentManager(master);
}
+ private void createRootTable() throws IOException, KeeperException {
+ RootTable rootTable = new RootTable(this, cleanerPool);
+ rootTable.initialize();
+ // try migrate data from zookeeper
+ try (RegionScanner scanner =
+ rootTable.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY))) {
+ List<Cell> cells = new ArrayList<>();
+ boolean moreRows = scanner.next(cells);
+ if (!cells.isEmpty() || moreRows) {
+ // notice that all replicas for a region are in the same row, so the migration can be
+ // done with in a one row put, which means if we have data in root table then we can make
+ // sure that the migration is done.
+ LOG.info("Root table already has data in it, skip migrating...");
+ this.rootTable = rootTable;
+ return;
+ }
+ }
+ // start migrating
+ byte[] row = MetaTableAccessor.getMetaKeyForRegion(RegionInfoBuilder.FIRST_META_REGIONINFO);
Review comment:
So, this row key is just serialized meta region name?
##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
##########
@@ -552,4 +553,6 @@ default SplitWALManager getSplitWALManager(){
* @return The state of the load balancer, or false if the load balancer isn't defined.
*/
boolean isBalancerOn();
+
+ RootTable getRootTable();
Review comment:
It can't be kept totally internal?
##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
##########
@@ -552,4 +553,6 @@ default SplitWALManager getSplitWALManager(){
* @return The state of the load balancer, or false if the load balancer isn't defined.
*/
boolean isBalancerOn();
+
+ RootTable getRootTable();
Review comment:
I suppose AM gets it this way or some other internal services?
##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/master/root/RootTable.java
##########
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.root;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.master.cleaner.DirScanPool;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
+import org.apache.hadoop.hbase.region.LocalRegion;
+import org.apache.hadoop.hbase.region.LocalRegionParams;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Used to store the location of meta region.
+ */
+@InterfaceAudience.Private
+public class RootTable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RootTable.class);
+
+ static final String MAX_WALS_KEY = "hbase.root.table.region.maxwals";
+
+ private static final int DEFAULT_MAX_WALS = 10;
+
+ static final String USE_HSYNC_KEY = "hbase.root.table.region.wal.hsync";
+
+ static final String RING_BUFFER_SLOT_COUNT = "hbase.root.table.region.maxwals";
+
+ private static final int DEFAULT_RING_BUFFER_SLOT_COUNT = 64;
+
+ static final String ROOT_TABLE_DIR = "RootTable";
+
+ static final String HFILECLEANER_PLUGINS = "hbase.root.table.region.hfilecleaner.plugins";
+
+ static final String FLUSH_SIZE_KEY = "hbase.root.table.region.flush.size";
+
+ static final long DEFAULT_FLUSH_SIZE = TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE;
+
+ static final String FLUSH_PER_CHANGES_KEY = "hbase.root.table.region.flush.per.changes";
+
+ private static final long DEFAULT_FLUSH_PER_CHANGES = 1_000_000;
+
+ static final String FLUSH_INTERVAL_MS_KEY = "hbase.root.table.region.flush.interval.ms";
+
+ // default to flush every 15 minutes, for safety
+ private static final long DEFAULT_FLUSH_INTERVAL_MS = TimeUnit.MINUTES.toMillis(15);
+
+ static final String COMPACT_MIN_KEY = "hbase.root.table.region.compact.min";
+
+ private static final int DEFAULT_COMPACT_MIN = 4;
+
+ static final String ROLL_PERIOD_MS_KEY = "hbase.root.table.region.walroll.period.ms";
+
+ private static final long DEFAULT_ROLL_PERIOD_MS = TimeUnit.MINUTES.toMillis(15);
+
+ static final TableName TABLE_NAME = TableName.valueOf("master:root");
+
+ private static final TableDescriptor TABLE_DESC = TableDescriptorBuilder.newBuilder(TABLE_NAME)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY)
+ .setMaxVersions(HConstants.DEFAULT_HBASE_META_VERSIONS).setInMemory(true)
+ .setBlocksize(HConstants.DEFAULT_HBASE_META_BLOCK_SIZE).setBloomFilterType(BloomType.ROWCOL)
+ .setDataBlockEncoding(DataBlockEncoding.ROW_INDEX_V1).build())
+ .build();
+
+ private final Server server;
+
+ private final DirScanPool cleanerPool;
+
+ private LocalRegion region;
+
+ public RootTable(Server server, DirScanPool cleanerPool) {
+ this.server = server;
+ this.cleanerPool = cleanerPool;
+ }
+
+ public void initialize() throws IOException {
+ LOG.info("Initializing root table...");
+ LocalRegionParams params = new LocalRegionParams().server(server).regionDirName(ROOT_TABLE_DIR)
Review comment:
Is this at ${root.dir}/RootTable?
##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
##########
@@ -216,12 +196,32 @@ private void updateUserRegionLocation(RegionInfo regionInfo, State state,
.build());
LOG.info(info.toString());
updateRegionLocation(regionInfo, state, put);
+ if (regionInfo.isMetaRegion() && regionInfo.isFirst()) {
+ // mirror the meta location to
+ mirrorMetaLocation(regionInfo, regionLocation, state);
+ }
}
- private void updateRegionLocation(RegionInfo regionInfo, State state, Put put)
+ public void mirrorMetaLocation(RegionInfo regionInfo, ServerName serverName, State state)
throws IOException {
- try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) {
- table.put(put);
+ try {
+ MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, regionInfo.getReplicaId(),
+ state);
+ } catch (KeeperException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private void updateRegionLocation(RegionInfo regionInfo, State state, Put put)
+ throws IOException {
+ try {
+ if (regionInfo.isMetaRegion()) {
+ master.getRootTable().update(put);
Review comment:
Nice
##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/master/root/RootTable.java
##########
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.root;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.master.cleaner.DirScanPool;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
+import org.apache.hadoop.hbase.region.LocalRegion;
+import org.apache.hadoop.hbase.region.LocalRegionParams;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Used to store the location of meta region.
+ */
+@InterfaceAudience.Private
+public class RootTable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RootTable.class);
+
+ static final String MAX_WALS_KEY = "hbase.root.table.region.maxwals";
+
+ private static final int DEFAULT_MAX_WALS = 10;
+
+ static final String USE_HSYNC_KEY = "hbase.root.table.region.wal.hsync";
+
+ static final String RING_BUFFER_SLOT_COUNT = "hbase.root.table.region.maxwals";
+
+ private static final int DEFAULT_RING_BUFFER_SLOT_COUNT = 64;
+
+ static final String ROOT_TABLE_DIR = "RootTable";
+
+ static final String HFILECLEANER_PLUGINS = "hbase.root.table.region.hfilecleaner.plugins";
+
+ static final String FLUSH_SIZE_KEY = "hbase.root.table.region.flush.size";
+
+ static final long DEFAULT_FLUSH_SIZE = TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE;
+
+ static final String FLUSH_PER_CHANGES_KEY = "hbase.root.table.region.flush.per.changes";
+
+ private static final long DEFAULT_FLUSH_PER_CHANGES = 1_000_000;
+
+ static final String FLUSH_INTERVAL_MS_KEY = "hbase.root.table.region.flush.interval.ms";
+
+ // default to flush every 15 minutes, for safety
+ private static final long DEFAULT_FLUSH_INTERVAL_MS = TimeUnit.MINUTES.toMillis(15);
+
+ static final String COMPACT_MIN_KEY = "hbase.root.table.region.compact.min";
+
+ private static final int DEFAULT_COMPACT_MIN = 4;
+
+ static final String ROLL_PERIOD_MS_KEY = "hbase.root.table.region.walroll.period.ms";
+
+ private static final long DEFAULT_ROLL_PERIOD_MS = TimeUnit.MINUTES.toMillis(15);
+
+ static final TableName TABLE_NAME = TableName.valueOf("master:root");
+
+ private static final TableDescriptor TABLE_DESC = TableDescriptorBuilder.newBuilder(TABLE_NAME)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY)
+ .setMaxVersions(HConstants.DEFAULT_HBASE_META_VERSIONS).setInMemory(true)
+ .setBlocksize(HConstants.DEFAULT_HBASE_META_BLOCK_SIZE).setBloomFilterType(BloomType.ROWCOL)
+ .setDataBlockEncoding(DataBlockEncoding.ROW_INDEX_V1).build())
+ .build();
+
+ private final Server server;
+
+ private final DirScanPool cleanerPool;
+
+ private LocalRegion region;
+
+ public RootTable(Server server, DirScanPool cleanerPool) {
+ this.server = server;
+ this.cleanerPool = cleanerPool;
+ }
+
+ public void initialize() throws IOException {
+ LOG.info("Initializing root table...");
+ LocalRegionParams params = new LocalRegionParams().server(server).regionDirName(ROOT_TABLE_DIR)
+ .tableDescriptor(TABLE_DESC);
+ Configuration conf = server.getConfiguration();
+ long flushSize = conf.getLong(FLUSH_SIZE_KEY, DEFAULT_FLUSH_SIZE);
+ long flushPerChanges = conf.getLong(FLUSH_PER_CHANGES_KEY, DEFAULT_FLUSH_PER_CHANGES);
+ long flushIntervalMs = conf.getLong(FLUSH_INTERVAL_MS_KEY, DEFAULT_FLUSH_INTERVAL_MS);
+ int compactMin = conf.getInt(COMPACT_MIN_KEY, DEFAULT_COMPACT_MIN);
+ params.flushSize(flushSize).flushPerChanges(flushPerChanges).flushIntervalMs(flushIntervalMs)
+ .compactMin(compactMin);
+ int maxWals = conf.getInt(MAX_WALS_KEY, DEFAULT_MAX_WALS);
+ params.maxWals(maxWals);
+ if (conf.get(USE_HSYNC_KEY) != null) {
+ params.useHsync(conf.getBoolean(USE_HSYNC_KEY, false));
+ }
+ params.ringBufferSlotCount(conf.getInt(RING_BUFFER_SLOT_COUNT, DEFAULT_RING_BUFFER_SLOT_COUNT));
+ long rollPeriodMs = conf.getLong(ROLL_PERIOD_MS_KEY, DEFAULT_ROLL_PERIOD_MS);
+ params.rollPeriodMs(rollPeriodMs)
+ .archivedWalSuffix(MasterProcedureUtil.ARCHIVED_PROC_WAL_SUFFIX)
+ .hfileCleanerPlugins(HFILECLEANER_PLUGINS).cleanerPool(cleanerPool);
+ region = LocalRegion.create(params);
+ }
+
+ public void update(Put put) throws IOException {
+ region.update(r -> r.put(put));
+ }
+
+ public void delete(Delete delete) throws IOException {
+ region.update(r -> r.delete(delete));
+ }
+
+ public RegionScanner getScanner(Scan scan) throws IOException {
+ return region.getScanner(scan);
+ }
+
+ public void close(boolean abort) {
+ LOG.info("Closing root table, isAbort={}", abort);
+ if (region != null) {
+ region.close(abort);
+ }
+ }
+}
Review comment:
It'd have its own Region?
Seems overkill having a full Region just for meta locations?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org