You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2019/03/05 18:16:01 UTC
[hadoop] branch trunk updated: HDDS-1222. Remove
TestContainerSQLCli unit test stub. Contributed by Elek, Marton.
This is an automated email from the ASF dual-hosted git repository.
arp pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 02da06d HDDS-1222. Remove TestContainerSQLCli unit test stub. Contributed by Elek, Marton.
new 549d54f Merge pull request #556 from elek/HDDS-1222
02da06d is described below
commit 02da06dd9f2d962e1db85d265b97ac3e269a60fd
Author: Márton Elek <el...@apache.org>
AuthorDate: Tue Mar 5 16:34:27 2019 +0100
HDDS-1222. Remove TestContainerSQLCli unit test stub. Contributed by Elek, Marton.
---
.../hadoop/ozone/scm/TestContainerSQLCli.java | 262 ---------------------
1 file changed, 262 deletions(-)
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
deleted file mode 100644
index bba5995..0000000
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright containerOwnership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm;
-
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.scm.container.ContainerManager;
-import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
-import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.ozone.scm.cli.SQLCLI;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.UUID;
-
-import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
-import static org.apache.hadoop.ozone.OzoneConsts.KB;
-import static org.junit.Assert.assertEquals;
-
-/**
- * This class tests the CLI that transforms container into SQLite DB files.
- */
-@RunWith(Parameterized.class)
-public class TestContainerSQLCli {
-
- private EventQueue eventQueue;
-
- @Parameterized.Parameters
- public static Collection<Object[]> data() {
- return Arrays.asList(new Object[][] {
- {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB},
- {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB}
- });
- }
-
- private static String metaStoreType;
-
- public TestContainerSQLCli(String type) {
- metaStoreType = type;
- }
-
- private static SQLCLI cli;
-
- private MiniOzoneCluster cluster;
- private OzoneConfiguration conf;
- private String datanodeIpAddress;
-
- private ContainerManager containerManager;
- private NodeManager nodeManager;
- private BlockManagerImpl blockManager;
-
- private HashMap<Long, Long> blockContainerMap;
-
- private final static long DEFAULT_BLOCK_SIZE = 4 * KB;
- private static HddsProtos.ReplicationFactor factor;
- private static HddsProtos.ReplicationType type;
- private static final String CONTAINER_OWNER = "OZONE";
-
-
- @Before
- public void setup() throws Exception {
- blockContainerMap = new HashMap<>();
-
- conf = new OzoneConfiguration();
- conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 2);
- conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
- if(conf.getBoolean(ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY,
- ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT)){
- factor = HddsProtos.ReplicationFactor.THREE;
- type = HddsProtos.ReplicationType.RATIS;
- } else {
- factor = HddsProtos.ReplicationFactor.ONE;
- type = HddsProtos.ReplicationType.STAND_ALONE;
- }
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(2).build();
- cluster.waitForClusterToBeReady();
- datanodeIpAddress = cluster.getHddsDatanodes().get(0)
- .getDatanodeDetails().getIpAddress();
- cluster.getOzoneManager().stop();
- cluster.getStorageContainerManager().stop();
- eventQueue = new EventQueue();
- nodeManager = cluster.getStorageContainerManager().getScmNodeManager();
- PipelineManager pipelineManager =
- new SCMPipelineManager(conf, nodeManager, eventQueue);
- containerManager = new SCMContainerManager(conf, nodeManager,
- pipelineManager, eventQueue);
- blockManager =
- new BlockManagerImpl(conf, cluster.getStorageContainerManager());
- eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, blockManager);
- eventQueue.fireEvent(SCMEvents.CHILL_MODE_STATUS, false);
- GenericTestUtils.waitFor(() -> {
- return !blockManager.isScmInChillMode();
- }, 10, 1000 * 15);
- // blockManager.allocateBlock() will create containers if there is none
- // stored in levelDB. The number of containers to create is the value of
- // OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT which we set to 2.
- // so the first allocateBlock() will create two containers. A random one
- // is assigned for the block.
-
- // loop until both the two datanodes are up, try up to about 4 seconds.
- for (int c = 0; c < 40; c++) {
- if (nodeManager.getAllNodes().size() == 2) {
- break;
- }
- Thread.sleep(100);
- }
- assertEquals(2, nodeManager.getAllNodes().size());
- AllocatedBlock ab1 = blockManager.allocateBlock(DEFAULT_BLOCK_SIZE, type,
- factor, CONTAINER_OWNER, new ExcludeList());
- blockContainerMap.put(ab1.getBlockID().getLocalID(),
- ab1.getBlockID().getContainerID());
-
- AllocatedBlock ab2;
- // we want the two blocks on the two provisioned containers respectively,
- // however blockManager picks containers randomly, keep retry until we
- // assign the second block to the other container. This seems to be the only
- // way to get the two containers.
- // although each retry will create a block and assign to a container. So
- // the size of blockContainerMap will vary each time the test is run.
- while (true) {
- ab2 = blockManager
- .allocateBlock(DEFAULT_BLOCK_SIZE, type, factor, CONTAINER_OWNER,
- new ExcludeList());
- blockContainerMap.put(ab2.getBlockID().getLocalID(),
- ab2.getBlockID().getContainerID());
- if (ab1.getBlockID().getContainerID() !=
- ab2.getBlockID().getContainerID()) {
- break;
- }
- }
-
- blockManager.close();
- containerManager.close();
- nodeManager.close();
-
- conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType);
- cli = new SQLCLI(conf);
-
- }
-
- @After
- public void shutdown() throws InterruptedException {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- @Test
- public void testConvertContainerDB() throws Exception {
- String dbOutPath = GenericTestUtils.getTempPath(
- UUID.randomUUID() + "/out_sql.db");
- // TODO : the following will fail due to empty Datanode list, need to fix.
- //String dnUUID = cluster.getDataNodes().get(0).getUuid();
- String dbRootPath = conf.get(HddsConfigKeys.OZONE_METADATA_DIRS);
- String dbPath = dbRootPath + "/" + SCM_CONTAINER_DB;
- String[] args = {"-p", dbPath, "-o", dbOutPath};
- Connection conn;
- String sql;
- ResultSet rs;
-
- cli.run(args);
-
- //verify the sqlite db
- // only checks the container names are as expected. Because other fields
- // such as datanode UUID are generated randomly each time
- conn = connectDB(dbOutPath);
- sql = "SELECT * FROM containerInfo";
- rs = executeQuery(conn, sql);
- ArrayList<Long> containerIDs = new ArrayList<>();
- while (rs.next()) {
- containerIDs.add(rs.getLong("containerID"));
- //assertEquals(dnUUID, rs.getString("leaderUUID"));
- }
- /* TODO: fix this later when the SQLCLI is fixed.
- assertTrue(containerIDs.size() == 2 &&
- containerIDs.contains(pipeline1.getContainerName()) &&
- containerIDs.contains(pipeline2.getContainerName()));
-
- sql = "SELECT * FROM containerMembers";
- rs = executeQuery(conn, sql);
- containerIDs = new ArrayList<>();
- while (rs.next()) {
- containerIDs.add(rs.getLong("containerID"));
- //assertEquals(dnUUID, rs.getString("datanodeUUID"));
- }
- assertTrue(containerIDs.size() == 2 &&
- containerIDs.contains(pipeline1.getContainerName()) &&
- containerIDs.contains(pipeline2.getContainerName()));
-
- sql = "SELECT * FROM datanodeInfo";
- rs = executeQuery(conn, sql);
- int count = 0;
- while (rs.next()) {
- assertEquals(datanodeIpAddress, rs.getString("ipAddress"));
- //assertEquals(dnUUID, rs.getString("datanodeUUID"));
- count += 1;
- }
- // the two containers maybe on the same datanode, maybe not.
- int expected = pipeline1.getLeader().getUuid().equals(
- pipeline2.getLeader().getUuid())? 1 : 2;
- assertEquals(expected, count);
- */
- Files.delete(Paths.get(dbOutPath));
- }
-
- private ResultSet executeQuery(Connection conn, String sql)
- throws SQLException {
- Statement stmt = conn.createStatement();
- return stmt.executeQuery(sql);
- }
-
- private Connection connectDB(String dbPath) throws Exception {
- Class.forName("org.sqlite.JDBC");
- String connectPath =
- String.format("jdbc:sqlite:%s", dbPath);
- return DriverManager.getConnection(connectPath);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org