You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hive.apache.org by "Sylwester Lachiewicz (Jira)" <ji...@apache.org> on 2022/05/28 10:29:00 UTC
[jira] [Resolved] (HIVE-25647) hadoop memo
[ https://issues.apache.org/jira/browse/HIVE-25647?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Sylwester Lachiewicz resolved HIVE-25647.
-----------------------------------------
Resolution: Invalid
> hadoop memo
> -----------
>
> Key: HIVE-25647
> URL: https://issues.apache.org/jira/browse/HIVE-25647
> Project: Hive
> Issue Type: Wish
> Components: Configuration
> Affects Versions: 3.1.2
> Environment: hadoop 2.7.3
> Reporter: St Li
> Assignee: St Li
> Priority: Major
> Fix For: All Versions
>
> Attachments: worldip.csv
>
>
> master,slave1,slave2master,slave1,slave2//opt represent wechat hadoop bigdata dev//2019 :bigdata competitionhadoop 50070hbase 16010storm 8080
> #hostnamehostnamectl set-hostname master && bash hostname master && bash hostname slave1/slave2 && bash vim /etc/hostname master/slave1/slave2vim /etc/hosts ip master ip slave1 ipslave2
> #yumcd /etc/yum.repos.d && rm -rf *wget http://172.16.47.240/bigdata/repofile/bigdata.repoyum clean all
> #firewallsystemctl stop firewalldsystemctl status firewalld
> #timezonetzselect 5-9-1-1echo "TZ='Asia/Shanghai'; export TZ" >> /etc/profile && source /etc/profile
> #ntpyum install -y ntpvim /etc/ntp.conf//#server 0~3.centos.pool.ntp.org iburstserver 127.127.1.0fudge 127.127.1.0 stratum 10/bin/systemctl restart ntpd.servicentpdate master (slave1,slave2)
> #crontabservice crond status/sbin/service crond startcrontab -e*/30 8-17 * * * /usr/sbin/ntpdate mastercrontab –l
> #ssh passwordssh-keygen -t dsa -P '' -f ~/.ssh/id_dsacat /root/.ssh/id_dsa.pub >> /root/.ssh/authorized_keysscp ~/.ssh/authorized_keys root@slave1:~/.ssh/scp ~/.ssh/authorized_keys root@slave2:~/.ssh/
> ssh-copy-id masterssh-copy-id slave1ssh-copy-id slave2
> #install jdkmkdir -p /usr/javatar -zxvf jdk-8u171-linux-x64.tar.gz -C /usr/java/
> vim /etc/profileexport JAVA_HOME=/usr/java/jdk1.8.0_171export CLASSPATH=$JAVA_HOME/lib/export PATH=$PATH:$JAVA_HOME/bin
> source /etc/profile && java -version
> scp -r /usr/java root@slave1:/usr/scp -r /usr/java root@slave2:/usr/
> #install hadoopmkdir -p /usr/hadoop && cd /usr/hadooptar -zxvf /usr/hadoop/hadoop-2.7.3.tar.gz -C /usr/hadoop/rm -rf /usr/hadoop/hadoop-2.7.3.tar.gzvim /etc/profileexport HADOOP_HOME=/usr/hadoop/hadoop-2.7.3export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbinhadoop //test
> hadoop-env.sh/mapred-env.sh/yarn-en.shexport JAVA_HOME=/usr/java/jdk1.8.0_171
> ##vim core-site.xmlfs.default.name \{hdfs://master:9000}hadoop.tmp.dir \{/usr/hadoop/hadoop-2.7.3/hdfs/tmp}io.file.buffer.size \{131072}fs.checkpoint.period \{60}fs.checkpoint.size \{67108864}
> ##hdfs-site.xmldfs.replication \{2}dfs.namenode.name.dir \{file:/usr/hadoop/hadoop-2.7.3/hdfs/name}dfs.datanode.data.dir \{file:/usr/hadoop/hadoop-2.7.3/hdfs/data}
> ##vim yarn-env.shyarn.resourcemanager.address \{master:18040}yarn.resourcemanager.scheduler.address \{master:18030}yarn.resourcemanager.webapp.address \{master:18088}yarn.resourcemanager.resource-tracker.address \{18025}yarn.resourcemanager.admin.address \{master:18141}yarn.nodemanager.aux-services \{mapreduce_shuffle}yarn.nodemanager.auxservices.mapreduce.shuffle.class \{org.apache.hadoop.mapred.ShuffleHandler}
> #vim mapred-site.xmlmapreduce.framework.name \{yarn}
> #slaves fileecho master > master && echo slave1 > slaves && echo slave2 >> slaves
> #hadoop formathadoop namenode -format (master) //has been successfully#start hadoopstart-all.shmaster :NameNode,SecondaryNameNode,ResourceManagerslave1~2:DataNode,NodeManager
> start-dfs.shstart-yarn.shhadoop-daemon.sh start namenodehadoop-daemon.sh start datanodehadoop-daemon.sh start secondarynamenodehadoop-daemon.sh start resourcemanagerhadoop-daemon.sh start nodemanager
> test hdfs& mapreducehadoop fs -mkdir /inputhadoop fs -put $HADOOP_HOME/README.txt /input
> http://master:50070hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.8.4.jar
> #install hiveyum -y install mysql-community-server slave2:mysqlserver slave1:hiveserver master:hiveclientsystemctl daemon-reloadsystemctl start mysqldcat /var/log/mysqld.log grep "temporary password"mysql -uroot -pset global validate_password_policy=0;set global validate_password_length=4;alter user 'root'@'localhost' identified by '123456';mysql -uroot -p123456create user 'root'@'%' identified by '123456';grant all privileges on *.* to 'root'@'%' with grant option;flush privileges;
> mkdir -p /usr/hive tar -zxvf /usr/hive/apache-hive-2.1.1-bin.tar.gz -C /usr/hive/
> vim /etc/profile //for hiveexport HIVE_HOME=/usr/hive/apache-hive-2.1.1-binexport PATH=$PATH:$HIVE_HOME/binsource /etc/profile
> vim hive-env.shcd $HIVE_HOME/conf && vim hive-env.shexport HADOOP_HOME=/usr/hadoop/hadoop-2.7.3export HIVE_CONF_DIR=/usr/hive/apache-hive-2.1.1-bin/confexport HIVE_AUX_JARS_PATH=/usr/hive/apache-hive-2.1.1-bin/lib
> cp $HIVE_HOME/lib/jline-2.12.jar $HADOOP_HOME/share/hadoop/yarn/lib/
> ##slave1 hive-servercd $HIVE_HOME/lib && wget or cp mysql-connector-java-5.1.47-bin.jar
> hive-site.xml (hive-server)hive.metastore.warehouse.dir \{/user/hive_remote/warehouse}javax.jdo.option.ConnectionDriverName \{com.mysql.jdbc.Driver}javax.jdo.option.ConnectionURL \{jdbc:mysql://slave2:3306/hive?createDatabaseIfNotExist=true&useSSL=false}javax.jdo.option.ConnectionUserName \{root}javax.jdo.option.ConnectionPassword \{123456}
> hive-site.xml (hive client)hive.metastore.warehouse.dir \{/user/hive_remote/warehouse}hive.metastore.local \{false}hive.metastore.uris \{thrift://slave1:9083}
> //opt hadoop fs -mkdir /tmp hadoop fs -mkdir -p /user/hive/warehousehadoop fs -chmod g+w /tmp& /user/hive/warehouse//opt modify log locationvim hive-log4j2.propertiesproperty.hive.log.dir=/home/hadoop
> //opt hive-site.xmlhive.cli.print.current.db \{true}hive.cli.print.header \{true}
> beeline connect shoud config core-site.xmlhadoop.proxyuser.hadoop.hosts \{*}..groups \{*}bin/beeline->!connect jdbc:hive2://centos01:10000or bin/beeline -u jdbc:hive2://centos01:10000 -n hadoop
> desc table/desc database/desc database extended testdb;
> hive-serverschematool -dbType mysql -initSchemahive --service metastore
> hive-clienthive \ncreate database hongyaa;
> #install zookeepermkdir -p /usr/zookeeper && cd /usr/zookeepertar -zxvf /usr/zookeeper/zookeeper-3.4.10.tar.gz -C /usr/zookeeper/vi /etc/profile //for zookeeperexport ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10PATH=$PATH:$ZOOKEEPER_HOME/binsource /etc/profile
> cd /usr/zookeeper/zookeeper-3.4.10/conf/ && mv zoo_sample.cfg zoo.cfg && vimzoo.cfgtickTime=2000initLimit=10syncLimit=5dataDir=/usr/zookeeper/zookeeper-3.4.10/zkdataclientPort=2181dataLogDir=/usr/zookeeper/zookeeper-3.4.10/zkdatalogserver.1=master:2888:3888server.2=slave1:2888:3888server.3=slave2:2888:3888
> cd /usr/zookeeper/zookeeper-3.4.10 && mkdir zkdata zkdatalogcd /usr/zookeeper/zookeeper-3.4.10/zkdata && echo "1" >> myid
> zkServer.sh start(master,slave1,slave2)netstat -antp|grep 2181
> #install hbase//opttar -zxf hbase-1.2.6.1-bin.tar.gz -C /opt/modules/export JAVA_HOME=/usr/java/jdk1.8.0_171export HBASE_MANAGES_ZK=true
> vim hbase-site.xmlhbase.rootdir \{hdfs://centos01:9000}hbase.cluster.distributed \{true}hbase.zookeeper.quorum \{centos01,centos02,centos03}hbase.zookeeper.property.dataDir \{/opt/modules/hbase-1.2.6/zkData}
> #install kafkatar -zxvf kafka_2.11-2.0.0.tgz -C /opt/modulesvim config/server.propertiesbroker.id=1num.partitions=2default.replication.factor=1lisheners=PLAINTEXT://centos01:9092log.dirs=/opt/modules/kafka_2.11-2.0.0/kafka-logszookeeper.connect=centos01:2181,centos02:2181,centos03:2181scp to centos02,centos03 and modify id
> bin/kafka-server-start.sh -daemon config/server.properties/kafka-server-start.sh -daemon config/server.properties
> create topicbin/kafka-topics.sh \--create \--zookeeper centos01:2181,centos02:2181,centos03:2181 \--replication-factor 2 \--partitions 2 \--topic topictest
> bin/kafka-topics.sh \--list \--zookeeper centos01:2181
> bin/kafka-topics.sh \--describe \--zookeeper centos01:2181//生产者bin/kafka-console-producer.sh \--broker-list centos01:9092,centos02:9092,centos03:9092 \--topic topictest//消费者bin/kafka-console-consumer.sh \--bootstrap-server centos01:9092,centos02:9092,centos03:9092 \--topic topictest
> java -Xms256m -Xmx512m -classpath .:./lib/* com.cmcc.hsd.loader.ScannerV2p9Loader -t 6 -d 202008250000
> #install stormtar zxvf apache-storm-1.1.0.tar.gz -C /opt/modulesmv apache-storm-1.1.0/ storm-1.1.0vim /etc/profileexport STORM_HOME=/opt/modules/storm-1.1.0export PATH=$PATH:$STORM_HOME/bin
> storm-env.shexport JAVA_HOME=/opt/softwares/jdk1.8.0_201export STORM_CONF_DIR="/opt/modules/storm-1.1.0/conf"vim storm.yaml storm.zookeeper.servers: - "centos01" - "centos02" - "centos03" supervisor.slots.ports: - 6700 - 6701 - 6702 - 6703 storm.zookeeper.port: 2181 storm.local.dir: "/opt/modules/storm-1.1.0/data" scp -r to centos01,centos02,centos03 storm nimbus > /dev/null 2>&1 & (master)storm ui > /dev/null 2>&1 & storm supervisor > /dev/null 2>&1 & (slave)
> jpsnimbus/Supervisor
> #install sparktar zxvf spark-2.4.0-bin-hadoop2.7.tgz -C /opt/modulesvim conf/slavescentos02centos03vim spark-env.shexport JAVA_HOME=/opt/softwares/jdk1.8.0_201export SPARK_MASTER_IP=centos01export SPARK_MASTER_PORT=7077
> export HADOOP_HOME=/opt/modules/hadoop-2.8.4export HADOOP_CONF_DIR=$HADOOP_HOME/etc/haoop
> scp -r to other node
> spark ha delete SPARK_MASTER_IP add export SPARK_DAEMON_JAVA_OPT="-Dspark.deploy.recoveryMode=ZOOKEEPER-Dspark.deploy.zookeeper.url=centos01:2181,centos02:2181,centos03:2181-Dspark.deploy.zookeepr.dir=/spark"
> bin/spark-submit \--class org.apache.spark.examples.SparkPi \--master yarn \--deploy-mode cluster \/opt/modules/spark-2.4.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.4.0.jar
> bin/spark-shell \--master spark://centos02:7077
> bin/spark-submit \--class org.apache.spark.examples.SparkPi \--master yarn \--deploy-mode cluster \--drive-memory 512m \--executor-memory 1g \--executor-cores 2 \/opt/modules/spark-2.4.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.4.0.jar
> yarn-site.xml ofen kill by yarnpmem-check-enablevmem-check-enable
> #analysis data *****create database hive;use hive;create table bike (duration int,startdate timestamp,enddate timestamp,type string) row format delimitedfields terminated by ',';
> LOAD DATA LOCAL INPATH './load.csv' INTO TABLE loan;insert overwrite local directory '/home/hadoop/college001/'row format delimited fields terminated by 't'select count(*) from bike;##about hiveDateType ,UDAF UDTFselect from_unixtime cast(concat(startdate,':00') as timestamp),'yyyyMMdd000000') from bike limit 10;select date_format('2019-10-07', 'yyyyMMdd000000')select from_unixtime(unix_timestamp('2019-10-07', 'yyyy-MM-dd'), 'yyyyMMdd000000')select unix_timestamp('2019-10-07 13:24:20','yyyy-MM-dd HH:mm:ss')
> JSON.stringify(option)
> iconv -f gbk -t utf8 a.txt –o b.txt
> mapreduce output -rw-r--r-- 2 hadoop supergroup 0 2021-10-26 07:06 /output2/_SUCCESS-rw-r--r-- 2 hadoop supergroup 1306 2021-10-26 07:06 /output2/part-r-00000
> #add node slave3yum install -y ntp && ntpdate mastervim /etc/hostsslave3scp ~/.ssh/authorized_keys root@slave3:~/.ssh/yum install -y ntp && ntpdate mastervim /etc/profile && source /etc/profileexport JAVA...export HADOOP...vim $HADOOP_HOME/etc/hadoop/slaves add slave3
> $HADOOP_HOME/sbin/hadoop-daemon.sh start datanode$HADOOP_HOME/sbin/yarn-daemon.sh start nodemanagerhdfs dfsadmin -refreshNodessbin/start-balancer.shhdfs dfsadmin -report
> echo "slave3" > excludes
> hdfs-site.xml (master)
> dfs.hosts.exclude \{/usr/hadoop/hadoop-2.7.3/etc/hadoop/excludes}
> hdfs dfsadmin -refreshNodeshdfs dfsadmin -reporthadoop-daemon.sh stop datanodeyarn-daemon.sh stop nodemanagerhdfs dfsadmin -reportsbin/start-balancer.sh
--
This message was sent by Atlassian Jira
(v8.20.7#820007)