{"id":5027,"date":"2024-01-13T19:29:19","date_gmt":"2024-01-13T11:29:19","guid":{"rendered":"https:\/\/www.kenpai.com\/cms\/5027\/"},"modified":"2024-01-13T19:29:19","modified_gmt":"2024-01-13T11:29:19","slug":"hbase-hadoop%e5%ae%8c%e5%85%a8%e5%88%86%e5%b8%83%e5%bc%8f%e7%8e%af%e5%a2%83%e6%90%ad%e5%bb%ba","status":"publish","type":"post","link":"https:\/\/www.kenpai.com\/cms\/5027\/","title":{"rendered":"hbase hadoop\u5b8c\u5168\u5206\u5e03\u5f0f\u73af\u5883\u642d\u5efa"},"content":{"rendered":"<\/p>\n

1\u3001\u73af\u5883\u51c6\u5907<\/strong><\/p>\n

linux\u7cfb\u7edf\uff1aCentos6.4 release (Final) \u4e0b\u8f7d\u5730\u5740\uff1ahttp:\/\/www.centos.org\/<\/p>\n<\/p>\n

jdk\uff1a1.7.0_45 \u4e0b\u8f7d\u5730\u5740\uff1ahttp:\/\/www.oracle.com\/technetwork\/java\/javase\/downloads\/index.html<\/p>\n<\/p>\n

hadoop\uff1ahadoop-2.6.0\u4e0b\u8f7d\u5730\u5740\uff1ahttp:\/\/apache.claz.org\/hadoop\/common\/stable2\/
hbase\uff1ahbase-0.98.8-hadoop2 \u4e0b\u8f7d\u5730\u5740\uff1ahttp:\/\/mirrors.gigenet.com\/apache\/hbase\/stable\/<\/p>\n

2\u3001\u914d\u7f6e\u6b65\u9aa4\uff1a<\/strong> 2.1 \u5b89\u88c5jdk\uff0c\u522b\u5fd8\u4e86\u4fee\u6539 vim \/etc\/profile\u3002<\/strong> 2.2 ssh\u514d\u5bc6\u7801\u767b\u9646<\/strong><\/p>\n

\u8bf4\u660e\uff1a\u7531\u4e8ejdk\u7684\u73af\u5883\u914d\u7f6e\u4ee5\u53cassh\u9762\u5bc6\u7801\u767b\u9646\u4e00\u822c\u4e0d\u51fa\u4ec0\u4e48\u95ee\u9898\uff0c\u7f51\u4e0a\u8d44\u6599\u8f83\u591a\uff0c\u6545\u5728\u6b64\u4e0d\u505a\u8d58\u8ff0<\/strong><\/p>\n

2.3 hadoop\u73af\u5883\u914d\u7f6e<\/strong><\/p>\n

\u672c\u673a\u5206\u5e03\u5f0f\u670d\u52a1\u5668<\/p>\n

ip30\uff1alingcloud30 \uff08\u7531\u4e8e\u672c\u53f0\u670d\u52a1\u5668\u914d\u7f6e\u76f8\u5bf9\u8f83\u4f4e\uff0c\u6545\u7528\u5176\u4f5c\u4e3aNameNode\uff0c\u4e5f\u662fSecondaryNameNode\u3002\u4e5f\u662f\u4e0b\u9762Hbase\u914d\u7f6e\u7684HMaster\uff09<\/p>\n<\/p>\n

ip29\uff1alingcloud29\uff08\u672c\u53f0\u4ee5\u53ca\u4e0b\u9762\u7684\u4e24\u53f0\u4f5c\u4e3aDateNode\u3002\u4ee5\u53ca\u4e0b\u9762Hbase\u914d\u7f6e\u7684HRegionServer\uff09<\/p>\n<\/p>\n

ip31\uff1alingcloud31
ip32\uff1alingcloud32<\/p>\n

2.3.1{hadoop}\/etc\/hadoop\/core-site.xml\u914d\u7f6e<\/strong><\/p>\n

\u4e0b\u9762\u662f\u6211\u7684lingcloud30\u7684\u914d\u7f6e\uff0c\u6709\u7684\u6709\u6ce8\u91ca\uff0c\u65b9\u4fbf\u7406\u89e3<\/p>\n

<code=xml><\/p>\n<\/p>\n

<prename="code" class="html"><configuration> <!– this file must be the same with the $hbase$\/conf\/ "hbase.rootdir"property –> <property> <name>fs.defaultFS<\/name> <value>hdfs:\/\/lingcloud30:9000<\/value> <\/property> <!– if the HMaster start servel seconds andthen HMaster server abort ,you should remove the file hadooptmp and restart thehbase –> <property> <name>hadoop.tmp.dir<\/name> <value>\/usr\/qhl\/hadoopWorkspace\/hadooptmp<\/value> <\/property> <property> <name>io.file.buffer.size<\/name> <value>131072<\/value> <\/property> <!–\u6dfb\u52a0 httpfs \u7684 \u9009\u9879–> <property> <name>hadoop.proxyuser.root.hosts<\/name> <value>lingcloud30<\/value> <\/property> <property> <name>hadoop.proxyuser.root.groups<\/name> <value>*<\/value> <\/property> <\/configuration><\/p>\n<\/p>\n

<\/code><\/p>\n

2.3.2{hadoop}\/etc\/hadoop\/hdfs-site.xml\u914d\u7f6e<\/strong><\/p>\n

<code=xml><\/p>\n<\/p>\n

<prename="code" class="html"><configuration> <property> <name>dfs.datanode.handler.count<\/name> <value>5<\/value> <description>The number ofserver threads for the datanode.<\/description> <\/property> <property> <name>dfs.namenode.handler.count<\/name> <value>5<\/value> <description>The number ofserver threads for the namenode.<\/description> <\/property> <property> <name>dfs.replication<\/name> <value>3<\/value> <\/property> <property> <name>dfs.namenode.name.dir<\/name> <value>file:\/usr\/qhl\/hadoopWorkspace\/hdfs\/name<\/value> <final>true<\/final> <\/property> <property> <name>dfs.permissions<\/name> <value>false<\/value> <\/property> <property> <name>dfs.federation.nameservice.id<\/name> <value>ns1<\/value> <\/property> <property> <name>dfs.namenode.backup.address.ns1<\/name> <value>lingcloud30:50100<\/value> <\/property> <property> <name>dfs.namenode.backup.http-address.ns1<\/name> <value>lingcloud30:50105<\/value> <\/property> <property> <name>dfs.federation.nameservices<\/name> <value>ns1<\/value> <\/property> <property> <name>dfs.namenode.rpc-address.ns1<\/name> <value>lingcloud30:9000<\/value> <\/property> <property> <name>dfs.namenode.rpc-address.ns2<\/name> <value>lingcloud30:9000<\/value> <\/property> <property> <name>dfs.namenode.http-address.ns1<\/name> <value>lingcloud30:23001<\/value> <!– this is the web browsre port if you browsre http:\/\/lingcloud30:23001. you\\\\\\’ll see some information –> <\/property> <property> <name>dfs.namenode.http-address.ns2<\/name> <value>lingcloud30:13001<\/value> <\/property> <property> <name>dfs.dataname.data.dir<\/name> <value>file:\/usr\/qhl\/hadoopWorkspace\/hdfs\/data<\/value> <final>true<\/final> <\/property> <property> <name>dfs.namenode.secondary.http-address.ns1<\/name> <value>lingcloud30:23002<\/value> <\/property> <property> <name>dfs.namenode.secondary.http-address.ns2<\/name> <value>lingcloud30:23002<\/value> <\/property> <property> <name>dfs.namenode.secondary.http-address.ns1<\/name> <value>lingcloud30:23003<\/value> <\/property> <property> <name>dfs.namenode.secondary.http-address.ns2<\/name> <value>lingcloud30:23003<\/value> <\/property> <property> <name>dfs.datanode.max.xcievers<\/name> <value>8192<\/value> <\/property> <\/configuration> <\/p>\n<\/p>\n

<\/code><\/p>\n

2.3.3{hadoop}\/etc\/hadoop\/yarn-site.xml\u914d\u7f6e<\/strong><\/p>\n

<code=xml><\/p>\n<\/p>\n

<configuration> <property> <name>yarn.resourcemanager.address<\/name> <value>lingcloud30:18040<\/value> <\/property> <property> <name>yarn.resourcemanager.scheduler.address<\/name> <value>lingcloud30:18030<\/value> <\/property> <property> <name>yarn.resourcemanager.webapp.address<\/name> <value>lingcloud30:18088<\/value> <!– this is the web browsre port, ifyou browsre http:\/\/lingcloud30:18088, you\\\\\\’ll see some information –> <\/property> <property> <name>yarn.resourcemanager.resource-tracker.address<\/name> <value>lingcloud30:18025<\/value> <\/property> <property> <name>yarn.resourcemanager.admin.address<\/name> <value>lingcloud30:18141<\/value> <\/property> <property> <name>yarn.nodemanager.aux-services<\/name> <value>mapreduce.shuffle<\/value> <\/property> <\/configuration> <\/p>\n<\/p>\n

<\/code><\/p>\n

2.3.4{hadoop}\/etc\/hadoop\/mapred-env.sh\u914d\u7f6e<\/strong><\/p>\n

\u6dfb\u52a0<\/p>\n

<code=xml><\/p>\n<\/p>\n

exportHADOOP_MAPRED_PID_DIR=\/usr\/qhl\/hadoopWorkspace\/haddopMapredPidDir # The pidfiles are stored. \/tmp by default.<\/p>\n<\/p>\n

<\/code><\/p>\n

2.3.5{hadoop}\/etc\/hadoop\/slaves\u914d\u7f6e<\/strong><\/p>\n

<code=xml><\/p>\n<\/p>\n

lingcloud32 lingcloud31 lingcloud29<\/p>\n<\/p>\n

<\/code><\/p>\n

2.3.6 \u5c0f\u7ed3<\/strong><\/p>\n

\u4ee5\u4e0a\u914d\u7f6e\u590d\u5236\u5230\u5176\u4ed6node\u8282\u70b9\uff0c\u6309\u7167\u76f8\u540c\u914d\u7f6e\u5373\u53ef\u3002<\/p>\n

3.hbase\u914d\u7f6e<\/strong> 3.1habs-env.sh\u73af\u5883\u914d\u7f6e<\/strong><\/p>\n

<code=xml><\/p>\n<\/p>\n

exportJAVA_HOME=\/usr\/lib\/jdk\/jdk1.7.0_45\/ #\u4f60\u7684jdk\u5b89\u88c5\u76ee\u5f55 exportHBASE_PID_DIR=\/usr\/qhl\/hbaseWorkspace\/pids # The directory where pid files arestored. \/tmp by default. exportHBASE_MANAGES_ZK=true #\u4f7f\u7528hhase\u81ea\u5e26\u7684zookeeper<\/p>\n<\/p>\n

<\/code><\/p>\n

3.2hbase-site.xml\u914d\u7f6e<\/strong><\/p>\n

<code=xml><\/p>\n<\/p>\n

<configuration> <property> <name>hbase.rootdir<\/name> <value>hdfs:\/\/lingcloud30:9000\/hbase<\/value> <!– this must be ths same with the{hadoop}\/etc\/hadoop:core-site.xml <property> <name>fs.defaultFS<\/name> <value>hdfs:\/\/lingcloud30:9000<\/value> <\/property> –> <\/property> <property> <name>hbase.zookeeper.property.dataDir<\/name> <value>\/usr\/qhl\/zookeeper<\/value> <\/property> <property> <name>hbase.cluster.distributed<\/name> <value>true<\/value> <description>which directsHBase to run in distributed mode, with one JVM instance perdaemon.<\/description> <\/property> <property> <name>hbase.tmp.dir<\/name> <value>\/usr\/qhl\/hbaseWorkspace\/hbasetmp<\/value> <!– if the HMaster start servel seconds andthen HMaster server abort ,you should remove the file hadooptmp and restart thehbase –> <\/property> <property> <name>hbase.zookeeper.quorum<\/name> <value>lingcloud29,lingcloud31,lingcloud32<\/value> <\/property> <property> <name>hbase.master<\/name> <value>lingcloud30:60000<\/value> <\/property> <property> <name>hbase.master.port<\/name> <value>60000<\/value> <description>The portmaster should bind to.<\/description> <\/property> <property> <name>hbase.master.maxclockskew<\/name> <value>200000<\/value> <description>Time difference ofregionserver from master<\/description> <\/property> <\/configuration> <\/p>\n<\/p>\n

<\/code><\/p>\n

3.3 regionserver \u914d\u7f6e<\/strong><\/p>\n

<code=xml><\/p>\n<\/p>\n

lingcloud29 lingcloud31 lingcloud32<\/p>\n<\/p>\n

<\/code><\/p>\n

4.\u914d\u7f6e\u6210\u529f\u540e\u6d4f\u89c8\u5668\u622a\u56fe<\/strong><\/p>\n

\u6211\u5728{hadoop}etc\/hadoop\u4e0b\u7684hdfs-site.xml\u4e2d\u914d\u7f6e\u4e86\u7aef\u53e3\u4e3a23001. \u914d\u7f6e\u9879\u662fdfs.namenode.http-address.ns1<\/p>\n<\/p>\n

\u8bbf\u95ee18088\u7aef\u53e3\u4f1a\u51fa\u73b0\u5982\u4e0b\u754c\u9762\uff1a<\/p>\n<\/p>\n

hbase\u9ed8\u8ba4\u7aef\u53e3\u4e3a60010<\/p>\n<\/p>\n

5.\u603b\u7ed3\u548c\u53c2\u8003\u6587\u732e<\/strong><\/p>\n

\u672c\u4eba\u5fc5\u987b\u627f\u8ba4\uff0c\u5f88\u591a\u4e1c\u897f\u90fd\u662f\u4ece\u7f51\u4e0a\u5b66\u5230\u7684\uff0c\u53c2\u8003\u4e86\u5f88\u591a\u7684\u535a\u6587\uff0c\u4e5f\u9047\u5230\u4e86\u5f88\u591a\u7684\u9519\u8bef\uff0c\u4f46\u662f\u7531\u4e8e\u65e5\u5fd7\u4e2d\u95f4\u6ca1\u6709\u4fdd\u5b58\uff0c\u9519\u8bef\u5c31\u4e0d\u8d34\u4e86\u3002\u540c\u65f6\u7531\u4e8e\u53c2\u8003\u4e86\u5f88\u591a\u7684\u535a\u6587\uff0c\u5728\u6b64\u4e5f\u4e0d\u80fd\u4e00\u4e00\u5217\u4e3e\u4e86\uff0c\u672c\u7bc7\u535a\u6587\u662f\u6211\u7684\u7b2c\u4e00\u7bc7\uff0c\u6709\u4ec0\u4e48\u4e0d\u5bf9\u4e4b\u5904\uff0c\u8fd8\u8bf7\u6279\u8bc4\u6307\u6b63\uff0c\u5171\u540c\u5b66\u4e60\u3002<\/p>\n<\/p>\n<\/p>\n

\u66f4\u591a\u5173\u4e8e\u4e91\u670d\u52a1\u5668\uff0c\u57df\u540d<\/a><\/span>\u6ce8\u518c\uff0c\u865a\u62df\u4e3b\u673a\u7684\u95ee\u9898\uff0c\u8bf7\u8bbf\u95ee\u57a6\u6d3e\u79d1\u6280<\/a><\/span>\u5b98\u7f51\uff1awww.kenpai.com<\/a><\/span><\/p>\n","protected":false},"excerpt":{"rendered":"

1\u3001\u73af\u5883\u51c6\u5907linux\u7cfb\u7edf\uff1aCentos6.4 release (Final) \u4e0b\u8f7d\u5730\u5740\uff1ahttp:\/\/www.centos.org\/ jdk<\/p>\n","protected":false},"author":2,"featured_media":4351,"comment_status":"","ping_status":"","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[3322],"tags":[3896],"class_list":["post-5027","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-ktougao","tag-hbase-hadoop"],"_links":{"self":[{"href":"https:\/\/www.kenpai.com\/cms\/wp-json\/wp\/v2\/posts\/5027","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.kenpai.com\/cms\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.kenpai.com\/cms\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.kenpai.com\/cms\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.kenpai.com\/cms\/wp-json\/wp\/v2\/comments?post=5027"}],"version-history":[{"count":0,"href":"https:\/\/www.kenpai.com\/cms\/wp-json\/wp\/v2\/posts\/5027\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.kenpai.com\/cms\/wp-json\/wp\/v2\/media\/4351"}],"wp:attachment":[{"href":"https:\/\/www.kenpai.com\/cms\/wp-json\/wp\/v2\/media?parent=5027"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.kenpai.com\/cms\/wp-json\/wp\/v2\/categories?post=5027"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.kenpai.com\/cms\/wp-json\/wp\/v2\/tags?post=5027"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}