{"id":116,"date":"2018-03-10T15:10:38","date_gmt":"2018-03-10T07:10:38","guid":{"rendered":"http:\/\/www.max-shu.com\/blog\/?p=116"},"modified":"2018-03-10T15:10:38","modified_gmt":"2018-03-10T07:10:38","slug":"hadoop%e7%9a%84gridmix2%e5%9f%ba%e5%87%86%e6%b5%8b%e8%af%95","status":"publish","type":"post","link":"http:\/\/www.max-shu.com\/blog\/?p=116","title":{"rendered":"Hadoop\u7684Gridmix2\u57fa\u51c6\u6d4b\u8bd5"},"content":{"rendered":"

1. \u628ahadoop\u914d\u7f6e\u597d\u3002<\/b><\/p>\n

2. \u7f16\u8bd1gridmix<\/b>
\n$ cd hadoop\/src\/benchmarks\/gridmix2<\/span>
\n$ ant<\/span>
\n$ cp build\/gridmix.jar .<\/span><\/p>\n

3. \u4fee\u6539\u914d\u7f6e<\/b>
\n$ vi gridmix-env-2<\/span><\/p>\n

export HADOOP_INSTALL_HOME=\/home\/max\/pangusou<\/span><\/p>\n

export HADOOP_VERSION=hadoop-0.20.2-cdh3u0<\/span><\/p>\n

export HADOOP_HOME=${HADOOP_INSTALL_HOME}\/${HADOOP_VERSION}<\/span><\/p>\n

export HADOOP_CONF_DIR=${HADOOP_HOME}\/conf<\/span><\/p>\n

export USE_REAL_DATASET=<\/span><\/span><\/p>\n

\u7ea2\u5b57\u6539\u4e3aTRUE<\/span>\u7684\u8bdd\uff0c\u5c31\u662f2T<\/span>\u975e\u538b\u7f29\u6570\u636e\u548c500G<\/span>\u538b\u7f29\u6570\u636e\uff0c\u7a7a\u7740\u7684\u8bdd\u5426\u5219\u4e3a500M<\/span>\u538b\u7f29\u6570\u636e\u548c2G<\/span>\u975e\u538b\u7f29\u6570\u636e\u3002<\/span><\/p>\n

export APP_JAR=${HADOOP_HOME}\/hadoop-test-0.20.2-cdh3u0.jar<\/span><\/p>\n

export EXAMPLE_JAR=${HADOOP_HOME}\/hadoop-examples-0.20.2-cdh3u0.jar<\/span><\/p>\n

export STREAMING_JAR=${HADOOP_HOME}\/contrib\/streaming\/hadoop-streaming-0.20.2-cdh3u0.jar<\/span><\/p>\n

$ vi gridmix_config.xml<\/span><\/b><\/p>\n

\u57fa\u51c6\u6d4b\u8bd5\u65f6\uff0c\u8fd9\u91cc\u5efa\u8bae\u4e0d\u4fee\u6539\uff08\u4f46\u662fjobs\u6570\u4e0d\u6539\u5c0f\u7684\u8bdd\uff0c\u5f88\u96be\u6d4b\u8bd5\u8fc7\u53bb\uff0c\u7ecf\u5e38\u4f1a\u5bfc\u81f4taskTracker\u9000\u51fa\uff09\u3002<\/p>\n

4. \u4ea7\u751f\u6d4b\u8bd5\u6570\u636e<\/b><\/p>\n

$ chmod +x generateGridmix2data.sh<\/span><\/p>\n

$ .\/ generateGridmix2data.sh<\/span><\/p>\n

5. \u8fd0\u884c\u6d4b\u8bd5<\/b><\/p>\n

$ chmod +x rungridmix_2<\/span><\/p>\n

$ .\/ rungridmix_2<\/span><\/p>\n

6. \u67e5\u770b\u6d4b\u8bd5\u7ed3\u679c\uff1a<\/b><\/p>\n

Shell<\/span>\u4e0a\u663e\u793a\u6700\u7ec8\u6d4b\u8bd5\u7ed3\u679c\uff0c\u65f6\u95f4\u8d8a\u77ed\u8d8a\u597d\u3002<\/p>\n

GridMix results:<\/span><\/p>\n

Total num of Jobs: 227<\/span><\/p>\n

ExecutionTime: 21513\u00a0<\/span>\uff08\u5355\u4f4d\u4e3a\u79d2\uff09<\/p>\n

\u4ecehttp:\/\/masterip:50030\u4e0a\u770b\uff1a<\/b><\/p>\n

\u4e00\u5171\u6709\u591a\u5c11\u4e2ajob<\/span>\u7531gridmix<\/span>\u81ea\u5df1\u51b3\u5b9a\uff0c\u4f46\u662f\u6bcf\u7c7bjob<\/span>\u6570\u7531gridmix_config.xml<\/span>\u91cc\u9762\u7684\u53c2\u6570\u51b3\u5b9a\uff08gridmix<\/span>\u4f1a\u6839\u636e\u53c2\u6570\u8ba1\u7b97\uff09\u3002\u6bcf\u4e2ajob<\/span>\u7684map<\/span>\u6570\u7531gridmix<\/span>\u8ba1\u7b97\uff0c\u57fa\u672c\u56fa\u5b9a\u572880\uff0c81<\/span>\uff0c\u6700\u540e\u51e0\u4e2a\u4f1a\u4e0d\u4e00\u6837\uff1b\u6bcf\u4e2ajob<\/span>\u7684reduce<\/span>\u6570\u7531gridmix_config.xml<\/span>\u91cc\u9762\u7684\u53c2\u6570\u51b3\u5b9a\uff08gridmix<\/span>\u4f1a\u6839\u636e\u53c2\u6570\u8ba1\u7b97\uff09\u3002gridmix_config.xml<\/span>\u91cc\u9762\u5b9a\u4e49\u7684\u6bcf\u4e00\u79cd\u7c7b\u578b\u90fd\u4f1a\u6267\u884c\u5230\u3002<\/p>\n

\u4ecehttp:\/\/masterip:50070\u4e0a\u770b\uff1a<\/b><\/p>\n

\/<\/a>gridmix<\/a>\/data<\/a>\/<\/span>\u4e0b\u6709\u4e09\u4e2a\u76ee\u5f55\uff08WebSimulationBlockCompressed\uff0cMonsterQueryBlockCompressed\uff0cSortUncompressed\uff09\uff0c\u6bcf\u4e2a\u76ee\u5f55\u4e0b\u90fd\u670910<\/span>0\u4e2a\u6587\u4ef6\uff0c\u662f\u7531generateGridmix2data.sh<\/span>\u91cc\u9762\u5b9a\u4e49\u7684NUM_MAPS=100<\/span>\u51b3\u5b9a\u7684\u3002<\/p>\n

\u73b0\u5728\u6b63\u5728\u5f00\u53d1\u7684\u662fGridmix3\uff0c\u4e4b\u540e\u4f1a\u53d1\u5e03\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"

1. \u628ahadoop\u914d\u7f6e\u597d\u3002 2. \u7f16\u8bd1gridmix $ cd hadoop\/src\/benchmarks\/ …<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[63],"tags":[83,81,80,82],"class_list":["post-116","post","type-post","status-publish","format-standard","hentry","category-63","tag-benchmark","tag-gridmix","tag-hadoop","tag-82"],"views":1774,"_links":{"self":[{"href":"http:\/\/www.max-shu.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/116","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/www.max-shu.com\/blog\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.max-shu.com\/blog\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.max-shu.com\/blog\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/www.max-shu.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=116"}],"version-history":[{"count":1,"href":"http:\/\/www.max-shu.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/116\/revisions"}],"predecessor-version":[{"id":117,"href":"http:\/\/www.max-shu.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/116\/revisions\/117"}],"wp:attachment":[{"href":"http:\/\/www.max-shu.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=116"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.max-shu.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=116"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.max-shu.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=116"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}