hadoop本地運行wordcount報錯解決，並成功運行

Hadoop 中的Hello world 代碼如下：

package com.wordcount;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.io.IOException;

/**
 * @program: hadoop_demo
 * @description:
 * @author: Mr.Walloce
 * @create: 2018/11/03 15:04
 **/
public class WordCount extends Configured implements Tool {

    /**
     * <LongWritable, Text, Text, IntWritable> 輸入和輸出的key-value類型
     */
    static class MyMap extends Mapper<LongWritable, Text, Text, IntWritable> {
        //結果輸出的字符串
        Text out_key = new Text();

        //結果輸出的默認值
        IntWritable out_value = new IntWritable(1);

        /**
         * @param key     輸入的字符串的偏移量
         * @param value   輸入的字符串
         * @param context
         * @throws IOException
         * @throws InterruptedException
         */
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            System.out.println("map階段開始執行，，，");
            String line = value.toString();
            long index = key.get();
            //對字符串進行處理，獲取到單詞
            String[] words = line.split(" ");
            if (words.length > 0) {
                for (String word : words) {
                    out_key.set(word);
                    context.write(out_key, out_value);
                }
            }
            System.out.println("map階段結束。。。");
        }
    }

    /**
     * <Text, IntWritable, Text, IntWritable>輸入和輸出的key-value類型
     */
    static class MyReduce extends Reducer<Text, IntWritable, Text, IntWritable> {
        private IntWritable result = new IntWritable();

        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
            System.out.println("Reduce階段開始執行...");
            int sum = 0;
            for (IntWritable value : values) {
                sum += value.get();
            }
            result.set(sum);
            System.out.println("單詞" + key.toString() + ":  " + result.get());
            context.write(key, result);
            System.out.println("Reduce階段結束。。。");
        }
    }

    static class MyCombiner extends Reducer<Text, IntWritable, Text, IntWritable> {
        private IntWritable result = new IntWritable();

        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
            System.out.println("Combiner階段開始...");
            int sum = 0;
            for (IntWritable value : values) {
                sum += value.get();
            }
            result.set(sum);
            context.write(key, result);
            System.out.println("Combiner階段結束。。。");
        }
    }

    public int run(String[] args) throws Exception {

        //Hadoop的八股文
        Configuration conf = this.getConf();
        Job job = Job.getInstance(conf, this.getClass().getSimpleName());

        //************************對job進行具體的設置*************************
        //在集群中運行時不寫會報錯，本地運行科不寫（最好寫上）
        job.setJarByClass(WordCount.class);

        //設置輸入輸出路徑
        Path in_path = new Path(args[0]);
        FileInputFormat.addInputPath(job, in_path);
        Path out_path = new Path(args[1]);
        FileOutputFormat.setOutputPath(job, out_path);

        //輸出前判斷輸出路徑是否存在，存在則刪除（輸出路徑不能重復）
        FileSystem fs = out_path.getFileSystem(conf);
        if (fs.exists(out_path)) {
            fs.delete(out_path, true);
        }

        //運行map類相關的參數設置
        job.setMapperClass(MyMap.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        //運行Shuffle相關的參數設置
        job.setCombinerClass(MyCombiner.class);

        //設置reduce類相關的參數設置
        job.setReducerClass(MyReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        //運行是否成功
        boolean isSuccess = job.waitForCompletion(true);

        //運行成功返回0，反之返回1
        return isSuccess ? 0 : 1;
    }

    public static void main(String args[]) {
        Configuration conf = new Configuration();

        args = new String[]{
                "hdfs://walloce.one:8020/walloce/data/test.txt",
                "hdfs://walloce.one:8020/walloce/output/"+ Math.random()
        };

        try {
            ToolRunner.run(conf, new WordCount(), args);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

 

 

 

運行異常代碼：

18/11/22 15:06:00 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
18/11/22 15:06:02 INFO client.RMProxy: Connecting to ResourceManager at walloce.one/192.168.206.143:8032
18/11/22 15:06:03 WARN mapreduce.JobSubmitter: No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
18/11/22 15:06:03 INFO input.FileInputFormat: Total input paths to process : 1
18/11/22 15:06:03 INFO mapreduce.JobSubmitter: number of splits:1
18/11/22 15:06:04 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1542897380554_0004
18/11/22 15:06:04 INFO mapred.YARNRunner: Job jar is not present. Not adding any jar to the list of resources.
18/11/22 15:06:04 INFO impl.YarnClientImpl: Submitted application application_1542897380554_0004
18/11/22 15:06:04 INFO mapreduce.Job: The url to track the job: http://walloce.one:8088/proxy/application_1542897380554_0004/
18/11/22 15:06:04 INFO mapreduce.Job: Running job: job_1542897380554_0004
18/11/22 15:06:08 INFO mapreduce.Job: Job job_1542897380554_0004 running in uber mode : false
18/11/22 15:06:08 INFO mapreduce.Job:  map 0% reduce 0%
18/11/22 15:06:08 INFO mapreduce.Job: Job job_1542897380554_0004 failed with state FAILED due to: Application application_1542897380554_0004 failed 2 times due to AM Container for appattempt_1542897380554_0004_000002 exited with  exitCode: 1 due to: Exception from container-launch: ExitCodeException exitCode=1: /bin/bash: line 0: fg: no job control

ExitCodeException exitCode=1: /bin/bash: line 0: fg: no job control

    at org.apache.hadoop.util.Shell.runCommand(Shell.java:538)
    at org.apache.hadoop.util.Shell.run(Shell.java:455)
    at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:702)
    at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:195)
    at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:300)
    at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:81)
    at java.util.concurrent.FutureTask.run(FutureTask.java:262)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:745)


Container exited with a non-zero exit code 1
.Failing this attempt.. Failing the application.

檢查配置文件yarn-site.xml：

    <!-- 指定計算模型在yarn上 -->
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>

由於是在本地運行，所以計算模型要指定在本地：

    <!-- 指定計算模型在yarn上 -->
    <property>
        <name>mapreduce.framework.name</name>
        <value>local</value>
    </property>

 

文件修改后，可以運行成功。。

 

運行結果：

　　 

"C:\Program Files\Java\jdk1.8.0_162\bin\java.exe" "-javaagent:C:\Program Files\JetBrains\IntelliJ IDEA Community Edition 2018.1.6\lib\idea_rt.jar=13319:C:\Program Files\JetBrains\IntelliJ IDEA Community Edition 2018.1.6\bin" -Dfile.encoding=UTF-8 -classpath "C:\Program Files\Java\jdk1.8.0_162\jre\lib\charsets.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\deploy.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\access-bridge-64.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\cldrdata.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\dnsns.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\jaccess.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\jfxrt.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\localedata.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\nashorn.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\sunec.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\sunjce_provider.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\sunmscapi.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\sunpkcs11.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\ext\zipfs.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\javaws.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\jce.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\jfr.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\jfxswt.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\jsse.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\management-agent.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\plugin.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\resources.jar;C:\Program Files\Java\jdk1.8.0_162\jre\lib\rt.jar;E:\IdeaWorkspace\hadoop_demo\target\classes;E:\repository3\org\apache\hadoop\hadoop-common\2.5.0\hadoop-common-2.5.0.jar;E:\repository3\org\apache\hadoop\hadoop-annotations\2.5.0\hadoop-annotations-2.5.0.jar;C:\Program Files\Java\jdk1.8.0_162\lib\tools.jar;E:\repository3\com\google\guava\guava\11.0.2\guava-11.0.2.jar;E:\repository3\commons-cli\commons-cli\1.2\commons-cli-1.2.jar;E:\repository3\org\apache\commons\commons-math3\3.1.1\commons-math3-3.1.1.jar;E:\repository3\xmlenc\xmlenc\0.52\xmlenc-0.52.jar;E:\repository3\commons-httpclient\commons-httpclient\3.1\commons-httpclient-3.1.jar;E:\repository3\commons-codec\commons-codec\1.4\commons-codec-1.4.jar;E:\repository3\commons-io\commons-io\2.4\commons-io-2.4.jar;E:\repository3\commons-net\commons-net\3.1\commons-net-3.1.jar;E:\repository3\commons-collections\commons-collections\3.2.1\commons-collections-3.2.1.jar;E:\repository3\javax\servlet\servlet-api\2.5\servlet-api-2.5.jar;E:\repository3\org\mortbay\jetty\jetty\6.1.26\jetty-6.1.26.jar;E:\repository3\org\mortbay\jetty\jetty-util\6.1.26\jetty-util-6.1.26.jar;E:\repository3\com\sun\jersey\jersey-core\1.9\jersey-core-1.9.jar;E:\repository3\com\sun\jersey\jersey-json\1.9\jersey-json-1.9.jar;E:\repository3\org\codehaus\jettison\jettison\1.1\jettison-1.1.jar;E:\repository3\com\sun\xml\bind\jaxb-impl\2.2.3-1\jaxb-impl-2.2.3-1.jar;E:\repository3\javax\xml\bind\jaxb-api\2.2.2\jaxb-api-2.2.2.jar;E:\repository3\javax\xml\stream\stax-api\1.0-2\stax-api-1.0-2.jar;E:\repository3\javax\activation\activation\1.1\activation-1.1.jar;E:\repository3\org\codehaus\jackson\jackson-jaxrs\1.8.3\jackson-jaxrs-1.8.3.jar;E:\repository3\org\codehaus\jackson\jackson-xc\1.8.3\jackson-xc-1.8.3.jar;E:\repository3\com\sun\jersey\jersey-server\1.9\jersey-server-1.9.jar;E:\repository3\asm\asm\3.1\asm-3.1.jar;E:\repository3\tomcat\jasper-compiler\5.5.23\jasper-compiler-5.5.23.jar;E:\repository3\tomcat\jasper-runtime\5.5.23\jasper-runtime-5.5.23.jar;E:\repository3\javax\servlet\jsp\jsp-api\2.1\jsp-api-2.1.jar;E:\repository3\commons-el\commons-el\1.0\commons-el-1.0.jar;E:\repository3\commons-logging\commons-logging\1.1.3\commons-logging-1.1.3.jar;E:\repository3\log4j\log4j\1.2.17\log4j-1.2.17.jar;E:\repository3\net\java\dev\jets3t\jets3t\0.9.0\jets3t-0.9.0.jar;E:\repository3\org\apache\httpcomponents\httpclient\4.1.2\httpclient-4.1.2.jar;E:\repository3\org\apache\httpcomponents\httpcore\4.1.2\httpcore-4.1.2.jar;E:\repository3\com\jamesmurty\utils\java-xmlbuilder\0.4\java-xmlbuilder-0.4.jar;E:\repository3\commons-lang\commons-lang\2.6\commons-lang-2.6.jar;E:\repository3\commons-configuration\commons-configuration\1.6\commons-configuration-1.6.jar;E:\repository3\commons-digester\commons-digester\1.8\commons-digester-1.8.jar;E:\repository3\commons-beanutils\commons-beanutils\1.7.0\commons-beanutils-1.7.0.jar;E:\repository3\commons-beanutils\commons-beanutils-core\1.8.0\commons-beanutils-core-1.8.0.jar;E:\repository3\org\slf4j\slf4j-api\1.7.5\slf4j-api-1.7.5.jar;E:\repository3\org\slf4j\slf4j-log4j12\1.7.5\slf4j-log4j12-1.7.5.jar;E:\repository3\org\codehaus\jackson\jackson-core-asl\1.9.13\jackson-core-asl-1.9.13.jar;E:\repository3\org\codehaus\jackson\jackson-mapper-asl\1.9.13\jackson-mapper-asl-1.9.13.jar;E:\repository3\org\apache\avro\avro\1.7.4\avro-1.7.4.jar;E:\repository3\com\thoughtworks\paranamer\paranamer\2.3\paranamer-2.3.jar;E:\repository3\org\xerial\snappy\snappy-java\1.0.4.1\snappy-java-1.0.4.1.jar;E:\repository3\com\google\protobuf\protobuf-java\2.5.0\protobuf-java-2.5.0.jar;E:\repository3\org\apache\hadoop\hadoop-auth\2.5.0\hadoop-auth-2.5.0.jar;E:\repository3\org\apache\directory\server\apacheds-kerberos-codec\2.0.0-M15\apacheds-kerberos-codec-2.0.0-M15.jar;E:\repository3\org\apache\directory\server\apacheds-i18n\2.0.0-M15\apacheds-i18n-2.0.0-M15.jar;E:\repository3\org\apache\directory\api\api-asn1-api\1.0.0-M20\api-asn1-api-1.0.0-M20.jar;E:\repository3\org\apache\directory\api\api-util\1.0.0-M20\api-util-1.0.0-M20.jar;E:\repository3\com\jcraft\jsch\0.1.42\jsch-0.1.42.jar;E:\repository3\com\google\code\findbugs\jsr305\1.3.9\jsr305-1.3.9.jar;E:\repository3\org\apache\commons\commons-compress\1.4.1\commons-compress-1.4.1.jar;E:\repository3\org\tukaani\xz\1.0\xz-1.0.jar;E:\repository3\org\apache\hadoop\hadoop-hdfs\2.5.0\hadoop-hdfs-2.5.0.jar;E:\repository3\commons-daemon\commons-daemon\1.0.13\commons-daemon-1.0.13.jar;E:\repository3\io\netty\netty\3.6.2.Final\netty-3.6.2.Final.jar;E:\repository3\org\apache\hadoop\hadoop-client\2.5.0\hadoop-client-2.5.0.jar;E:\repository3\org\apache\hadoop\hadoop-mapreduce-client-app\2.5.0\hadoop-mapreduce-client-app-2.5.0.jar;E:\repository3\org\apache\hadoop\hadoop-mapreduce-client-common\2.5.0\hadoop-mapreduce-client-common-2.5.0.jar;E:\repository3\org\apache\hadoop\hadoop-yarn-client\2.5.0\hadoop-yarn-client-2.5.0.jar;E:\repository3\com\sun\jersey\jersey-client\1.9\jersey-client-1.9.jar;E:\repository3\org\apache\hadoop\hadoop-yarn-server-common\2.5.0\hadoop-yarn-server-common-2.5.0.jar;E:\repository3\org\apache\hadoop\hadoop-mapreduce-client-shuffle\2.5.0\hadoop-mapreduce-client-shuffle-2.5.0.jar;E:\repository3\org\fusesource\leveldbjni\leveldbjni-all\1.8\leveldbjni-all-1.8.jar;E:\repository3\org\apache\hadoop\hadoop-yarn-api\2.5.0\hadoop-yarn-api-2.5.0.jar;E:\repository3\org\apache\hadoop\hadoop-mapreduce-client-core\2.5.0\hadoop-mapreduce-client-core-2.5.0.jar;E:\repository3\org\apache\hadoop\hadoop-yarn-common\2.5.0\hadoop-yarn-common-2.5.0.jar;E:\repository3\org\apache\hadoop\hadoop-mapreduce-client-jobclient\2.5.0\hadoop-mapreduce-client-jobclient-2.5.0.jar;E:\repository3\org\apache\hadoop\hadoop-mapreduce-examples\2.5.0\hadoop-mapreduce-examples-2.5.0.jar;E:\repository3\org\apache\zookeeper\zookeeper\3.4.5\zookeeper-3.4.5.jar;E:\repository3\jline\jline\0.9.94\jline-0.9.94.jar;E:\repository3\org\jboss\netty\netty\3.2.2.Final\netty-3.2.2.Final.jar" com.wordcount.WordCount
18/11/22 15:37:44 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
18/11/22 15:37:47 INFO Configuration.deprecation: session.id is deprecated. Instead, use dfs.metrics.session-id
18/11/22 15:37:47 INFO jvm.JvmMetrics: Initializing JVM Metrics with processName=JobTracker, sessionId=
18/11/22 15:37:49 WARN mapreduce.JobSubmitter: No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
18/11/22 15:37:49 INFO input.FileInputFormat: Total input paths to process : 1
18/11/22 15:37:50 INFO mapreduce.JobSubmitter: number of splits:1
18/11/22 15:37:50 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_local857633983_0001
18/11/22 15:37:50 WARN conf.Configuration: file:/opt/module/hadoop-2.5.0/data/tmp/mapred/staging/YinYichang857633983/.staging/job_local857633983_0001/job.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.retry.interval;  Ignoring.
18/11/22 15:37:50 WARN conf.Configuration: file:/opt/module/hadoop-2.5.0/data/tmp/mapred/staging/YinYichang857633983/.staging/job_local857633983_0001/job.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.attempts;  Ignoring.
18/11/22 15:37:51 WARN conf.Configuration: file:/opt/module/hadoop-2.5.0/data/tmp/mapred/local/localRunner/YinYichang/job_local857633983_0001/job_local857633983_0001.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.retry.interval;  Ignoring.
18/11/22 15:37:51 WARN conf.Configuration: file:/opt/module/hadoop-2.5.0/data/tmp/mapred/local/localRunner/YinYichang/job_local857633983_0001/job_local857633983_0001.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.attempts;  Ignoring.
18/11/22 15:37:51 INFO mapreduce.Job: The url to track the job: http://localhost:8080/
18/11/22 15:37:51 INFO mapreduce.Job: Running job: job_local857633983_0001
18/11/22 15:37:51 INFO mapred.LocalJobRunner: OutputCommitter set in config null
18/11/22 15:37:51 INFO mapred.LocalJobRunner: OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
18/11/22 15:37:51 INFO mapred.LocalJobRunner: Waiting for map tasks
18/11/22 15:37:51 INFO mapred.LocalJobRunner: Starting task: attempt_local857633983_0001_m_000000_0
18/11/22 15:37:51 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
18/11/22 15:37:51 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@3a8763db
18/11/22 15:37:51 INFO mapred.MapTask: Processing split: hdfs://walloce.one:8020/walloce/data/test.txt:0+173
18/11/22 15:37:51 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
18/11/22 15:37:51 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
18/11/22 15:37:51 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
18/11/22 15:37:51 INFO mapred.MapTask: soft limit at 83886080
18/11/22 15:37:51 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
18/11/22 15:37:51 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
map階段開始執行，，，
map階段結束。。。
map階段開始執行，，，
map階段結束。。。
map階段開始執行，，，
map階段結束。。。
map階段開始執行，，，
map階段結束。。。
map階段開始執行，，，
map階段結束。。。
map階段開始執行，，，
map階段結束。。。
18/11/22 15:37:52 INFO mapred.LocalJobRunner: 
18/11/22 15:37:52 INFO mapreduce.Job: Job job_local857633983_0001 running in uber mode : false
18/11/22 15:37:52 INFO mapreduce.Job:  map 0% reduce 0%
18/11/22 15:37:52 INFO mapred.MapTask: Starting flush of map output
18/11/22 15:37:52 INFO mapred.MapTask: Spilling map output
18/11/22 15:37:52 INFO mapred.MapTask: bufstart = 0; bufend = 321; bufvoid = 104857600
18/11/22 15:37:52 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 26214252(104857008); length = 145/6553600
Combiner階段開始...
Combiner階段結束。。。
Combiner階段開始...
Combiner階段結束。。。
Combiner階段開始...
Combiner階段結束。。。
Combiner階段開始...
Combiner階段結束。。。
Combiner階段開始...
Combiner階段結束。。。
Combiner階段開始...
Combiner階段結束。。。
Combiner階段開始...
Combiner階段結束。。。
Combiner階段開始...
Combiner階段結束。。。
Combiner階段開始...
Combiner階段結束。。。
18/11/22 15:37:52 INFO mapred.MapTask: Finished spill 0
18/11/22 15:37:52 INFO mapred.Task: Task:attempt_local857633983_0001_m_000000_0 is done. And is in the process of committing
18/11/22 15:37:52 INFO mapred.LocalJobRunner: map
18/11/22 15:37:52 INFO mapred.Task: Task 'attempt_local857633983_0001_m_000000_0' done.
18/11/22 15:37:52 INFO mapred.LocalJobRunner: Finishing task: attempt_local857633983_0001_m_000000_0
18/11/22 15:37:52 INFO mapred.LocalJobRunner: map task executor complete.
18/11/22 15:37:52 INFO mapred.LocalJobRunner: Waiting for reduce tasks
18/11/22 15:37:52 INFO mapred.LocalJobRunner: Starting task: attempt_local857633983_0001_r_000000_0
18/11/22 15:37:52 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
18/11/22 15:37:52 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@32eb8b1d
18/11/22 15:37:52 INFO mapred.ReduceTask: Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@5b831a05
18/11/22 15:37:52 INFO reduce.MergeManagerImpl: MergerManager: memoryLimit=1291845632, maxSingleShuffleLimit=322961408, mergeThreshold=852618176, ioSortFactor=10, memToMemMergeOutputsThreshold=10
18/11/22 15:37:52 INFO reduce.EventFetcher: attempt_local857633983_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events
18/11/22 15:37:52 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local857633983_0001_m_000000_0 decomp: 101 len: 105 to MEMORY
18/11/22 15:37:52 INFO reduce.InMemoryMapOutput: Read 101 bytes from map-output for attempt_local857633983_0001_m_000000_0
18/11/22 15:37:52 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 101, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->101
18/11/22 15:37:52 INFO reduce.EventFetcher: EventFetcher is interrupted.. Returning
18/11/22 15:37:52 INFO mapred.LocalJobRunner: 1 / 1 copied.
18/11/22 15:37:52 INFO reduce.MergeManagerImpl: finalMerge called with 1 in-memory map-outputs and 0 on-disk map-outputs
18/11/22 15:37:52 INFO mapred.Merger: Merging 1 sorted segments
18/11/22 15:37:52 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 97 bytes
18/11/22 15:37:52 INFO reduce.MergeManagerImpl: Merged 1 segments, 101 bytes to disk to satisfy reduce memory limit
18/11/22 15:37:52 INFO reduce.MergeManagerImpl: Merging 1 files, 105 bytes from disk
18/11/22 15:37:52 INFO reduce.MergeManagerImpl: Merging 0 segments, 0 bytes from memory into reduce
18/11/22 15:37:52 INFO mapred.Merger: Merging 1 sorted segments
18/11/22 15:37:52 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 97 bytes
18/11/22 15:37:52 INFO mapred.LocalJobRunner: 1 / 1 copied.
18/11/22 15:37:52 INFO Configuration.deprecation: mapred.skip.on is deprecated. Instead, use mapreduce.job.skiprecords
Reduce階段開始執行...
單詞I:  5
Reduce階段結束。。。
Reduce階段開始執行...
單詞am:  5
Reduce階段結束。。。
Reduce階段開始執行...
單詞an:  5
Reduce階段結束。。。
Reduce階段開始執行...
單詞for:  5
Reduce階段結束。。。
Reduce階段開始執行...
單詞hello:  1
Reduce階段結束。。。
Reduce階段開始執行...
單詞linux:  5
Reduce階段結束。。。
Reduce階段開始執行...
單詞new:  5
Reduce階段結束。。。
Reduce階段開始執行...
單詞programer:  5
Reduce階段結束。。。
Reduce階段開始執行...
單詞world!:  1
Reduce階段結束。。。
18/11/22 15:37:52 INFO mapred.Task: Task:attempt_local857633983_0001_r_000000_0 is done. And is in the process of committing
18/11/22 15:37:52 INFO mapred.LocalJobRunner: 1 / 1 copied.
18/11/22 15:37:52 INFO mapred.Task: Task attempt_local857633983_0001_r_000000_0 is allowed to commit now
18/11/22 15:37:52 INFO output.FileOutputCommitter: Saved output of task 'attempt_local857633983_0001_r_000000_0' to hdfs://walloce.one:8020/walloce/output/_temporary/0/task_local857633983_0001_r_000000
18/11/22 15:37:52 INFO mapred.LocalJobRunner: reduce > reduce
18/11/22 15:37:52 INFO mapred.Task: Task 'attempt_local857633983_0001_r_000000_0' done.
18/11/22 15:37:52 INFO mapred.LocalJobRunner: Finishing task: attempt_local857633983_0001_r_000000_0
18/11/22 15:37:52 INFO mapred.LocalJobRunner: reduce task executor complete.
18/11/22 15:37:53 INFO mapreduce.Job:  map 100% reduce 100%
18/11/22 15:37:53 INFO mapreduce.Job: Job job_local857633983_0001 completed successfully
18/11/22 15:37:53 INFO mapreduce.Job: Counters: 38
    File System Counters
        FILE: Number of bytes read=576
        FILE: Number of bytes written=490527
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
        HDFS: Number of bytes read=346
        HDFS: Number of bytes written=63
        HDFS: Number of read operations=15
        HDFS: Number of large read operations=0
        HDFS: Number of write operations=6
    Map-Reduce Framework
        Map input records=6
        Map output records=37
        Map output bytes=321
        Map output materialized bytes=105
        Input split bytes=110
        Combine input records=37
        Combine output records=9
        Reduce input groups=9
        Reduce shuffle bytes=105
        Reduce input records=9
        Reduce output records=9
        Spilled Records=18
        Shuffled Maps =1
        Failed Shuffles=0
        Merged Map outputs=1
        GC time elapsed (ms)=3
        CPU time spent (ms)=0
        Physical memory (bytes) snapshot=0
        Virtual memory (bytes) snapshot=0
        Total committed heap usage (bytes)=372244480
    Shuffle Errors
        BAD_ID=0
        CONNECTION=0
        IO_ERROR=0
        WRONG_LENGTH=0
        WRONG_MAP=0
        WRONG_REDUCE=0
    File Input Format Counters 
        Bytes Read=173
    File Output Format Counters 
        Bytes Written=63

Process finished with exit code 0

 

由wordcount運行結果可以看出，MapReduce的執行順序：

　　1、Map階段開始

　　　　因為map階段輸入的數據時以行為單位，偏移量即為行序號，有多少行map就執行多少次。

　　2、Combiner階段開始

　　　　經過map階段的Shuffle后Map階段結束，開始Combiner階段，Combiner階段進行的是每個分區里的數據小聚合，有多少key進行多少次。

　　3、Reduce階段開始

　　　　Combiner階段結束后，Reduce階段需要將所有分區的所有數據進行聚合，得出最終的結果。

 

記： 心酸的學習歷程！