空指針異常Hadoop

Question

您好我目前正在編寫一個地圖減少作業，以通過包含圖像和記錄輸出的文件包含紅色或不包含紅色的hdfs。我目前遇到NullPointerException錯誤，我似乎無法找出它來自哪裏。下面是我的代碼

{ 

package hipi.examples.redfilter; 


import java.awt.Color; 
import java.awt.image.BufferedImage; 
import java.io.IOException; 

import javax.imageio.ImageIO; 

import org.apache.hadoop.conf.Configuration; 
import org.apache.hadoop.conf.Configured; 
import org.apache.hadoop.fs.FileStatus; 
import org.apache.hadoop.fs.FileSystem; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.io.BooleanWritable; 
import org.apache.hadoop.io.IntWritable; 
import org.apache.hadoop.io.NullWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Job; 
import org.apache.hadoop.mapreduce.Mapper; 
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
import org.apache.hadoop.util.Tool; 
import org.apache.hadoop.util.ToolRunner; 

public class RedFilter extends Configured implements Tool{ 
    private static String inputPath; 
    public static class RedFilterMapper extends Mapper<NullWritable, BufferedImage, Text, BooleanWritable> { 


     public void map(IntWritable key, BufferedImage currImage, Context content) throws IOException, InterruptedException 
     { 
      System.out.println("I am in the mapper method"); 
      FileSystem fs = FileSystem.get(new Configuration()); 
      FileStatus[] status = fs.listStatus(new Path(inputPath)); 
      System.out.println("I am in the mapper method"); 

      for (int i=0;i<status.length;i++){ 
       String fileName = fs.open(status[i].getPath()).toString(); 
        currImage= ImageIO.read(fs.open(status[i].getPath())); 


       int width = currImage.getWidth(); 
       int height = currImage.getHeight(); 
       boolean exit = false; 

       for(int x = 0; x < width; x++) { 
        for(int y = 0; y < height; y++) { 
         Color c = new Color(currImage.getRGB(x, y)); 
         int red = c.getRed(); 

         if (red > 200 && c.getBlue() < 100 && c.getGreen() < 100) { 
          content.write(new Text(fileName), new BooleanWritable(true)); 
          exit = true; 
          break; 

         } 

        } 
        if(exit) { 
         break; 
        } 
       } 
       if(!exit) { 
       content.write(new Text(fileName), new BooleanWritable(false)); 
       } 


     } 

     }  

    } 

    public int run(String[] args) throws Exception { 
     Configuration conf = new Configuration(); 
     if (args.length < 3) { 
      System.out.println("Usage: dumphib <input hib> <outputdir>"); 
      System.exit(0); 
     } 
     inputPath = args[1]; 
     String outputPath = args[2]; 

     Job job = new Job(conf, "refilter"); 
     System.out.println("Jar Files r being set"); 
     job.setJarByClass(RedFilter.class); 
     System.out.println("Mapper is being set"); 
     job.setMapperClass(RedFilterMapper.class); 


     // Set formats 
     job.setOutputKeyClass(Text.class); 
     job.setOutputValueClass(Boolean.class); 
//  job.setInputFormatClass(ImageBundleInputFormat.class); 

     // Set out/in paths 
     removeDir(outputPath, conf); 
     FileOutputFormat.setOutputPath(job, new Path(outputPath)); 
     FileInputFormat.setInputPaths(job, new Path(inputPath));  

     job.setNumReduceTasks(1); 
     System.exit(job.waitForCompletion(true) ? 0 : 1); 
     return 0; 

    } 

    public static void main(String[] args) throws Exception { 
     int exitCode = ToolRunner.run(new RedFilter(), args); 
     System.exit(exitCode); 
    } 

    public static void removeDir(String path, Configuration conf) throws IOException { 
     Path output_path = new Path(path); 

     FileSystem fs = FileSystem.get(conf); 

     if (fs.exists(output_path)) { 
      fs.delete(output_path, true); 
     } 
    } 


} 

} 

下面是從作業服務器

{ 

stdout logs 


stderr logs 
2014-11-23 23:25:19.979 java[6176:1003] Unable to load realm info from SCDynamicStore 


syslog logs 
2014-11-23 23:25:20,740 WARN org.apache.hadoop.util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 
2014-11-23 23:25:21,569 WARN org.apache.hadoop.metrics2.impl.MetricsSystemImpl: Source name ugi already exists! 
2014-11-23 23:25:21,727 INFO org.apache.hadoop.mapred.Task: Using ResourceCalculatorPlugin : null 
2014-11-23 23:25:21,759 INFO org.apache.hadoop.mapred.MapTask: Processing split: hdfs://localhost/user/oladotunopasina/redfilterinput/31b5ea5d982cf2b8b4ce27744d812d285b9e3.jpg:0+684033 
2014-11-23 23:25:21,778 INFO org.apache.hadoop.mapred.MapTask: io.sort.mb = 100 
2014-11-23 23:25:22,119 INFO org.apache.hadoop.mapred.MapTask: data buffer = 79691776/99614720 
2014-11-23 23:25:22,119 INFO org.apache.hadoop.mapred.MapTask: record buffer = 262144/327680 
2014-11-23 23:25:22,222 INFO org.apache.hadoop.mapred.TaskLogsTruncater: Initializing logs' truncater with mapRetainSize=-1 and reduceRetainSize=-1 
2014-11-23 23:25:22,481 WARN org.apache.hadoop.mapred.Child: Error running child 
java.lang.NullPointerException 
    at org.apache.hadoop.io.serializer.SerializationFactory.getSerializer(SerializationFactory.java:73) 
    at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.<init>(MapTask.java:970) 
    at org.apache.hadoop.mapred.MapTask$NewOutputCollector.<init>(MapTask.java:673) 
    at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:756) 
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:364) 
    at org.apache.hadoop.mapred.Child$4.run(Child.java:255) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.security.auth.Subject.doAs(Subject.java:394) 
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190) 
    at org.apache.hadoop.mapred.Child.main(Child.java:249) 
2014-11-23 23:25:22,485 INFO org.apache.hadoop.mapred.Task: Runnning cleanup for the task 

} 

好心幫日誌結果。

Answer 1

我發現一個錯誤行：

job.setOutputValueClass(BooleanWritable.class); 

我認爲，當它不能找到一個序列化的Boolean對象Hadoop的失敗：

job.setOutputValueClass(Boolean.class); 

你應該更換。