0
您好我目前正在編寫一個地圖減少作業,以通過包含圖像和記錄輸出的文件包含紅色或不包含紅色的hdfs。我目前遇到NullPointerException錯誤,我似乎無法找出它來自哪裏。下面是我的代碼空指針異常Hadoop
{
package hipi.examples.redfilter;
import java.awt.Color;
import java.awt.image.BufferedImage;
import java.io.IOException;
import javax.imageio.ImageIO;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class RedFilter extends Configured implements Tool{
private static String inputPath;
public static class RedFilterMapper extends Mapper<NullWritable, BufferedImage, Text, BooleanWritable> {
public void map(IntWritable key, BufferedImage currImage, Context content) throws IOException, InterruptedException
{
System.out.println("I am in the mapper method");
FileSystem fs = FileSystem.get(new Configuration());
FileStatus[] status = fs.listStatus(new Path(inputPath));
System.out.println("I am in the mapper method");
for (int i=0;i<status.length;i++){
String fileName = fs.open(status[i].getPath()).toString();
currImage= ImageIO.read(fs.open(status[i].getPath()));
int width = currImage.getWidth();
int height = currImage.getHeight();
boolean exit = false;
for(int x = 0; x < width; x++) {
for(int y = 0; y < height; y++) {
Color c = new Color(currImage.getRGB(x, y));
int red = c.getRed();
if (red > 200 && c.getBlue() < 100 && c.getGreen() < 100) {
content.write(new Text(fileName), new BooleanWritable(true));
exit = true;
break;
}
}
if(exit) {
break;
}
}
if(!exit) {
content.write(new Text(fileName), new BooleanWritable(false));
}
}
}
}
public int run(String[] args) throws Exception {
Configuration conf = new Configuration();
if (args.length < 3) {
System.out.println("Usage: dumphib <input hib> <outputdir>");
System.exit(0);
}
inputPath = args[1];
String outputPath = args[2];
Job job = new Job(conf, "refilter");
System.out.println("Jar Files r being set");
job.setJarByClass(RedFilter.class);
System.out.println("Mapper is being set");
job.setMapperClass(RedFilterMapper.class);
// Set formats
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Boolean.class);
// job.setInputFormatClass(ImageBundleInputFormat.class);
// Set out/in paths
removeDir(outputPath, conf);
FileOutputFormat.setOutputPath(job, new Path(outputPath));
FileInputFormat.setInputPaths(job, new Path(inputPath));
job.setNumReduceTasks(1);
System.exit(job.waitForCompletion(true) ? 0 : 1);
return 0;
}
public static void main(String[] args) throws Exception {
int exitCode = ToolRunner.run(new RedFilter(), args);
System.exit(exitCode);
}
public static void removeDir(String path, Configuration conf) throws IOException {
Path output_path = new Path(path);
FileSystem fs = FileSystem.get(conf);
if (fs.exists(output_path)) {
fs.delete(output_path, true);
}
}
}
}
下面是從作業服務器
{
stdout logs
stderr logs
2014-11-23 23:25:19.979 java[6176:1003] Unable to load realm info from SCDynamicStore
syslog logs
2014-11-23 23:25:20,740 WARN org.apache.hadoop.util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2014-11-23 23:25:21,569 WARN org.apache.hadoop.metrics2.impl.MetricsSystemImpl: Source name ugi already exists!
2014-11-23 23:25:21,727 INFO org.apache.hadoop.mapred.Task: Using ResourceCalculatorPlugin : null
2014-11-23 23:25:21,759 INFO org.apache.hadoop.mapred.MapTask: Processing split: hdfs://localhost/user/oladotunopasina/redfilterinput/31b5ea5d982cf2b8b4ce27744d812d285b9e3.jpg:0+684033
2014-11-23 23:25:21,778 INFO org.apache.hadoop.mapred.MapTask: io.sort.mb = 100
2014-11-23 23:25:22,119 INFO org.apache.hadoop.mapred.MapTask: data buffer = 79691776/99614720
2014-11-23 23:25:22,119 INFO org.apache.hadoop.mapred.MapTask: record buffer = 262144/327680
2014-11-23 23:25:22,222 INFO org.apache.hadoop.mapred.TaskLogsTruncater: Initializing logs' truncater with mapRetainSize=-1 and reduceRetainSize=-1
2014-11-23 23:25:22,481 WARN org.apache.hadoop.mapred.Child: Error running child
java.lang.NullPointerException
at org.apache.hadoop.io.serializer.SerializationFactory.getSerializer(SerializationFactory.java:73)
at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.<init>(MapTask.java:970)
at org.apache.hadoop.mapred.MapTask$NewOutputCollector.<init>(MapTask.java:673)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:756)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:364)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:394)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
2014-11-23 23:25:22,485 INFO org.apache.hadoop.mapred.Task: Runnning cleanup for the task
}
好心幫日誌結果。
爲什麼'job.setInputFormatClass'被註釋掉了? – blackSmith 2014-11-24 05:22:47