春數據 - Hadoop的連接

Hadoop的1.1.2，3.2.4春春-數據的Hadoop 1.0執行遠程羣集上MR代碼嘗試Spring Data - Hadoop。 0

我的bean配置文件， 的applicationContext.xml如下：

 <?xml version="1.0" encoding="UTF-8"?> 
<beans xmlns="http://www.springframework.org/schema/beans" 
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hdp="http://www.springframework.org/schema/hadoop" 
    xmlns:context="http://www.springframework.org/schema/context" 
    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd 
    http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd 
    http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.2.xsd"> 

    <context:property-placeholder location="resources/hadoop.properties" /> 

    <hdp:configuration file-system-uri="${hd.fs}" job-tracker-uri="${hd.jobtracker.uri}"> 

    </hdp:configuration> 

    <hdp:job id="wc-job" mapper="com.hadoop.basics.WordCounter.WCMapper" 
     reducer="com.hadoop.basics.WordCounter.WCReducer" input-path="${wordcount.input.path}" 
     output-path="${wordcount.output.path}" user="bigdata"> 
    </hdp:job> 

    <hdp:job-runner id="myjobs-runner" job-ref="wc-job" 
     run-at-startup="true" /> 

    <hdp:resource-loader id="resourceLoader" uri="${hd.fs}" 
     user="bigdata" /> 
</beans>

hadoop.properties

hd.fs=hdfs://cloudx-843-770:9000 
hd.jobtracker.uri=cloudx-843-770:9001 

wordcount.input.path=/scratchpad/input/Childhood_days.txt 
wordcount.output.path=/scratchpad/output

，我正在做的Java類 '運行方式...'

package com.hadoop.basics; 

import java.io.IOException; 
import java.util.StringTokenizer; 

import org.apache.hadoop.io.IntWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Mapper; 
import org.apache.hadoop.mapreduce.Reducer; 
import org.springframework.context.support.AbstractApplicationContext; 
import org.springframework.context.support.ClassPathXmlApplicationContext; 

public class WordCounter { 

    private static IntWritable one = new IntWritable(1); 

    public class WCMapper extends Mapper<Text, Text, Text, IntWritable> { 

     @Override 
     protected void map(
       Text key, 
       Text value, 
       org.apache.hadoop.mapreduce.Mapper<Text, Text, Text, IntWritable>.Context context) 
       throws IOException, InterruptedException { 
      // TODO Auto-generated method stub 
      StringTokenizer strTokenizer = new StringTokenizer(value.toString()); 
      Text token = new Text(); 

      while (strTokenizer.hasMoreTokens()) { 
       token.set(strTokenizer.nextToken()); 
       context.write(token, one); 
      } 

     } 
    } 

    public class WCReducer extends 
      Reducer<Text, IntWritable, Text, IntWritable> { 
     @Override 
     protected void reduce(
       Text key, 
       Iterable<IntWritable> values, 
       org.apache.hadoop.mapreduce.Reducer<Text, IntWritable, Text, IntWritable>.Context context) 
       throws IOException, InterruptedException { 
      // TODO Auto-generated method stub 

      int sum = 0; 

      for (IntWritable value : values) { 
       sum += value.get(); 
      } 

      context.write(key, new IntWritable(sum)); 
     } 
    } 

    public static void main(String[] args) { 
     AbstractApplicationContext context = new ClassPathXmlApplicationContext(
       "applicationContext.xml", WordCounter.class); 
     System.out.println("Word Count Application Running"); 
     context.registerShutdownHook(); 
    } 
}

輸出是：

Aug 22, 2013 9:59:02 AM org.springframework.context.support.AbstractApplicationContext prepareRefresh 
INFO: Refreshing org[email protected]1815338: startup date [Thu Aug 22 09:59:02 IST 2013]; root of context hierarchy 
Aug 22, 2013 9:59:03 AM org.springframework.beans.factory.xml.XmlBeanDefinitionReader loadBeanDefinitions 
INFO: Loading XML bean definitions from class path resource [com/hadoop/basics/applicationContext.xml] 
Aug 22, 2013 9:59:03 AM org.springframework.core.io.support.PropertiesLoaderSupport loadProperties 
INFO: Loading properties file from class path resource [resources/hadoop.properties] 
Aug 22, 2013 9:59:03 AM org.springframework.beans.factory.support.DefaultListableBeanFactory preInstantiateSingletons 
INFO: Pre-instantiating singletons in org.s[email protected]7c197e: defining beans [org.springframework.context.support.PropertySourcesPlaceholderConfigurer#0,hadoopConfiguration,wc-job,myjobs-runner,resourceLoader]; root of factory hierarchy 
Aug 22, 2013 9:59:03 AM org.springframework.data.hadoop.mapreduce.JobExecutor$2 run 
INFO: Starting job [wc-job] 
Aug 22, 2013 9:59:03 AM org.apache.hadoop.security.UserGroupInformation doAs 
SEVERE: PriviledgedActionException as:bigdata via 298790 cause:org.apache.hadoop.ipc.RemoteException: User: 298790 is not allowed to impersonate bigdata 
Aug 22, 2013 9:59:03 AM org.springframework.data.hadoop.mapreduce.JobExecutor$2 run 
WARNING: Cannot start job [wc-job] 
org.apache.hadoop.ipc.RemoteException: User: 298790 is not allowed to impersonate bigdata 
    at org.apache.hadoop.ipc.Client.call(Client.java:1107) 
    at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:229) 
    at org.apache.hadoop.mapred.$Proxy2.getProtocolVersion(Unknown Source) 
    at org.apache.hadoop.ipc.RPC.getProxy(RPC.java:411) 
    at org.apache.hadoop.mapred.JobClient.createRPCProxy(JobClient.java:499) 
    at org.apache.hadoop.mapred.JobClient.init(JobClient.java:490) 
    at org.apache.hadoop.mapred.JobClient.<init>(JobClient.java:473) 
    at org.apache.hadoop.mapreduce.Job$1.run(Job.java:513) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.security.auth.Subject.doAs(Unknown Source) 
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149) 
    at org.apache.hadoop.mapreduce.Job.connect(Job.java:511) 
    at org.apache.hadoop.mapreduce.Job.submit(Job.java:499) 
    at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:530) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor$2.run(JobExecutor.java:197) 
    at org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:49) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor.startJobs(JobExecutor.java:168) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor.startJobs(JobExecutor.java:160) 
    at org.springframework.data.hadoop.mapreduce.JobRunner.call(JobRunner.java:52) 
    at org.springframework.data.hadoop.mapreduce.JobRunner.afterPropertiesSet(JobRunner.java:44) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1541) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1479) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:521) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:458) 
    at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:295) 
    at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:223) 
    at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:292) 
    at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:194) 
    at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:628) 
    at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:932) 
    at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:479) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:197) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:172) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:158) 
    at com.hadoop.basics.WordCounter.main(WordCounter.java:58) 

Aug 22, 2013 9:59:03 AM org.springframework.beans.factory.support.DefaultSingletonBeanRegistry destroySingletons 
INFO: Destroying singletons in org.s[email protected]7c197e: defining beans [org.springframework.context.support.PropertySourcesPlaceholderConfigurer#0,hadoopConfiguration,wc-job,myjobs-runner,resourceLoader]; root of factory hierarchy 
Exception in thread "main" org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'myjobs-runner': Invocation of init method failed; nested exception is java.lang.IllegalStateException: org.apache.hadoop.ipc.RemoteException: User: 298790 is not allowed to impersonate bigdata 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1482) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:521) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:458) 
    at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:295) 
    at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:223) 
    at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:292) 
    at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:194) 
    at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:628) 
    at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:932) 
    at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:479) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:197) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:172) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:158) 
    at com.hadoop.basics.WordCounter.main(WordCounter.java:58) 
Caused by: java.lang.IllegalStateException: org.apache.hadoop.ipc.RemoteException: User: 298790 is not allowed to impersonate bigdata 
    at org.springframework.data.hadoop.mapreduce.JobExecutor$2.run(JobExecutor.java:209) 
    at org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:49) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor.startJobs(JobExecutor.java:168) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor.startJobs(JobExecutor.java:160) 
    at org.springframework.data.hadoop.mapreduce.JobRunner.call(JobRunner.java:52) 
    at org.springframework.data.hadoop.mapreduce.JobRunner.afterPropertiesSet(JobRunner.java:44) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1541) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1479) 
    ... 13 more 
Caused by: org.apache.hadoop.ipc.RemoteException: User: 298790 is not allowed to impersonate bigdata 
    at org.apache.hadoop.ipc.Client.call(Client.java:1107) 
    at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:229) 
    at org.apache.hadoop.mapred.$Proxy2.getProtocolVersion(Unknown Source) 
    at org.apache.hadoop.ipc.RPC.getProxy(RPC.java:411) 
    at org.apache.hadoop.mapred.JobClient.createRPCProxy(JobClient.java:499) 
    at org.apache.hadoop.mapred.JobClient.init(JobClient.java:490) 
    at org.apache.hadoop.mapred.JobClient.<init>(JobClient.java:473) 
    at org.apache.hadoop.mapreduce.Job$1.run(Job.java:513) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.security.auth.Subject.doAs(Unknown Source) 
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149) 
    at org.apache.hadoop.mapreduce.Job.connect(Job.java:511) 
    at org.apache.hadoop.mapreduce.Job.submit(Job.java:499) 
    at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:530) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor$2.run(JobExecutor.java:197) 
    ... 20 more

很顯然，用戶298790（我的本地Windows機器用戶）在集羣上無法識別 - 這就是爲什麼在配置中。文件

我在文檔中提到的作業配置中指定了user =「bigdata」。
The doc。還提到：

SHDP服從HDFS權限，使用當前用戶（缺省情況）的標識，用於與文件系統交互。特別是， HdfsResourceLoader考慮在進行模式匹配時，只有文件認爲它看起來並沒有執行任何特權操作。但是，可以指定不同的用戶，這意味着資源加載使用該用戶的權利與HDFS進行交互 - 然而，這服從用戶模擬規則按照該api，我決定使用HdfsResourceLoader，但找不到任何例子甚至在文檔中配置 - 任何人都可以提供任何指針？

按Hadoop Secure Impersonation，我相信，我需要我的Windows用戶298790添加到遠程集羣機器（Ubuntu的）用戶組，也我覺得這在很大程度上沒有的情況下不可行我的Windows主機名。的用戶和更改Windows客戶端計算機。如果我的假設是正確的，可以做些什麼來避免添加和配置所有這些用戶？

/添加/

<?xml version="1.0"?> 
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> 

<!-- Put site-specific property overrides in this file. --> 

<configuration> 

<property> 
<name>fs.default.name</name> 
<value>hdfs://cloudx-843-770:9000</value> 
</property> 

<property> 
<name>hadoop.proxyuser.298790.groups</name> 
<value>bigdata</value> 
<description>Allow the superuser bigdatato impersonate any members of the group bigdata</description> 
</property> 

<property> 
<name>hadoop.proxyuser.298790.hosts</name> 
<value>*</value> 
<description>The superuser can connect only from INFVA03351 to impersonate a user</description> 
</property> 

</configuration>

我重新啓動所有Hadoop的過程，但錯誤仍然存在更改到核心site.xml中。

然後，我決定創建一個新用戶即viz。298790遠程Ubuntu的機器上，並把它添加到組bigdata用於模擬目的：

[email protected]:/home/bigdata# useradd -G bigdata 298790 
    [email protected]:/home/bigdata# 
    [email protected]:/home/bigdata# 
    [email protected]:/home/bigdata# usermod -G bigdata 298790 
    [email protected]:/home/bigdata# 
    [email protected]:/home/bigdata# su 298790 
    $ groups 
    298790 bigdata 
[email protected]:/home/bigdata# 
[email protected]:/home/bigdata# cat /etc/passwd 
root:x:0:0:root:/root:/bin/bash 
daemon:x:1:1:daemon:/usr/sbin:/bin/sh 
bin:x:2:2:bin:/bin:/bin/sh 
sys:x:3:3:sys:/dev:/bin/sh 
sync:x:4:65534:sync:/bin:/bin/sync 
games:x:5:60:games:/usr/games:/bin/sh 
man:x:6:12:man:/var/cache/man:/bin/sh 
lp:x:7:7:lp:/var/spool/lpd:/bin/sh 
mail:x:8:8:mail:/var/mail:/bin/sh 
news:x:9:9:news:/var/spool/news:/bin/sh 
uucp:x:10:10:uucp:/var/spool/uucp:/bin/sh 
proxy:x:13:13:proxy:/bin:/bin/sh 
www-data:x:33:33:www-data:/var/www:/bin/sh 
backup:x:34:34:backup:/var/backups:/bin/sh 
list:x:38:38:Mailing List Manager:/var/list:/bin/sh 
irc:x:39:39:ircd:/var/run/ircd:/bin/sh 
gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/bin/sh 
nobody:x:65534:65534:nobody:/nonexistent:/bin/sh 
libuuid:x:100:101::/var/lib/libuuid:/bin/sh 
syslog:x:101:103::/home/syslog:/bin/false 
mysql:x:102:105:MySQL Server,,,:/nonexistent:/bin/false 
messagebus:x:103:106::/var/run/dbus:/bin/false 
whoopsie:x:104:107::/nonexistent:/bin/false 
landscape:x:105:110::/var/lib/landscape:/bin/false 
sshd:x:106:65534::/var/run/sshd:/usr/sbin/nologin 
tomcat6:x:107:113::/usr/share/tomcat6:/bin/false 
coesystem:x:1000:1000:coesystem,,,:/home/coesystem:/bin/bash 
hpcc:x:999:1001:hpcc Runtime User:/home/hpcc:/bin/sh 
hduser:x:1001:1002:hduser,1,1,1,1:/home/hduser:/bin/bash 
bigdata:x:1002:1003:Big Data,1,1,1,1:/home/bigdata:/bin/bash 
298790:x:1003:1004::/home/298790:/bin/sh

但現在，當我試圖停止（再啓動）羣集，它要求密碼的所有進程：

[email protected]:~/hadoop_ecosystem/apache_hadoop/hadoop-1.1.2/bin$ stop-all.sh 
Warning: $HADOOP_HOME is deprecated. 

stopping jobtracker 
[email protected]'s password: 
localhost: stopping tasktracker 
stopping namenode 
[email protected]'s password: 
localhost: stopping datanode 
[email protected]'s password: 
localhost: stopping secondarynamenode

而現在的誤差稍微修改 - 它第一次連接失敗，然後冒充：

Aug 22, 2013 5:14:17 PM org.springframework.context.support.AbstractApplicationContext prepareRefresh 
INFO: Refreshing org[email protected]922804: startup date [Thu Aug 22 17:14:17 IST 2013]; root of context hierarchy 
Aug 22, 2013 5:14:17 PM org.springframework.beans.factory.xml.XmlBeanDefinitionReader loadBeanDefinitions 
INFO: Loading XML bean definitions from class path resource [com/hadoop/basics/applicationContext.xml] 
Aug 22, 2013 5:14:17 PM org.springframework.core.io.support.PropertiesLoaderSupport loadProperties 
INFO: Loading properties file from class path resource [resources/hadoop.properties] 
Aug 22, 2013 5:14:17 PM org.springframework.beans.factory.support.DefaultListableBeanFactory preInstantiateSingletons 
INFO: Pre-instantiating singletons in org.s[email protected]7c197e: defining beans [org.springframework.context.support.PropertySourcesPlaceholderConfigurer#0,hadoopConfiguration,wc-job,myjobs-runner,resourceLoader]; root of factory hierarchy 
Aug 22, 2013 5:14:18 PM org.springframework.data.hadoop.mapreduce.JobExecutor$2 run 
INFO: Starting job [wc-job] 
Aug 22, 2013 5:14:20 PM org.apache.hadoop.ipc.Client$Connection handleConnectionFailure 
INFO: Retrying connect to server: cloudx-843-770/172.25.37.135:9001. Already tried 0 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1 SECONDS) 
Aug 22, 2013 5:14:22 PM org.apache.hadoop.ipc.Client$Connection handleConnectionFailure 
INFO: Retrying connect to server: cloudx-843-770/172.25.37.135:9001. Already tried 1 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1 SECONDS) 
Aug 22, 2013 5:14:24 PM org.apache.hadoop.ipc.Client$Connection handleConnectionFailure 
INFO: Retrying connect to server: cloudx-843-770/172.25.37.135:9001. Already tried 2 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1 SECONDS) 
Aug 22, 2013 5:14:26 PM org.apache.hadoop.ipc.Client$Connection handleConnectionFailure 
INFO: Retrying connect to server: cloudx-843-770/172.25.37.135:9001. Already tried 3 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1 SECONDS) 
Aug 22, 2013 5:14:36 PM org.apache.hadoop.security.UserGroupInformation doAs 
SEVERE: PriviledgedActionException as:bigdata via 298790 cause:org.apache.hadoop.ipc.RemoteException: User: 298790 is not allowed to impersonate bigdata 
Aug 22, 2013 5:14:36 PM org.springframework.data.hadoop.mapreduce.JobExecutor$2 run 
WARNING: Cannot start job [wc-job] 
org.apache.hadoop.ipc.RemoteException: User: 298790 is not allowed to impersonate bigdata 
    at org.apache.hadoop.ipc.Client.call(Client.java:1107) 
    at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:229) 
    at org.apache.hadoop.mapred.$Proxy2.getProtocolVersion(Unknown Source) 
    at org.apache.hadoop.ipc.RPC.getProxy(RPC.java:411) 
    at org.apache.hadoop.mapred.JobClient.createRPCProxy(JobClient.java:499) 
    at org.apache.hadoop.mapred.JobClient.init(JobClient.java:490) 
    at org.apache.hadoop.mapred.JobClient.<init>(JobClient.java:473) 
    at org.apache.hadoop.mapreduce.Job$1.run(Job.java:513) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.security.auth.Subject.doAs(Unknown Source) 
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149) 
    at org.apache.hadoop.mapreduce.Job.connect(Job.java:511) 
    at org.apache.hadoop.mapreduce.Job.submit(Job.java:499) 
    at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:530) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor$2.run(JobExecutor.java:197) 
    at org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:49) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor.startJobs(JobExecutor.java:168) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor.startJobs(JobExecutor.java:160) 
    at org.springframework.data.hadoop.mapreduce.JobRunner.call(JobRunner.java:52) 
    at org.springframework.data.hadoop.mapreduce.JobRunner.afterPropertiesSet(JobRunner.java:44) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1541) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1479) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:521) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:458) 
    at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:295) 
    at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:223) 
    at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:292) 
    at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:194) 
    at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:628) 
    at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:932) 
    at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:479) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:197) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:172) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:158) 
    at com.hadoop.basics.WordCounter.main(WordCounter.java:58) 

Aug 22, 2013 5:14:36 PM org.springframework.beans.factory.support.DefaultSingletonBeanRegistry destroySingletons 
INFO: Destroying singletons in org.s[email protected]7c197e: defining beans [org.springframework.context.support.PropertySourcesPlaceholderConfigurer#0,hadoopConfiguration,wc-job,myjobs-runner,resourceLoader]; root of factory hierarchy 
Exception in thread "main" org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'myjobs-runner': Invocation of init method failed; nested exception is java.lang.IllegalStateException: org.apache.hadoop.ipc.RemoteException: User: 298790 is not allowed to impersonate bigdata 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1482) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:521) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:458) 
    at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:295) 
    at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:223) 
    at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:292) 
    at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:194) 
    at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:628) 
    at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:932) 
    at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:479) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:197) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:172) 
    at org.springframework.context.support.ClassPathXmlApplicationContext.<init>(ClassPathXmlApplicationContext.java:158) 
    at com.hadoop.basics.WordCounter.main(WordCounter.java:58) 
Caused by: java.lang.IllegalStateException: org.apache.hadoop.ipc.RemoteException: User: 298790 is not allowed to impersonate bigdata 
    at org.springframework.data.hadoop.mapreduce.JobExecutor$2.run(JobExecutor.java:209) 
    at org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:49) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor.startJobs(JobExecutor.java:168) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor.startJobs(JobExecutor.java:160) 
    at org.springframework.data.hadoop.mapreduce.JobRunner.call(JobRunner.java:52) 
    at org.springframework.data.hadoop.mapreduce.JobRunner.afterPropertiesSet(JobRunner.java:44) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1541) 
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1479) 
    ... 13 more 
Caused by: org.apache.hadoop.ipc.RemoteException: User: 298790 is not allowed to impersonate bigdata 
    at org.apache.hadoop.ipc.Client.call(Client.java:1107) 
    at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:229) 
    at org.apache.hadoop.mapred.$Proxy2.getProtocolVersion(Unknown Source) 
    at org.apache.hadoop.ipc.RPC.getProxy(RPC.java:411) 
    at org.apache.hadoop.mapred.JobClient.createRPCProxy(JobClient.java:499) 
    at org.apache.hadoop.mapred.JobClient.init(JobClient.java:490) 
    at org.apache.hadoop.mapred.JobClient.<init>(JobClient.java:473) 
    at org.apache.hadoop.mapreduce.Job$1.run(Job.java:513) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.security.auth.Subject.doAs(Unknown Source) 
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149) 
    at org.apache.hadoop.mapreduce.Job.connect(Job.java:511) 
    at org.apache.hadoop.mapreduce.Job.submit(Job.java:499) 
    at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:530) 
    at org.springframework.data.hadoop.mapreduce.JobExecutor$2.run(JobExecutor.java:197) 
    ... 20 more

來源

2013-08-21 Kaliyug Antagonist

如果您希望將用戶模擬爲另一個用戶，Apache Hadoop要求您的服務端配置允許這樣的事情。

這意味着，如果你正在運行的「富」，而要真正地提交作業爲「欄」，那麼你的NameNode/JobTracker的要求其加載核心site.xml的配置被允許「富「到代理其他用戶，並且通常類似於下面必須存在於的NameNode的/ JobTracker的核心-site.xml中：

<property> 
    <name>hadoop.proxyuser.foo.groups</name> 
    <value>*</value> 
</property> 

<property> 
    <name>hadoop.proxyuser.foo.hosts</name> 
    <value>*</value> 
</property>

這將允許用戶foo模仿任何其他用戶（*團體），以及從任何主機提交時（*爲主機）。

雖然它不是必需，但肯定建議定義的用戶和組需要存在於NameNode上，以便以正確的方式工作，如權限，組解析等。更多在這裏：http://www.cloudera.com/blog/2012/03/authorization-and-authentication-in-hadoop/

http://static.springsource.org/spring-hadoop/docs/1.0.x/reference/html/security.html#security:kerberos的文件可以更清楚地解決這個問題。

來源

2013-08-22 10:35:04

已編輯帖子以列出不同的嘗試。我承認它已經變得龐大，但我想以一種明確的方式提及這些步驟 - 您能否提供任何投入？ –

我的帖子仍然有效。請再讀一遍。你想運行作業AS bigdata，但你的程序RUNS爲298790，因此298790是你必須允許代理其他用戶的用戶，而不是其他方式。 –

是的，它的工作！目前，尚未在羣集上創建298790，但僅在core-site.xml中提到了它。雖然進一步錯誤，至少模仿是通過！ –

春數據 - Hadoop的連接

回答

相關問題