hadoop ClassNotFoundException解决办法
时间:2015-10-08 11:31 来源:linux.it.net.cn 作者:IT
我在测试hadoop时候，自己写了一个类，并且这个类中引用了2个类，运行的时候找不到引用的类。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
StudentinfoRecord这个类确实有，并且和DBInput在同一目录下
hduser@tianbaoxing-virtual-machine:/app/hadoop/tmp/java$ hadoop DBInput
ava$ hadoop DBInputWarning: $HADOOP_HOME is deprecated.
13/04/24 10:41:59 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.
13/04/24 10:41:59 WARN mapred.JobClient: No job jar file set. User classes may not be found. See JobConf(Class) or JobConf#setJar(String).
13/04/24 10:42:00 INFO mapred.JobClient: Running job: job_201304241011_0004
13/04/24 10:42:01 INFO mapred.JobClient: map 0% reduce 0%
13/04/24 10:42:12 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000000_0, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:247)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
... 10 more
13/04/24 10:42:12 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000001_0, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:247)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
... 10 more
13/04/24 10:42:21 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000000_1, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:247)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
... 10 more
13/04/24 10:42:21 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000001_1, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:247)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
... 10 more
13/04/24 10:42:28 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000000_2, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:247)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
... 10 more
13/04/24 10:42:29 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000001_2, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:247)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
... 10 more
13/04/24 10:42:40 INFO mapred.JobClient: Job complete: job_201304241011_0004
13/04/24 10:42:40 INFO mapred.JobClient: Counters: 6
13/04/24 10:42:40 INFO mapred.JobClient: Job Counters
13/04/24 10:42:40 INFO mapred.JobClient: SLOTS_MILLIS_MAPS=70989
13/04/24 10:42:40 INFO mapred.JobClient: Total time spent by all reduces waiting after reserving slots (ms)=0
13/04/24 10:42:40 INFO mapred.JobClient: Total time spent by all maps waiting after reserving slots (ms)=0
13/04/24 10:42:40 INFO mapred.JobClient: Launched map tasks=8
13/04/24 10:42:40 INFO mapred.JobClient: SLOTS_MILLIS_REDUCES=0
13/04/24 10:42:40 INFO mapred.JobClient: Failed map tasks=1
13/04/24 10:42:40 INFO mapred.JobClient: Job Failed: # of failed Map Tasks exceeded allowed limit. FailedCount: 1. LastFailedTask: task_201304241011_0004_m_000000
Exception in thread "main" java.io.IOException: Job failed!
at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1327)
at DBInput.main(DBInput.java:37)
hduser@tianbaoxing-virtual-machine:/app/hadoop/tmp/java$
这个问题找到了，我使用的是1.1.2，需要打包成jar文件运行。然后就没有报错。
程序不是以jar的形式运行的,所以不会上传jar到HDFS中，以致节点外的所有节点在执行task任务时上不能找到map和reduce类，所以在运行task时会出现错误。
同时还要注意命令： hadoop jar DBInput.jar ，中间不能省去jar
还要注意： DBInput.jar必须放在当前的文件下
即时是伪分布式下(一个节点既是主节点又是从节点)下直接在eclipse中也不能运行成功，也需要打包成jar
(责任编辑：IT)
我在测试hadoop时候，自己写了一个类，并且这个类中引用了2个类，运行的时候找不到引用的类。
这个问题找到了，我使用的是1.1.2，需要打包成jar文件运行。然后就没有报错。 程序不是以jar的形式运行的,所以不会上传jar到HDFS中，以致节点外的所有节点在执行task任务时上不能找到map和reduce类，所以在运行task时会出现错误。 同时还要注意命令： hadoop jar DBInput.jar ，中间不能省去jar 还要注意： DBInput.jar必须放在当前的文件下 即时是伪分布式下(一个节点既是主节点又是从节点)下直接在eclipse中也不能运行成功，也需要打包成jar |