idea实现wordcount报错:
D:\hadoop\jdk1.8.0_171\jdk\bin\java.exe "-javaagent:C:\Program Files\JetBrains\IntelliJ IDEA Community Edition 2022.2.3\lib\idea_rt.jar=51656:C:\Program Files\JetBrains\IntelliJ IDEA Community Edition 2022.2.3\bin" -Dfile.encoding=GBK -classpath D:\hadoop\jdk1.8.0_171\jdk\jre\lib\charsets.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\deploy.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\access-bridge-64.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\cldrdata.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\dnsns.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\jaccess.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\jfxrt.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\localedata.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\nashorn.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\sunec.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\sunjce_provider.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\sunmscapi.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\sunpkcs11.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\ext\zipfs.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\javaws.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\jce.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\jfr.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\jfxswt.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\jsse.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\management-agent.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\plugin.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\resources.jar;D:\hadoop\jdk1.8.0_171\jdk\jre\lib\rt.jar;D:\java\WC\out\production\WC;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\xz-1.0.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\asm-3.2.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\avro-1.7.4.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\gson-2.2.4.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\junit-4.11.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jsch-0.1.54.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jsp-api-2.1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\xmlenc-0.52.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\guava-11.0.2.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jets3t-0.9.0.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jettison-1.1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jetty-6.1.26.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jsr305-3.0.0.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\log4j-1.2.17.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\paranamer-2.3.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\activation-1.1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-io-2.4.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\httpcore-4.2.5.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jaxb-api-2.2.2.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\stax-api-1.0-2.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-cli-1.2.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-net-3.1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jersey-core-1.9.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jersey-json-1.9.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\servlet-api-2.5.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\zookeeper-3.4.6.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-lang-2.6.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\httpclient-4.2.5.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\slf4j-api-1.7.10.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-codec-1.4.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\hadoop-auth-2.7.7.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\hamcrest-core-1.3.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jackson-xc-1.9.13.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jaxb-impl-2.2.3-1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jersey-server-1.9.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jetty-util-6.1.26.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\mockito-all-1.8.5.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\netty-3.6.2.Final.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\api-util-1.0.0-M20.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-math3-3.1.1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\java-xmlbuilder-0.4.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\protobuf-java-2.5.0.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\snappy-java-1.0.4.1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-digester-1.8.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\curator-client-2.7.1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jackson-jaxrs-1.9.13.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\slf4j-log4j12-1.7.10.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-logging-1.1.3.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\curator-recipes-2.7.1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\api-asn1-api-1.0.0-M20.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-compress-1.4.1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-httpclient-3.1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jetty-sslengine-6.1.26.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\apacheds-i18n-2.0.0-M15.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-beanutils-1.7.0.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\curator-framework-2.7.1.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jackson-core-asl-1.9.13.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\hadoop-annotations-2.7.7.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-collections-3.2.2.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-configuration-1.6.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\jackson-mapper-asl-1.9.13.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\commons-beanutils-core-1.8.0.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\htrace-core-3.1.0-incubating.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\lib\apacheds-kerberos-codec-2.0.0-M15.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\hdfs\hadoop-hdfs-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\hdfs\hadoop-hdfs-nfs-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\hdfs\hadoop-hdfs-2.7.7-tests.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\mapreduce\hadoop-mapreduce-examples-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\mapreduce\hadoop-mapreduce-client-app-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\mapreduce\hadoop-mapreduce-client-core-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\mapreduce\hadoop-mapreduce-client-common-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\mapreduce\hadoop-mapreduce-client-shuffle-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-plugins-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.7.7-tests.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-api-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-client-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-common-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-registry-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-server-tests-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-server-common-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-server-web-proxy-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-server-nodemanager-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-server-resourcemanager-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-server-sharedcachemanager-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-applications-distributedshell-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-server-applicationhistoryservice-2.7.7.jar;D:\hadoop\wordcount\hadoop-2.7.7\share\hadoop\yarn\hadoop-yarn-applications-unmanaged-am-launcher-2.7.7.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\hadoop-nfs-2.7.7.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\hadoop-common-2.7.7.jar;D:\hadoop\hadoop-2.7.7\share\hadoop\common\hadoop-common-2.7.7-tests.jar main.edu.WordCountMaster input output
2023-05-20 18:58:31,357 ERROR [main] util.Shell (Shell.java:getWinUtilsPath(400)) - Failed to locate the winutils binary in the hadoop binary path
java.io.IOException: Could not locate executable null\bin\winutils.exe in the Hadoop binaries.
at org.apache.hadoop.util.Shell.getQualifiedBinPath(Shell.java:382)
at org.apache.hadoop.util.Shell.getWinUtilsPath(Shell.java:397)
at org.apache.hadoop.util.Shell.(Shell.java:390)
at org.apache.hadoop.util.StringUtils.(StringUtils.java:80)
at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:610)
at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:277)
at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:265)
at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:810)
at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:780)
at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:653)
at org.apache.hadoop.mapreduce.task.JobContextImpl.(JobContextImpl.java:72)
at org.apache.hadoop.mapreduce.Job.(Job.java:142)
at org.apache.hadoop.mapreduce.Job.getInstance(Job.java:185)
at org.apache.hadoop.mapreduce.Job.getInstance(Job.java:204)
at main.edu.WordCountMaster.main(WordCountMaster.java:21)
2023-05-20 18:58:31,557 INFO [main] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1243)) - session.id is deprecated. Instead, use dfs.metrics.session-id
2023-05-20 18:58:31,557 INFO [main] jvm.JvmMetrics (JvmMetrics.java:init(76)) - Initializing JVM Metrics with processName=JobTracker, sessionId=
2023-05-20 18:58:32,133 WARN [main] mapreduce.JobResourceUploader (JobResourceUploader.java:uploadFiles(64)) - Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
2023-05-20 18:58:32,138 WARN [main] mapreduce.JobResourceUploader (JobResourceUploader.java:uploadFiles(171)) - No job jar file set. User classes may not be found. See Job or Job#setJar(String).
2023-05-20 18:58:32,148 INFO [main] mapreduce.JobSubmitter (JobSubmitter.java:submitJobInternal(249)) - Cleaning up the staging area file:/tmp/hadoop-1/mapred/staging/1949437446/.staging/job_local949437446_0001
Exception in thread "main" java.lang.RuntimeException: Error while running command to get file permissions : java.io.IOException: (null) entry in command string: null ls -F D:\java\WC\input\REMOVE.txt
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:772)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:869)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:852)
at org.apache.hadoop.fs.FileUtil.execCommand(FileUtil.java:1108)
at org.apache.hadoop.fs.RawLocalFileSystem$DeprecatedRawLocalFileStatus.loadPermissionInfo(RawLocalFileSystem.java:659)
at org.apache.hadoop.fs.RawLocalFileSystem$DeprecatedRawLocalFileStatus.getPermission(RawLocalFileSystem.java:634)
at org.apache.hadoop.fs.LocatedFileStatus.(LocatedFileStatus.java:47)
at org.apache.hadoop.fs.FileSystem$4.next(FileSystem.java:1732)
at org.apache.hadoop.fs.FileSystem$4.next(FileSystem.java:1712)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:305)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.listStatus(FileInputFormat.java:265)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getSplits(FileInputFormat.java:387)
at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:301)
at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:318)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:196)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1287)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1308)
at main.edu.WordCountMaster.main(WordCountMaster.java:39)
at org.apache.hadoop.fs.RawLocalFileSystem$DeprecatedRawLocalFileStatus.loadPermissionInfo(RawLocalFileSystem.java:697)
at org.apache.hadoop.fs.RawLocalFileSystem$DeprecatedRawLocalFileStatus.getPermission(RawLocalFileSystem.java:634)
at org.apache.hadoop.fs.LocatedFileStatus.<init>(LocatedFileStatus.java:47)
at org.apache.hadoop.fs.FileSystem$4.next(FileSystem.java:1732)
at org.apache.hadoop.fs.FileSystem$4.next(FileSystem.java:1712)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:305)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.listStatus(FileInputFormat.java:265)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getSplits(FileInputFormat.java:387)
at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:301)
at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:318)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:196)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1287)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1308)
at main.edu.WordCountMaster.main(WordCountMaster.java:39)
Process finished with exit code 1
如何解决
以下答案由GPT-4.5Plus大模型(语音)与博主A努力中.......共同编写:
从错误日志可以看出,主要有两个错误:
此错误是由于WordCount程序试图获取input目录下REMOVE.txt文件的权限信息时出现的。解决方法是: