diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java index 209ba69..85fd32a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java @@ -130,6 +130,7 @@ void setLinkCountCmdTemplate(String[] template) { */ @Override String[] linkCount(File file) throws IOException { + Shell.getWinutilsPathStrict(); String[] buf = new String[getLinkCountCommand.length]; System.arraycopy(getLinkCountCommand, 0, buf, 0, getLinkCountCommand.length); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index e426955..2dace9f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -19,6 +19,7 @@ import java.io.BufferedReader; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.io.InputStream; @@ -42,17 +43,17 @@ * df. It also offers facilities to gate commands by * time-intervals. */ [email protected]({"HDFS", "MapReduce"}) [email protected] [email protected] [email protected] abstract public class Shell { public static final Log LOG = LogFactory.getLog(Shell.class); - - private static boolean IS_JAVA7_OR_ABOVE = - System.getProperty("java.version").substring(0, 3).compareTo("1.7") >= 0; + + private static final String WINDOWS_PROBLEMS = "https://wiki.apache.org/hadoop/WindowsProblems"; + private static final String WINUTILS_EXE = "winutils.exe"; public static boolean isJava7OrAbove() { - return IS_JAVA7_OR_ABOVE; + return true; } /** @@ -308,7 +309,7 @@ private static String checkHadoopHome() { try { // couldn't find either setting for hadoop's home directory if (home == null) { - throw new IOException("HADOOP_HOME or hadoop.home.dir are not set."); + throw new FileNotFoundException("HADOOP_HOME or hadoop.home.dir are not set."); } if (home.startsWith("\"") && home.endsWith("\"")) { @@ -318,7 +319,7 @@ private static String checkHadoopHome() { // check that the home setting is actually a directory that exists File homedir = new File(home); if (!homedir.isAbsolute() || !homedir.exists() || !homedir.isDirectory()) { - throw new IOException("Hadoop home directory " + homedir + throw new FileNotFoundException("Hadoop home directory " + homedir + " does not exist, is not a directory, or is not an absolute path."); } @@ -335,55 +336,144 @@ private static String checkHadoopHome() { } private static String HADOOP_HOME_DIR = checkHadoopHome(); + /** + * Optionally extend an error message with some OS-specific text + * @param message core error message + * @return error message, possibly with some extra text + */ + private static String addOsText(String message) { + return WINDOWS ? (message + " -see " + WINDOWS_PROBLEMS) : message; + } + // Public getter, throws an exception if HADOOP_HOME failed validation // checks and is being referenced downstream. - public static final String getHadoopHome() throws IOException { + public static String getHadoopHome() throws IOException { if (HADOOP_HOME_DIR == null) { - throw new IOException("Misconfigured HADOOP_HOME cannot be referenced."); + throw new FileNotFoundException("Misconfigured HADOOP_HOME cannot be referenced."); } return HADOOP_HOME_DIR; } - /** fully qualify the path to a binary that should be in a known hadoop + /** + * Fully qualify the path to a binary that should be in a known hadoop * bin location. This is primarily useful for disambiguating call-outs * to executable sub-components of Hadoop to avoid clashes with other * executables that may be in the path. Caveat: this call doesn't * just format the path to the bin directory. It also checks for file * existence of the composed path. The output of this call should be * cached by callers. - * */ - public static final String getQualifiedBinPath(String executable) - throws IOException { + * + * @param executable executable + * @return executable file reference + * @throws FileNotFoundException if the path does not exist + */ + public static File getQualifiedBin(String executable) + throws FileNotFoundException { // construct hadoop bin path to the specified executable - String fullExeName = HADOOP_HOME_DIR + File.separator + "bin" - + File.separator + executable; + if (HADOOP_HOME_DIR == null) { + throw new FileNotFoundException(addOsText("No HADOOP_HOME_DIR.")); + } + + String fullExeName = HADOOP_HOME_DIR + File.separator + "bin" + + File.separator + executable; File exeFile = new File(fullExeName); if (!exeFile.exists()) { - throw new IOException("Could not locate executable " + fullExeName - + " in the Hadoop binaries."); + throw new FileNotFoundException(addOsText("Could not locate executable " + + fullExeName + " in the Hadoop binaries.")); } + return exeFile; + } - return exeFile.getCanonicalPath(); + /** + * Fully qualify the path to a binary that should be in a known hadoop + * bin location. This is primarily useful for disambiguating call-outs + * to executable sub-components of Hadoop to avoid clashes with other + * executables that may be in the path. Caveat: this call doesn't + * just format the path to the bin directory. It also checks for file + * existence of the composed path. The output of this call should be + * cached by callers. + * + * @param executable executable + * @return executable file reference + * @throws FileNotFoundException if the path does not exist + * @throws IOException on path canonicalization failures + */ + public static String getQualifiedBinPath(String executable) + throws IOException { + return getQualifiedBin(executable).getCanonicalPath(); } - /** a Windows utility to emulate Unix commands */ - public static final String WINUTILS = getWinUtilsPath(); + /** + * Location of winutils as a string; null if not found. + *

+ * Important: caller must check for this value being null. + * The lack of such checks has led to many support issues being raised. + *

+ * It is better to use one of the exception-raising getter methods, + * specifically {@link #getWinutilsPathOrRTE()} or + * {@link #getWinutilsPathStrict()} + */ + public static final String WINUTILS; - public static final String getWinUtilsPath() { - String winUtilsPath = null; + /** the exception raised on a failure to init the WINUTILS field */ + private static final IOException WINUTILS_FAILURE; + static { + File exe; + IOException ioe; + String path; try { - if (WINDOWS) { - winUtilsPath = getQualifiedBinPath("winutils.exe"); - } - } catch (IOException ioe) { - LOG.error("Failed to locate the winutils binary in the hadoop binary path", - ioe); + exe = getQualifiedBin(WINUTILS_EXE); + path = exe.getCanonicalPath(); + ioe = null; + } catch (IOException e) { + exe = null; + path = null; + ioe = e; } + WINUTILS = path; + WINUTILS_FAILURE = ioe; + } - return winUtilsPath; + /** + * Get the winutils path. + * @return the path to winutils: or "null" if the path could not be found. + * @deprecated Unless the caller implements their own checks, use one of the stricter + * methods to retrieve the value. + */ + public static String getWinUtilsPath() { + return WINUTILS; + } + + /** + * Locate the winutils binary, or fail with a meaningful + * exception and stack trace as an RTE + * @return the path to {@link #WINUTILS_EXE} + */ + public static String getWinutilsPathOrRTE() { + if (WINUTILS_FAILURE == null) { + return WINUTILS; + } else { + throw new RuntimeException(WINUTILS_FAILURE.toString(), + WINUTILS_FAILURE); + } + } + + /** + * Locate the winutils binary, or fail with a meaningful + * exception and stack trace as an IOException + * @return the path to {@link #WINUTILS_EXE} + * @throws IOException if the path is not resolvable + */ + public static String getWinutilsPathStrict() throws IOException { + if (WINUTILS_FAILURE == null) { + return WINUTILS; + } else { + throw new IOException(WINUTILS_FAILURE.toString(), + WINUTILS_FAILURE); + } } public static final boolean isSetsidAvailable = isSetsidSupported(); @@ -734,10 +824,18 @@ public ShellCommandExecutor(String[] execString, File dir, } timeOutInterval = timeout; } - - /** Execute the shell command. */ + /** + * Execute the shell command. + * @throws IOException if the command fails, or if the command is not well constructed. + */ public void execute() throws IOException { + for (String s : command) { + if (s == null) { + throw new IOException("(null) entry in command string: " + + StringUtils.join(" ", command)); + } + } this.run(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java index f3fb364..49f4c2b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java @@ -70,9 +70,9 @@ void reset() { } String getSystemInfoInfoFromShell() { - ShellCommandExecutor shellExecutor = new ShellCommandExecutor( - new String[] {Shell.WINUTILS, "systeminfo" }); try { + ShellCommandExecutor shellExecutor = new ShellCommandExecutor( + new String[] {Shell.getWinutilsPathOrRTE(), "systeminfo" }); shellExecutor.execute(); return shellExecutor.getOutput(); } catch (IOException e) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java index 5b8eac6..e75beeb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java @@ -218,7 +218,7 @@ public void testGetServerSideGroups() throws IOException, } // get the groups pp = Runtime.getRuntime().exec(Shell.WINDOWS ? - Shell.WINUTILS + " groups -F" : "id -Gn"); + Shell.getWinutilsPathStrict() + " groups -F" : "id -Gn"); br = new BufferedReader(new InputStreamReader(pp.getInputStream())); String line = br.readLine(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java index 987c706..ecc0733 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java @@ -59,6 +59,10 @@ public void tearDown() throws IOException { FileUtil.fullyDelete(TEST_DIR); } + private void requireWinutils() throws IOException { + Shell.getWinutilsPathStrict(); + } + // Helper routine that writes the given content to the file. private void writeFile(File file, String content) throws IOException { byte[] data = content.getBytes(); @@ -77,6 +81,7 @@ private String readFile(File file) throws IOException { @Test (timeout = 30000) public void testLs() throws IOException { + requireWinutils(); final String content = "6bytes"; final int contentSize = content.length(); File testFile = new File(TEST_DIR, "file1"); @@ -106,6 +111,7 @@ public void testLs() throws IOException { @Test (timeout = 30000) public void testGroups() throws IOException { + requireWinutils(); String currentUser = System.getProperty("user.name"); // Verify that groups command returns information about the current user @@ -151,6 +157,7 @@ private void assertPermissions(File file, String expected) private void testChmodInternal(String mode, String expectedPerm) throws IOException { + requireWinutils(); File a = new File(TEST_DIR, "file1"); assertTrue(a.createNewFile()); @@ -168,6 +175,7 @@ private void testChmodInternal(String mode, String expectedPerm) } private void testNewFileChmodInternal(String expectedPerm) throws IOException { + requireWinutils(); // Create a new directory File dir = new File(TEST_DIR, "dir1"); @@ -190,6 +198,7 @@ private void testNewFileChmodInternal(String expectedPerm) throws IOException { private void testChmodInternalR(String mode, String expectedPerm, String expectedPermx) throws IOException { + requireWinutils(); // Setup test folder hierarchy File a = new File(TEST_DIR, "a"); assertTrue(a.mkdir()); @@ -226,6 +235,7 @@ private void testChmodInternalR(String mode, String expectedPerm, @Test (timeout = 30000) public void testBasicChmod() throws IOException { + requireWinutils(); // - Create a file. // - Change mode to 377 so owner does not have read permission. // - Verify the owner truly does not have the permissions to read. @@ -278,6 +288,7 @@ public void testBasicChmod() throws IOException { /** Validate behavior of chmod commands on directories on Windows. */ @Test (timeout = 30000) public void testBasicChmodOnDir() throws IOException { + requireWinutils(); // Validate that listing a directory with no read permission fails File a = new File(TEST_DIR, "a"); File b = new File(a, "b"); @@ -356,6 +367,7 @@ public void testBasicChmodOnDir() throws IOException { @Test (timeout = 30000) public void testChmod() throws IOException { + requireWinutils(); testChmodInternal("7", "-------rwx"); testChmodInternal("70", "----rwx---"); testChmodInternal("u-x,g+r,o=g", "-rw-r--r--"); @@ -390,6 +402,7 @@ private void assertOwners(File file, String expectedUser, @Test (timeout = 30000) public void testChown() throws IOException { + requireWinutils(); File a = new File(TEST_DIR, "a"); assertTrue(a.createNewFile()); String username = System.getProperty("user.name"); @@ -415,6 +428,7 @@ public void testChown() throws IOException { @Test (timeout = 30000) public void testSymlinkRejectsForwardSlashesInLink() throws IOException { + requireWinutils(); File newFile = new File(TEST_DIR, "file"); assertTrue(newFile.createNewFile()); String target = newFile.getPath(); @@ -431,6 +445,7 @@ public void testSymlinkRejectsForwardSlashesInLink() throws IOException { @Test (timeout = 30000) public void testSymlinkRejectsForwardSlashesInTarget() throws IOException { + requireWinutils(); File newFile = new File(TEST_DIR, "file"); assertTrue(newFile.createNewFile()); String target = newFile.getPath().replaceAll("\\\\", "/"); @@ -447,6 +462,7 @@ public void testSymlinkRejectsForwardSlashesInTarget() throws IOException { @Test (timeout = 30000) public void testReadLink() throws IOException { + requireWinutils(); // Create TEST_DIR\dir1\file1.txt // File dir1 = new File(TEST_DIR, "dir1"); @@ -529,6 +545,7 @@ public void testReadLink() throws IOException { @SuppressWarnings("deprecation") @Test(timeout=10000) public void testTaskCreate() throws IOException { + requireWinutils(); File batch = new File(TEST_DIR, "testTaskCreate.cmd"); File proof = new File(TEST_DIR, "testTaskCreate.out"); FileWriter fw = new FileWriter(batch); @@ -550,6 +567,7 @@ public void testTaskCreate() throws IOException { @Test (timeout = 30000) public void testTaskCreateWithLimits() throws IOException { + requireWinutils(); // Generate a unique job id String jobId = String.format("%f", Math.random()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java index ebe8df1..8dbab6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java @@ -51,8 +51,11 @@ public static boolean isAvailable() { if (Shell.WINDOWS) { + if (Shell.WINUTILS == null) { + return false; + } ShellCommandExecutor shellExecutor = new ShellCommandExecutor( - new String[] { Shell.WINUTILS, "help" }); + new String[] { Shell.getWinutilsPathOrRTE(), "help" }); try { shellExecutor.execute(); } catch (IOException e) { @@ -75,9 +78,9 @@ public WindowsBasedProcessTree(String pid) { // helper method to override while testing String getAllProcessInfoFromShell() { - ShellCommandExecutor shellExecutor = new ShellCommandExecutor( - new String[] { Shell.WINUTILS, "task", "processList", taskProcessId }); try { + ShellCommandExecutor shellExecutor = new ShellCommandExecutor( + new String[] { Shell.getWinutilsPathStrict(), "task", "processList", taskProcessId }); shellExecutor.execute(); return shellExecutor.getOutput(); } catch (IOException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 68bfbbf..12e1862 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -401,7 +401,7 @@ protected Path getPidFilePath(ContainerId containerId) { cpuRate = Math.min(10000, (int) (containerCpuPercentage * 100)); } } - return new String[] { Shell.WINUTILS, "task", "create", "-m", + return new String[] { Shell.getWinutilsPathOrRTE(), "task", "create", "-m", String.valueOf(memory), "-c", String.valueOf(cpuRate), groupId, "cmd /c " + command }; } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index fd2e31b..e897ffd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -578,7 +578,8 @@ public void setConf(Configuration conf) { LOG.debug(String.format("getRunCommand: %s exists:%b", command, f.exists())); } - return new String[] { Shell.WINUTILS, "task", "createAsUser", groupId, + return new String[] { Shell.getWinutilsPathOrRTE(), "task", + "createAsUser", groupId, userName, pidFile.toString(), "cmd /c " + command }; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index bf00d74..7bbb87b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -652,16 +652,9 @@ protected void link(Path src, Path dst) throws IOException { File srcFile = new File(src.toUri().getPath()); String srcFileStr = srcFile.getPath(); String dstFileStr = new File(dst.toString()).getPath(); - // If not on Java7+ on Windows, then copy file instead of symlinking. - // See also FileUtil#symLink for full explanation. - if (!Shell.isJava7OrAbove() && srcFile.isFile()) { - lineWithLenCheck(String.format("@copy \"%s\" \"%s\"", srcFileStr, dstFileStr)); - errorCheck(); - } else { - lineWithLenCheck(String.format("@%s symlink \"%s\" \"%s\"", Shell.WINUTILS, + lineWithLenCheck(String.format("@%s symlink \"%s\" \"%s\"", Shell.getWinutilsPathOrRTE(), dstFileStr, srcFileStr)); - errorCheck(); - } + errorCheck(); } @Override