[Java] Compile Hadoop 2.6 for Windows XP 32bit

博客首页 » Java Compile Hadoop 2.6 for Windows XP 32bit

发布于 18 Jan 2016 09:34
标签 blog
Hadoop 2已经支持在Windows上运行,可是Apache官方的2.6版本连Windows上的winutils.exe都不带,需要自己Build。于是只有自己动手丰衣足食。

一般容易入手的是Windows7以上的64bit版本。还有一个Windows XP 32bit的机器用不了很不爽。连JDK8都跑得没问题,怎么就你Hadoop这么不考虑兼容性。于是下载Hadoop源码,几下调试下来,弄出了在Windows XP 32bit上跑的Hadoop包。虽然有symbol link这样的小功能不能用,不过经过测试跑Spark也没什么大问题。现在把踩到的坑记录成小文章。


** 更新记录 **

在用到hadoop.dll的时候,发现了还有几个Windows Vista以上的系统调用,已经修改完毕。修改过程更新放在了最后的部分。 2016/01/19


成果hadoop-common-2.6.3-bin-x86-win32-xp.zip
链接1本地: http://wiki.myoa.cf/local--files/zh-blog:312/hadoop-common-2.6.3-bin-x86-win32-xp.zip
链接2百度云: http://pan.baidu.com/s/1hrbCLmO ,下载密码: a5m7

下载代码配置编译环境

主要是下载Hadoop源码,安装Windows .Net SDK 4 for Windows 7,JDK 1.6以上,Maven 3.1.1以上,CMake,protocbuf,这里就不赘述了,细节参见这个文章。
http://www.srccodes.com/p/article/38/build-install-configure-run-apache-hadoop-2.2.0-microsoft-wi
我用的是Hadoop 2.6.3。

在编译环境都配置好以后,就可以用下面的maven命令编译了。

mvn package -Pdist -Dtar -Dmaven.javadoc.skip=true -DskipTests -fail-at-end -Pnative-win

比起上面的文章,在Hadoop 2.6.3里,有JavaDoc的编写等错误,需要加些选项才能过关。

64bit工程到32bit工程

搜索项目里的sln/vcxproj,把所有的x64和Win64修改成Win32。

hadoop-common-project\hadoop-common\src\main\winutils下
winutils.sln
libwinutils.vcxproj
winutils.vcxproj

hadoop-common-project\hadoop-common\src\main\native下
native.sln
native.vcxproj

修改CMake自动生成,把x64去掉

Maven的自动生成里,有一个Ant的任务是用CMake生成VC的解决方案,需要把x64的指定去掉。

hadoop-hdfs-project\hadoop-hdfs\pom.xml中
找到下面这段代码,把原来Visual Studio 10 64去掉64位指定。

<exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
  <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse} -G 'Visual Studio 10'"/>
</exec>

添加WINAPI函数修饰并去掉_tls_used的外部引用

在apache的mail里有关于这个修改的描述
http://mail-archives.apache.org/mod_mbox/hadoop-user/201502.mbox/%3CCAErxogE3EZuB2iAo_oNT583QxqPt=X5LJRdaBS2k_3WLt47aig@mail.gmail.com%3E

1. hadoop-hdfs-project\hadoop-hdfs\src\main\native\libhdfs\os\windows\thread.c
Line 31: Add "WINAPI" to declaration

 static DWORD WINAPI runThread(LPVOID toRun) {

2. hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/os/windows/thread_local_storage.c
Comment lines 99 and 105

//#pragma comment(linker, "/INCLUDE:_tls_used")
//#pragma comment(linker, "/INCLUDE:pTlsCallback")

把CompareStringEx换成CompareStringW

这是Vista以后才支持的功能,换成老版本的CompareStringW
在hadoop-common-project\hadoop-common\src\main\winutils\service.c中,搜索CompareStringEx,改成这样的代码:

    //compareResult = CompareStringEx(
      //LOCALE_NAME_INVARIANT,
      //NORM_IGNORECASE,
      //localBuffer, gCchLocalDir[crt] <= cchLocalBuffer ? gCchLocalDir[crt] : cchLocalBuffer, 
      //gLocalDirs[crt], gCchLocalDir[crt],
      //NULL, // lpVersionInformation
      //NULL, // lpReserved
      //NULL); // lParam
    compareResult = CompareString(
      LOCALE_INVARIANT,
      NORM_IGNORECASE,
      localBuffer, gCchLocalDir[crt] <= cchLocalBuffer ? gCchLocalDir[crt] : cchLocalBuffer, 
      gLocalDirs[crt], gCchLocalDir[crt]);

这些链接是MSDN对于两个函数的描述。
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317759(v=vs.85).aspx
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317761(v=vs.85).aspx

去掉CreateSymbolicLinkW

这是Vista以后才支持的功能,需要去掉。
这里用递归目录/文件拷贝替代了这个功能,需要注意winutils.exe的symbol link不要用于链接大文件(基本是用来把jar文件和目录放到执行集群内,所以似乎没有问题)
因为Windows XP下的Junction Point行为会导致用del删除symbol link时删除目录内所有内容,而synbol link本身却删除不掉。

是下面这两段代码,分别完成权限检查和调用,都不能调用。

  if (EnablePrivilege(L"SeCreateSymbolicLinkPrivilege") != ERROR_SUCCESS)
  {
    fwprintf(stderr,
      L"No privilege to create symbolic links.\n");
    ret = SYMLINK_NO_PRIVILEGE;
    goto SymlinkEnd;
  }
  if (!CreateSymbolicLinkW(longLinkName, longFileName, dwFlag))
  {
    ReportErrorCode(L"CreateSymbolicLink", GetLastError());
    ret = FAILURE;
    goto SymlinkEnd;
  }

我把整个函数替换成了这个代码。

#include "winutils.h"
#include <direct.h>
 
// replace create symbolic link with copy files/directories
BOOL CopyDir(LPCWSTR from, LPCWSTR to, BOOL bFlag) {
  DWORD dwErrorCode = ERROR_SUCCESS;
 
  BOOL isDir = FALSE;
 
  DWORD dwRtnCode = ERROR_SUCCESS;
 
  int ret = SUCCESS;
 
  WCHAR fromDir[MAX_PATH];
  WCHAR fromPath[MAX_PATH];
  WCHAR toPath[MAX_PATH];
  WIN32_FIND_DATA FindFileData;
  HANDLE hFind;
 
  if ((dwRtnCode = DirectoryCheck(from, &isDir)) != ERROR_SUCCESS)
  {
    ReportErrorCode(L"DirectoryCheck", dwRtnCode);
    ret = FAILURE;
  } 
  else 
  {
    if (isDir)
    {
      ret = _wmkdir(to);
 
      _snwprintf(fromDir, sizeof(fromDir), L"%s\\*.*", from);
 
      hFind = FindFirstFile(fromDir, &FindFileData);
 
      while (hFind != INVALID_HANDLE_VALUE && ret != FAILURE) 
      {
        if (wcsncmp(L".", FindFileData.cFileName, sizeof(FindFileData.cFileName)) != 0 &&
            wcsncmp(L"..", FindFileData.cFileName, sizeof(FindFileData.cFileName)) != 0)
        {
          _snwprintf(fromPath, sizeof(fromPath), L"%s\\%s", from, FindFileData.cFileName);
          _snwprintf(toPath, sizeof(toPath), L"%s\\%s", to, FindFileData.cFileName);
          ret = CopyDir(fromPath, toPath, bFlag);
        }
 
        if (!FindNextFile(hFind, &FindFileData) || ret == FAILURE)
        {
          FindClose(hFind);
          hFind = INVALID_HANDLE_VALUE;
        }
      } // while hFind
    } // isDir
    else if (!(dwRtnCode = CopyFile(from, to, bFlag)))
    {
      ReportErrorCode(L"CopyFile", dwRtnCode);
      ret = FAILURE;
    }
  }
  return ret;
}
 
//----------------------------------------------------------------------------
// Function: Symlink
//
// Description:
//    The main method for symlink command
//
// Returns:
//    0: on success
//
// Notes:
//
int Symlink(__in int argc, __in_ecount(argc) wchar_t *argv[])
{
  PWSTR longLinkName = NULL;
  PWSTR longFileName = NULL;
  DWORD dwErrorCode = ERROR_SUCCESS;
 
  BOOL isDir = FALSE;
 
  DWORD dwRtnCode = ERROR_SUCCESS;
  DWORD dwFlag = 0;
 
  int ret = SUCCESS;
 
  if (argc != 3)
  {
    SymlinkUsage();
    return FAILURE;
  }
 
  dwErrorCode = ConvertToLongPath(argv[1], &longLinkName);
  if (dwErrorCode != ERROR_SUCCESS)
  {
    ret = FAILURE;
    goto SymlinkEnd;
  }
  dwErrorCode = ConvertToLongPath(argv[2], &longFileName);
  if (dwErrorCode != ERROR_SUCCESS)
  {
    ret = FAILURE;
    goto SymlinkEnd;
  }
 
  if (wcschr(longLinkName, L'/') != NULL || wcschr(longFileName, L'/') != NULL)
  {
    // Reject forward-slash separated paths as they result in unusable symlinks.
    //
    fwprintf(stderr,
      L"Rejecting forward-slash separated path which would result in an "
      L"unusable symlink: link = %s, target = %s\n", longLinkName, longFileName);
    ret = FAILURE;
    goto SymlinkEnd;
  }
 
  // Check if the the process's access token has the privilege to create
  // symbolic links. Without this step, the call to CreateSymbolicLink() from
  // users have the privilege to create symbolic links will still succeed.
  // This is just an additional step to do the privilege check by not using
  // error code from CreateSymbolicLink() method.
  //
  // remove invalid syscall for xp
  // if (EnablePrivilege(L"SeCreateSymbolicLinkPrivilege") != ERROR_SUCCESS)
  // {
  //   fwprintf(stderr,
  //     L"No privilege to create symbolic links.\n");
  //   ret = SYMLINK_NO_PRIVILEGE;
  //   goto SymlinkEnd;
  // }
 
  //if ((dwRtnCode = DirectoryCheck(longFileName, &isDir)) != ERROR_SUCCESS)
  //{
  //  ReportErrorCode(L"DirectoryCheck", dwRtnCode);
  //  ret = FAILURE;
  //  goto SymlinkEnd;
  //}
 
  //if (isDir)
  //  dwFlag = SYMBOLIC_LINK_FLAG_DIRECTORY;
  //{
  //  fwprintf(stderr,
  //  L"No privilege to create symbolic links.\n");
  //  ret = SYMLINK_NO_PRIVILEGE;
  //  goto SymlinkEnd;
  //}
 
  // remove invalid syscall for xp
  // if (!CreateSymbolicLinkW(longLinkName, longFileName, dwFlag))
  //{
  //  ReportErrorCode(L"CreateSymbolicLink", GetLastError());
  //  ret = FAILURE;
  //  goto SymlinkEnd;
  //}
  // replace create symbolic link with copy files/directories
  if ((dwRtnCode = CopyDir(longFileName, longLinkName, FALSE)) != ERROR_SUCCESS)
  {
    ret = FAILURE;
    goto SymlinkEnd;
  }
 
SymlinkEnd:
  LocalFree(longLinkName);
  LocalFree(longFileName);
  return ret;
}

关于Windows 2000开始NTFS支持Junction Point目录的软链接,参考以下文章。

http://blog.nsfocus.net/shortcuthard-linkjunction-pointsymbolic-link/?utm_source=tuicool

https://support.microsoft.com/zh-cn/kb/205524
https://technet.microsoft.com/en-us/sysinternals/bb896768.aspx

侦测hadoop.dll中不支持Windows XP的API

写一个简单的程序在Windows XP装载hadoop.dll,这样当有不支持的系统调用时,就会出现对话框报错。

#include "stdafx.h"
 
using namespace std;
 
int _tmain(int argc, _TCHAR* argv[])
{
    HANDLE hModule = LoadLibrary(TEXT("hadoop.dll"));
    if (hModule == 0) {
        UINT errCode = GetLastError();
        printf("Error code: %i\n", errCode);
    }
    cout << "Press ENTER to exit " << endl;
    char ch = cin.get();
    return 0;
}

去掉GetFinalPathNameByHandle调用

GetFinalPathNameByHandle是在libwinutils.c中的FindFileOwnerAndPermissionByHandle函数中被调用的,因为我们大多用Administrator权限在XP上操作,所以去掉检查也问题不大。

/*
  dwRtnCode = GetFinalPathNameByHandle(fileHandle, path, cchPathLen, 0);
  if (dwRtnCode == 0)
  {
    ret = GetLastError();
    goto FindFileOwnerAndPermissionByHandleEnd;
  }
  cchPathLen = dwRtnCode;
  path = (LPWSTR) LocalAlloc(LPTR, cchPathLen * sizeof(WCHAR));
  if (path == NULL)
  {
    ret = GetLastError();
    goto FindFileOwnerAndPermissionByHandleEnd;
  }
 
  dwRtnCode = GetFinalPathNameByHandle(fileHandle, path, cchPathLen, 0);
  if (dwRtnCode != cchPathLen - 1)
  {
    ret = GetLastError();
    goto FindFileOwnerAndPermissionByHandleEnd;
  }
 
  dwRtnCode = FindFileOwnerAndPermission(path, TRUE, pOwnerName, pGroupName, pMask);
  if (dwRtnCode != ERROR_SUCCESS)
  {
    ret = dwRtnCode;
    goto FindFileOwnerAndPermissionByHandleEnd;
  }
 
FindFileOwnerAndPermissionByHandleEnd:
  LocalFree(path);
*/

SetProcessWorkingSetSizeEx换成SetProcessWorkingSetSize

SetProcessWorkingSetSizeEx是在NativeIO.c中的Java_org_apache_hadoop_io_nativeio_NativeIO_00024Windows_extendWorkingSetSize函数中被调用的,可以置换成SetProcessWorkingSetSize完成相应的功能。
 
  //if (!SetProcessWorkingSetSizeEx(hProcess, min + delta, max + delta,
  //    QUOTA_LIMITS_HARDWS_MIN_DISABLE | QUOTA_LIMITS_HARDWS_MAX_DISABLE)) {
  if (!SetProcessWorkingSetSize(hProcess, min + delta, max + delta)) {

本页面的文字允许在知识共享 署名-相同方式共享 3.0协议和GNU自由文档许可证下修改和再使用,仅有一个特殊要求,请用链接方式注明文章引用出处及作者。请协助维护作者合法权益。


系列文章

文章列表

  • Java Compile Hadoop 2.6 for Windows XP 32bit

这篇文章对你有帮助吗,投个票吧?

rating: +1+x

留下你的评论

Add a New Comment