OpenMPI源码剖析1：MPI_Init初探
OpenMPI的底层实现:
我们知道，OpenMPI应用起来还是比较简单的，但是如果让我自己来实现一个MPI的并行计算，你会怎么设计呢？————这就涉及到比较底层的东西了。
回想起我们最简单的代码，通过comm_rank来决定做不同的事情，那么这个comm_rank是怎么得到的呢？
源代码从哪里看起？在百度，谷歌都没有找到关于源码剖析的一些资料，只能先找找头文件
mpi.h搜索找到了在ompi/include/mpi.h.in中的一个文件，查找一下最简单的函数 MPI_Comm_size 和 MPI_Init 函数，
作为总头文件，所有函数声明似乎都在这里了，那么函数定义怎么找呢....如果有vs那样的F12就爽歪歪了。
面对的应用场景： 我要找OpenMpi文件目录下含有字符串 "MPI_Init"的文本文件，有什么工具吗？
幸好，windows已经提供了这样一个cmd命令findstr————参考 http://www.netingcn.com/window-findstr-command.html
搜出来了，发现有一大堆文件满足上述条件啊，突然间看到C文件里面有很多熟悉的函数名
那么应该是一个函数对应一个文件了，去找找看！
有comm_rank.c 还有 init.c：
进入init.c发现了入门第一个函数: MPI_Init(int *argc, char ***argv)
由于是第一个接触到的函数，所以要认真仔细地一行一行去学习：
#include "ompi_config.h"
#include <stdlib.h>
#include "opal/util/show_help.h"
#include "ompi/mpi/c/bindings.h"
#include "ompi/communicator/communicator.h"
#include "ompi/errhandler/errhandler.h"
#include "ompi/constants.h"
#if OMPI_BUILD_MPI_PROFILING
#if OPAL_HAVE_WEAK_SYMBOLS
#pragma weak MPI_Init = PMPI_Init
#endif
#define MPI_Init PMPI_Init
#endif
static const char FUNC_NAME[] = "MPI_Init";
int MPI_Init(int *argc, char ***argv)
{
int err;
int provided;
char *env;
int required = MPI_THREAD_SINGLE;
/* check for environment overrides for required thread level. If
there is, check to see that it is a valid/supported thread level.
If not, default to MPI_THREAD_MULTIPLE. */
if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL"))) {
required = atoi(env);
if (required < MPI_THREAD_SINGLE || required > MPI_THREAD_MULTIPLE) {
required = MPI_THREAD_MULTIPLE;
}
}
/* Call the back-end initialization function (we need to put as
little in this function as possible so that if it's profiled, we
don't lose anything) */
if (NULL != argc && NULL != argv) {
err = ompi_mpi_init(*argc, *argv, required, &provided);
} else {
err = ompi_mpi_init(0, NULL, required, &provided);
}
/* Since we don't have a communicator to invoke an errorhandler on
here, don't use the fancy-schmancy ERRHANDLER macros; they're
really designed for real communicator objects. Just use the
back-end function directly. */
if (MPI_SUCCESS != err) {
return ompi_errhandler_invoke(NULL, NULL,
OMPI_ERRHANDLER_TYPE_COMM,
err <
0 ? ompi_errcode_get_mpi_code(err) :
err, FUNC_NAME);
}
OPAL_CR_INIT_LIBRARY();
return MPI_SUCCESS;
}
首先看到的代码块是:
int required = MPI_THREAD_SINGLE;
/* check for environment overrides for required thread level. If
there is, check to see that it is a valid/supported thread level.
If not, default to MPI_THREAD_MULTIPLE. */
if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL"))) {
required = atoi(env);
if (required < MPI_THREAD_SINGLE || required > MPI_THREAD_MULTIPLE) {
required = MPI_THREAD_MULTIPLE;
}
}
这明显和多线程设置相关，
查阅资料，参考http://web.mit.edu/course/13/13.715/build/mpich2-1.0.6p1/www/www3/MPI_Init_thread.html 或者 某软 的MPI资料，
发现这个required的变量还是比较好懂的。就是表示进程中只有主线程。由于本人现在接触的都是单线程，所以可以这里可以跳过。
接下来的这个代码块是:
/* Call the back-end initialization function (we need to put as
little in this function as possible so that if it's profiled, we
don't lose anything) */
if (NULL != argc && NULL != argv) {
err = ompi_mpi_init(*argc, *argv, required, &provided);
} else {
err = ompi_mpi_init(0, NULL, required, &provided);
}
就是调用back-end初始化函数 ompi_mpi_init ,这个函数在 ompi_mpi_init.c定义了，但是很不幸，这个函数有600多行。
因此，本人决定，先暂时跳过这个函数。这个函数内部进行了许多重要的初始化操作，可能需要后续一点点刨出来。
/* Since we don't have a communicator to invoke an errorhandler on
here, don't use the fancy-schmancy ERRHANDLER macros; they're
really designed for real communicator objects. Just use the
back-end function directly. */
if (MPI_SUCCESS != err) {
return ompi_errhandler_invoke(NULL, NULL,
OMPI_ERRHANDLER_TYPE_COMM,
err <
0 ? ompi_errcode_get_mpi_code(err) :
err, FUNC_NAME);
}
如果初始化函数返回的不是 MPI_SUCCESS， 就返回错误码，那这个函数在哪里呢？
在errhandler.h可以找到函数声明，返回和参数中一致的errcode————找了很久，最后用微软的黑科技findstr /S命令，终于找了对应的文件:
openmpi-3.0.1\ompi\errhandler\errhandler_invoke.c
我想了想，决定还是把这个函数的代码贴出来，虽然有60多行，无非也就是switch的几个case分支而已:
int ompi_errhandler_invoke(ompi_errhandler_t *errhandler, void *mpi_object,
int object_type, int err_code, const char *message)
{
MPI_Fint fortran_handle, fortran_err_code = OMPI_INT_2_FINT(err_code);
ompi_communicator_t *comm;
ompi_win_t *win;
ompi_file_t *file;
/* If we got no errorhandler, then just invoke errors_abort */
if (NULL == errhandler) {
ompi_mpi_errors_are_fatal_comm_handler(NULL, NULL, message); //-------------注意到我们传入了NULL，所以Init失败后进入了这里
return err_code;
}
/* Figure out what kind of errhandler it is, figure out if it's
fortran or C, and then invoke it */
switch (object_type) {
case OMPI_ERRHANDLER_TYPE_COMM: // ompi/errhandler/errhandler.h:
comm = (ompi_communicator_t *) mpi_object; // Enum used to describe what kind MPI object an error handler is used for
switch (errhandler->eh_lang) {
case OMPI_ERRHANDLER_LANG_C: // C语言
errhandler->eh_comm_fn(&comm, &err_code, message, NULL);
break;
case OMPI_ERRHANDLER_LANG_CXX:
errhandler->eh_cxx_dispatch_fn(&comm, &err_code, message,
(ompi_errhandler_generic_handler_fn_t *)errhandler->eh_comm_fn);
break;
case OMPI_ERRHANDLER_LANG_FORTRAN:
fortran_handle = OMPI_INT_2_FINT(comm->c_f_to_c_index);
errhandler->eh_fort_fn(&fortran_handle, &fortran_err_code);
err_code = OMPI_FINT_2_INT(fortran_err_code);
break;
}
break;
case OMPI_ERRHANDLER_TYPE_WIN:
win = (ompi_win_t *) mpi_object;
switch (errhandler->eh_lang) {
case OMPI_ERRHANDLER_LANG_C:
errhandler->eh_win_fn(&win, &err_code, message, NULL);
break;
case OMPI_ERRHANDLER_LANG_CXX:
errhandler->eh_cxx_dispatch_fn(&win, &err_code, message,
(ompi_errhandler_generic_handler_fn_t *)errhandler->eh_win_fn);
break;
case OMPI_ERRHANDLER_LANG_FORTRAN:
fortran_handle = OMPI_INT_2_FINT(win->w_f_to_c_index);
errhandler->eh_fort_fn(&fortran_handle, &fortran_err_code);
err_code = OMPI_FINT_2_INT(fortran_err_code);
break;
}
break;
case OMPI_ERRHANDLER_TYPE_FILE:
file = (ompi_file_t *) mpi_object;
switch (errhandler->eh_lang) {
case OMPI_ERRHANDLER_LANG_C:
errhandler->eh_file_fn(&file, &err_code, message, NULL);
break;
case OMPI_ERRHANDLER_LANG_CXX:
errhandler->eh_cxx_dispatch_fn(&file, &err_code, message,
(ompi_errhandler_generic_handler_fn_t *)errhandler->eh_file_fn);
break;
case OMPI_ERRHANDLER_LANG_FORTRAN:
fortran_handle = OMPI_INT_2_FINT(file->f_f_to_c_index);
errhandler->eh_fort_fn(&fortran_handle, &fortran_err_code);
err_code = OMPI_FINT_2_INT(fortran_err_code);
break;
}
break;
}
/* All done */
return err_code;
}
可以看到，其实这里60多行的代码，就只是掉进了一个函数: ompi_errhandler_t 类的 eh_comm_fn 函数:
struct ompi_errhandler_t {
opal_object_t super;
char eh_name[MPI_MAX_OBJECT_NAME];
/* Type of MPI object that this handler is for */
ompi_errhandler_type_t eh_mpi_object_type;
/* What language was the error handler created in */
ompi_errhandler_lang_t eh_lang;
/* Function pointers. Note that we *have* to have all 4 types
(vs., for example, a union) because the predefined errhandlers
can be invoked on any MPI object type, so we need callbacks for
all of three. */
MPI_Comm_errhandler_function *eh_comm_fn;
ompi_file_errhandler_fn *eh_file_fn;
MPI_Win_errhandler_function *eh_win_fn;
ompi_errhandler_fortran_handler_fn_t *eh_fort_fn;
/* Have separate callback for C++ errhandlers. This pointer is
initialized to NULL and will be set explicitly by the C++
bindings for Create_errhandler. This function is invoked
when eh_lang==OMPI_ERRHANDLER_LANG_CXX so that the user's
callback function can be invoked with the right language
semantics. */
ompi_errhandler_cxx_dispatch_fn_t *eh_cxx_dispatch_fn;
/* index in Fortran <-> C translation array */
int eh_f_to_c_index;
};
而这个 ompi_errhandler_t 对象的创建接口，来自于：
OMPI_DECLSPEC ompi_errhandler_t *ompi_errhandler_create(ompi_errhandler_type_t object_type,
ompi_errhandler_generic_handler_fn_t *func, //这就是 ompi_errhandler_t 结构体的 eh_comm_fn 函数
ompi_errhandler_lang_t language);
因为在 MPI_Init 函数中调用时传入的是NULL(MPI的这些东西初始化失败，自然也不能传入一个ompi_errhandler_t结构了)
这篇文章就到此了，留下了3个分支没有继续跳进去深入了解:
1. ompi_mpi_init因为代码快过长，放在这里不合适
2. init失败，按照正常流程就会调用的: ompi_mpi_errors_are_fatal_comm_handler函数
3. ompi_errhandler_t 这个结构体包含的信息，这跟异常处理有关，对我们弄清楚以后并行计算实际运行可能发生的错误会有帮助
