Erlang open_port极度影响性能的因素

Erlang的port相当于系统的IO，打开了Erlang世界通往外界的通道，可以很方便的执行外部程序。 但是open_port的性能对整个系统来讲非常的重要，我就带领大家看看open_port影响性能的因素。

首先**open_port的文档：

{spawn, Command}

Starts an external program. Command is the name of the external program which will be run. Command runs outside the Erlang work space unless an Erlang driver with the name Command is found. If found, that driver will be started. A driver runs in the Erlang workspace, which means that it is linked with the Erlang runtime **.

When starting external programs on Solaris, the ** call vfork is used in preference to fork for performance reasons, although it has a history of being less robust. If there are problems with using vfork, setting the environment variable ERL_NO_VFORK to any value will cause fork to be used instead.

For external programs, the PATH is searched (or an equivalent method is used to find programs, depending on operating **). This is done by invoking the shell och certain platforms. The first space separated token of the command will be considered as the name of the executable (or driver). This (among other things) makes this option unsuitable for running programs having spaces in file or directory names. Use {spawn_executable, Command} instead if spaces in executable file names is desired.

open_port一个外部程序的时候流程大概是这样的：beam.smp先vfork, 子进程调用child_setup程序，做进一步的清理**作。 清理完成后才真正exec我们的外部程序。

再来**open_port实现的代码：

// sys.c:L1352
02
static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts)
03
{
04
...
05
#if !DISABLE_VFORK
06
int no_vfork;
07
size_t no_vfork_sz = sizeof(no_vfork);
08

09
no_vfork = (erts_sys_getenv("ERL_NO_VFORK",
10
(char *) &no_vfork,
11
&no_vfork_sz) >= 0);
12
#endif
13
...
14
else { /* Use vfork() */
15
char **cs_argv= erts_alloc(ERTS_ALC_T_TMP,(CS_ARGV_NO_OF_ARGS + 1)*
16
sizeof(char *));
17
char fd_close_range; /* 44 bytes are enough to */
18
char dup2_op; /* hold any "%d:%d" string */
19
/* on a 64-bit machine. */
20

21
/* Setup argv[] for the child setup program (implemented in
22
erl_child_setup.c) */
23
i = 0;
24
if (opts->use_stdio) {
25
if (opts->read_write & DO_READ){
26
/* stdout for process */
27
sprintf(&dup2_op, "%d:%d", ifd, 1);
28
if(opts->redir_stderr)
29
/* stderr for process */
30
sprintf(&dup2_op, "%d:%d", ifd, 2);
31
}
32
if (opts->read_write & DO_WRITE)
33
/* stdin for process */
34
sprintf(&dup2_op, "%d:%d", ofd, 0);
35
} else { /* ** will fail if ofd == 4 (unlikely..) */
36
if (opts->read_write & DO_READ)
37
sprintf(&dup2_op, "%d:%d", ifd, 4);
38
if (opts->read_write & DO_WRITE)
39
sprintf(&dup2_op, "%d:%d", ofd, 3);
40
}
41
for (; i < CS_ARGV_NO_OF_DUP2_OPS; i++)
42
strcpy(&dup2_op, "-");
43
sprintf(fd_close_range, "%d:%d", opts->use_stdio ? 3 : 5, max_files-1);
44

45
cs_argv = child_setup_prog;
46
cs_argv = opts->wd ? opts->wd : ".";
47
cs_argv = erts_sched_bind_atvfork_child(unbind);
48
cs_argv = fd_close_range;
49
for (i = 0; i < CS_ARGV_NO_OF_DUP2_OPS; i++)
50
cs_argv = &dup2_op;
51
if (opts->spawn_type == ERTS_SPAWN_EXECUTABLE) {
52
int num = 0;
53
int j = 0;
54
if (opts->argv != NULL) {
55
for(; opts->argv != NULL; ++num)
56
;
57
}
58
cs_argv = erts_realloc(ERTS_ALC_T_TMP,cs_argv, (CS_ARGV_NO_OF_ARGS + 1 + num + 1) * sizeof(char *));
59
cs_argv = "-";
60
cs_argv = cmd_line;
61
if (opts->argv != NULL) {
62
for (;opts->argv != NULL; ++j) {
63
if (opts->argv == erts_default_arg0) {
64
cs_argv = cmd_line;
65
} else {
66
cs_argv = opts->argv;
67
}
68
}
69
}
70
cs_argv = NULL;
71
} else {
72
cs_argv = cmd_line; /* Command */
73
cs_argv = NULL;
74
}
75
DEBUGF(("Using vfork\n"));
76
pid = vfork();
77

78
if (pid == 0) {
79
/* The child! */
80

81
/* Observe!
82
* OTP-4389: The child setup program (implemented in
83
* erl_child_setup.c) will perform the necessary setup of the
84
* child before it execs to the user program. This because
85
* vfork() only allow an *immediate* execve() or _exit() in the
86
* child.
87
*/
88
execve(child_setup_prog, cs_argv, new_environ);
89
_exit(1);
90
}
91
erts_free(ERTS_ALC_T_TMP,cs_argv);
92
...
93
}
在支持vfork的系统下，比如说linux，除非禁止，默认会采用vfork来执行child_setup来调用外部程序。
**vfork的文档：

vfork() differs from fork() in that the parent is suspended until the child makes a call to execve(2) or _exit(2). The child shares all memory
with its parent, including the stack, until execve() is issued by the child. The child must not return from the current function or call
exit(), but may call _exit().

vfork的时候beam.smp整个进程会被阻塞，所以这里是个很重要的性能影响点。

我们再**erl_child_setup.c的代码：
// erl_child_setup.c:111
02
// 1. 取消绑定
03
if (strcmp("false", argv) != 0)
04
if (erts_unbind_from_cpu_str(argv) != 0)
05
return 1;
06
// 2. 复制句柄
07
for (i = 0; i < CS_ARGV_NO_OF_DUP2_OPS; i++) {
08
if (argv == '-'
09
&& argv == '\0')
10
break;
11
if (sscanf(argv, "%d:%d", &from, &to) != 2)
12
return 1;
13
if (dup2(from, to) < 0)
14
return 1;
15
}
16
// 3. 关闭句柄
17
if (sscanf(argv, "%d:%d", &from, &to) != 2)
18
return 1;
19
for (i = from; i <= to; i++)
20
(void) close(i);
21

22
// 4. 调用外部程序
23
if (erts_spawn_executable) {
24
if (argv == NULL) {
25
execl(argv,argv,
26
(char *) NULL);
27
} else {
28
execv(argv,&(argv));
29
}
30
} else {
31
execl("/bin/sh", "sh", "-c", argv, (char *) NULL);
32
}
33
...
这是一个非常流程多的过程，而且1，2，3这三个步骤都非常的耗时。 特别是3对于一个繁忙的IO服务器来讲，会打开大量的句柄，可能都有几十万，关闭这么多的句柄会是个灾难。

我们来演习下这个流程和具体的性能数字：
首先我们设计个open_port的场景，服务器打开768个socke句柄，再运行cat外部程序。
$ cat demo.erl
2
-module(demo).
3
-compile(export_all).
4

5
start()->
6
_ = ,
7
Port = open_port({spawn, "/bin/cat"}, ),
8
port_close(Port),
9
ok.
我们再准备个stap脚本，用来分析这些行为和性能数字：

$ cat demo.stp
02
global t0, t1, t2
03

04
probe process("beam.smp").function("spawn_start") {
05
printf("spawn %\s\n", user_string($name))
06
t0 = gettimeofday_us()
07
}
08

09
probe process("beam.smp").statement("*@sys.c:1607") {
10
t1 = gettimeofday_ns()
11
}
12

13
probe process("beam.smp").statement("*@sys.c:1627") {
14
printf("vfork take %d ns\n", gettimeofday_ns() - t1);
15
}
16

17
probe process("child_setup").function("main") {
18
t2 = gettimeofday_us()
19
}
20

21
probe process("child_setup").statement("*@erl_child_setup.c:111") {
22
t3 = gettimeofday_us()
23
printf("spawn take %d us, child_setup take %d us\n", t3 - t0, t3 - t2)
24
}
25

26
probe syscall.execve {
27
printf("%s, arg %s\n", name, argstr)
28
}
29

30
probe syscall.fork {
31
printf("%s, arg %s\n", name, argstr)
32
}
33

34
probe begin {
35
println(")");
我们在一个终端下运行stap脚本观察行为：
$ erlc demo.erl
02
$ PATH=otp/bin/x86_64-unknown-linux-gnu/:$PATH sudo stap demo.stp
03
)
04
fork, arg
05
execve, arg otp/bin/erl
06
fork, arg
07
fork, arg
08
fork, arg
09
execve, arg /bin/sed "s/.*\\///"
10
execve, arg /home/chuba/otp/bin/x86_64-unknown-linux-gnu/erlexec
11
execve, arg /home/chuba/otp/bin/x86_64-unknown-linux-gnu/beam.smp "--" "-root" "/home/chuba/otp" "-progname" "erl" "--" "-home" "/home/chuba" "--"
12
clone, arg .
13
..
14
clone, arg CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID
15
spawn inet_gethost 4
16
fork, arg
17
execve, arg /home/chuba/otp/bin/x86_64-unknown-linux-gnu/child_setup "FFFF" "." "exec inet_gethost 4 " "3:327679" "8:1" "9:0" "-"
18
vfork take 8487 ns
19
spawn take 173707 us, child_setup take 94535 us
20
execve, arg /bin/sh "-c" "exec inet_gethost 4 "
21
execve, arg /home/chuba/otp/bin/x86_64-unknown-linux-gnu/inet_gethost "4"
22
fork, arg
23
clone, arg CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID
24
clone, arg CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID
25
spawn /bin/cat
26
fork, arg
27
execve, arg /home/chuba/otp/bin/x86_64-unknown-linux-gnu/child_setup "FFFF" "." "exec /bin/cat" "3:327679" "2312:1" "2313:0" "-"
28
vfork take 5298 ns
29
spawn take 180974 us, child_setup take 101646 us
30
execve, arg /bin/sh "-c" "exec /bin/cat"
31
execve, arg /bin/cat
32
spawn /bin/cat
33
fork, arg
34
execve, arg /home/chuba/otp/bin/x86_64-unknown-linux-gnu/child_setup "FFFF" "." "exec /bin/cat" "3:327679" "3080:1" "3081:0" "-"
35
vfork take 8929 ns
36
spawn take 169569 us, child_setup take 90163 us
37
execve, arg /bin/sh "-c" "exec /bin/cat"
38
execve, arg /bin/cat
39
...
在另外一个终端下运行我们的**案例：
$ otp/bin/erl
2
Erlang R14B04 (erts-5.8.5) 1
3

4
Eshell V5.8.5 (abort with ^G)
5
1> demo:start().
6
ok
7
2> demo:start().
8
ok
9
3>
我们可以看到二次执行的开销差不多：
vfork take 8929 ns
spawn take 169569 us, child_setup take 90163 us

从实验得来的数字来看：
vfork需要阻塞beam.smp 8个us时间，而整个spawn下来要169ms, 其中 child_setup关闭句柄等等花了90ms， 数字无情的告诉我们这些性能杀手不容忽视。

解决方案：
1. 改用fork避免阻塞beam.smp, erl -env ERL_NO_VFORK 1
2. 减少文件句柄,如果确实需要大量的open_port让另外一个专注的节点来做。

祝玩得开心!