问题由来
前些天对系统做了一个优化，将原来从queue 轮询刷出数据后每条消息一个 spawn 进程单独处理，修改为批量刷出。一条一条刷轮询刷 queue 存在问题：刷queue 进程太多时，占用CPU，刷queue 少时容易受网络延时影响。修改后，queue 中数据越多，batch 数越大，提高刷queue速度。
但问题随之而来：erlang节点，进程数蹭蹭往上涨，CPU 占用很低，直到 outofmemory， 节点remsh 无法连接。
问题排查
通过kill -SIGUSR1 PID 创建crashdump 发现，我们使用的一个一致性hash gen_server 里面的message_queue_len 30w+，堵塞。
存在问题代码： https://github.com/chrismoos/hash-ring 使用计算一致性hash 节点的gen_server
猜想原因：
1. hash_ring 效率低？测试单程 30w/s +, 排除
2. 进程优先级不高
erlang 所有进程优先级一样，平等执行，大量进程情况下如果CPU较高，热点进程消息往往得不到处理，恶性顺序，将这些进程优先级process_flag(priority, hight) 这个设置曾经帮忙解决过问题，不过这次无效
3. Port 进程优先级
实际业务由Port 完成，Port 调度也是平等的，但是没有找到设置Port优先级的方式
4. 从问题产生代码修改
与原来相比 就是spawn 进程变得一波一波 速度比较快，产生大量进程。也就是 gen_server:call 数量较多，gen_server 和Port 交互的时候往往receive match 方式，看下代码果然。
问题原因分析：
gen_server:call虽然都是同步调用，但是如果有10w 进程都是调用，那么gen_server 的message_queue 将是10w；
gen_server 内部处理消息：
Port ! Msg,
receive
Match -> dostring
end
这个时候receive， Match 操作非常费时，因为Match 需要因为不断新的gen_server:call 对10w 的消息一遍又一遍的遍历匹配执行，能快才怪呢。
解决方案
参考文章：http://wqtn22.iteye.com/blog/1572258
我们选用第二种方法
当然也想过使用gen_server2 处理消息前将message_queue 倒入state 中自己队列，那样消息reply 会更及时一点（Port 的reply会放到队尾，最后处理），不过消耗还是有一些的，handle_call 期间可以有很大消息进来。
测试 并行 spwan 10w 进程，老版本执行2分钟以上，改造后1s内执行完成。
修改如下：https://github.com/chrismoos/hash-ring/pull/11/files
| |
@@ -97,14 +97,16 @@ set_mode(Ring, Mode) when is_integer(Mode) ->
|
| |
|
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
| |
|
-record(state, {
|
| |
|
port,
|
| |
|
- rings
|
| |
|
+ rings,
|
| |
|
+ queue
|
| |
|
}).
|
| |
|
|
| |
|
init(Drv) ->
|
| |
|
Port = open_port({spawn, Drv}, [binary]),
|
| |
|
{ok, #state{
|
| |
|
port = Port,
|
| |
|
- rings = dict:new() }}.
|
| |
|
+ rings = dict:new(),
|
| |
|
+ queue = queue:new() }}.
|
| |
|
|
| |
|
|
| |
|
|
| |
@@ -160,21 +162,12 @@ handle_call({remove_node, {Ring, Node}}, _From, #state{ port = Port, rings = Rin
|
| |
|
{reply, {error, ring_not_found}, State}
|
| |
|
end;
|
| |
|
|
| |
|
-handle_call({find_node, {Ring, Key}}, _From, #state{ port = Port, rings = Rings } = State) ->
|
| |
|
+handle_call({find_node, {Ring, Key}}, From, #state{ port = Port, rings = Rings, queue = Queue } = State) ->
|
| |
|
case dict:find(Ring, Rings) of
|
| |
|
{ok, Index} ->
|
| |
|
KeySize = size(Key),
|
| |
|
Port ! {self(), {command, <<5:8, Index:32, KeySize:32, Key/binary>>}},
|
| |
|
- receive
|
| |
|
- {Port, {data, <<3:8>>}} ->
|
| |
|
- {reply, {error, invalid_ring}, State};
|
| |
|
- {Port, {data, <<2:8>>}} ->
|
| |
|
- {reply, {error, node_not_found}, State};
|
| |
|
- {Port, {data, <<1:8>>}} ->
|
| |
|
- {reply, {error, unknown_error}, State};
|
| |
|
- {Port, {data, <<Node/binary>>}} ->
|
| |
|
- {reply, {ok, Node}, State}
|
| |
|
- end;
|
| |
|
+ {noreply, State#state{queue = queue:in(From, Queue)}};
|
| |
|
_ ->
|
| |
|
{reply, {error, ring_not_found}, State}
|
| |
|
end;
|
| |
@@ -198,6 +191,21 @@ handle_call(stop, _From, State) ->
|
| |
|
|
| |
|
handle_cast(_, State) ->
|
| |
|
{noreply, State}.
|
| |
|
+handle_info({Port, {data, Data}}, #state{port = Port, queue = Queue} = State) ->
|
| |
|
+ R =
|
| |
|
+ case Data of
|
| |
|
+ <<3:8>> ->
|
| |
|
+ {error, invalid_ring};
|
| |
|
+ <<2:8>> ->
|
| |
|
+ {error, node_not_found};
|
| |
|
+ <<1:8>> ->
|
| |
|
+ {error, unknown_error};
|
| |
|
+ <<Node/binary>> ->
|
| |
|
+ {ok, Node}
|
| |
|
+ end,
|
| |
|
+ {{value, Pid}, QTail} = queue:out(Queue),
|
| |
|
+ safe_reply(Pid, R),
|
| |
|
+ {noreply, State#state{queue = QTail}};
|
| |
|
|
| |
|
handle_info({'EXIT', Port, _Reason} = PortExit, #state{ port = Port } = State ) ->
|
| |
|
{stop, PortExit, State}.
|
| |
@@ -214,6 +222,10 @@ terminate(_, #state{ port = Port }) ->
|
| |
|
code_change(_OldVsn, State, _Extra) ->
|
| |
|
{ok, State}.
|
| |
|
|
| |
|
+safe_reply(undefined, _Value) ->
|
| |
|
+ ok;
|
| |
|
+safe_reply(From, Value) ->
|
| |
|
+ gen_server:reply(From, Value).
|
| |
|
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
| |
|
%% Tests
|
| |
|
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
| |
