partIII
实现 分发MapReduce 任务，实现 schedule() 方法在 mapreduce/schedule.go。
schedule()函数的职责是把任务分给可用的worker.
schedule()通过信道registerChan参数读取 注册的worker.
worker与schedule()之间的顺序是没有确定的。
schedule()告诉worker 执行远程函数调用Worker.DoTask.
远程调用的参数 DoTaskArgs定义在 mapreduce/common_rpc.go
File 参数只有 mapTask 才用到.
rpc 使用call() function 定义在 mapreduce/common_rpc.go
测试命令
go test -run TestParallel
会执行两边，TestParallelBasic and TestParallelCheck;
master@master:~/study/6.824/src/mapreduce$ go test -run TestParallel
2019/03/22 14:34:40 rpc.Register: method "CleanupFiles" has 1 input parameters; needs exactly three
2019/03/22 14:34:40 rpc.Register: method "Lock" has 1 input parameters; needs exactly three
2019/03/22 14:34:40 rpc.Register: method "Unlock" has 1 input parameters; needs exactly three
2019/03/22 14:34:40 rpc.Register: method "Wait" has 1 input parameters; needs exactly three
2019/03/22 14:34:40 rpc.Register: method "Lock" has 1 input parameters; needs exactly three
2019/03/22 14:34:40 rpc.Register: method "Unlock" has 1 input parameters; needs exactly three
/var/tmp/824-1000/mr20412-master: Starting Map/Reduce task test
2019/03/22 14:34:40 rpc.Register: method "Lock" has 1 input parameters; needs exactly three
2019/03/22 14:34:40 rpc.Register: method "Unlock" has 1 input parameters; needs exactly three
Schedule: 20 mapPhase tasks (10 I/Os)
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #0 on file 824-mrinput-0.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker1: given mapPhase task #1 on file 824-mrinput-1.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker1: mapPhase task #1 done
/var/tmp/824-1000/mr20412-worker0: mapPhase task #0 done
/var/tmp/824-1000/mr20412-worker1: given mapPhase task #2 on file 824-mrinput-2.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #3 on file 824-mrinput-3.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #3 done
/var/tmp/824-1000/mr20412-worker1: mapPhase task #2 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #4 on file 824-mrinput-4.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker1: given mapPhase task #5 on file 824-mrinput-5.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #4 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #6 on file 824-mrinput-6.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker1: mapPhase task #5 done
/var/tmp/824-1000/mr20412-worker1: given mapPhase task #7 on file 824-mrinput-7.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #6 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #8 on file 824-mrinput-8.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker1: mapPhase task #7 done
/var/tmp/824-1000/mr20412-worker1: given mapPhase task #9 on file 824-mrinput-9.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #8 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #10 on file 824-mrinput-10.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker1: mapPhase task #9 done
/var/tmp/824-1000/mr20412-worker1: given mapPhase task #11 on file 824-mrinput-11.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #10 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #12 on file 824-mrinput-12.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker1: mapPhase task #11 done
/var/tmp/824-1000/mr20412-worker1: given mapPhase task #13 on file 824-mrinput-13.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #12 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #14 on file 824-mrinput-14.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker1: mapPhase task #13 done
/var/tmp/824-1000/mr20412-worker1: given mapPhase task #15 on file 824-mrinput-15.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #14 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #16 on file 824-mrinput-16.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker1: mapPhase task #15 done
/var/tmp/824-1000/mr20412-worker1: given mapPhase task #17 on file 824-mrinput-17.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #16 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #18 on file 824-mrinput-18.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker1: mapPhase task #17 done
/var/tmp/824-1000/mr20412-worker1: given mapPhase task #19 on file 824-mrinput-19.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #18 done
/var/tmp/824-1000/mr20412-worker1: mapPhase task #19 done
Schedule: mapPhase done
Schedule: 10 reducePhase tasks (20 I/Os)
/var/tmp/824-1000/mr20412-worker1: given reducePhase task #1 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker0: given reducePhase task #0 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker1: reducePhase task #1 done
/var/tmp/824-1000/mr20412-worker1: given reducePhase task #2 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker0: reducePhase task #0 done
/var/tmp/824-1000/mr20412-worker0: given reducePhase task #3 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker1: reducePhase task #2 done
/var/tmp/824-1000/mr20412-worker1: given reducePhase task #4 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker0: reducePhase task #3 done
/var/tmp/824-1000/mr20412-worker0: given reducePhase task #5 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker1: reducePhase task #4 done
/var/tmp/824-1000/mr20412-worker1: given reducePhase task #6 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker0: reducePhase task #5 done
/var/tmp/824-1000/mr20412-worker0: given reducePhase task #7 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker1: reducePhase task #6 done
/var/tmp/824-1000/mr20412-worker1: given reducePhase task #8 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker0: reducePhase task #7 done
/var/tmp/824-1000/mr20412-worker0: given reducePhase task #9 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker1: reducePhase task #8 done
/var/tmp/824-1000/mr20412-worker0: reducePhase task #9 done
Schedule: reducePhase done
Merge: read mrtmp.test-res-0
Merge: read mrtmp.test-res-1
Merge: read mrtmp.test-res-2
Merge: read mrtmp.test-res-3
Merge: read mrtmp.test-res-4
Merge: read mrtmp.test-res-5
Merge: read mrtmp.test-res-6
Merge: read mrtmp.test-res-7
Merge: read mrtmp.test-res-8
Merge: read mrtmp.test-res-9
/var/tmp/824-1000/mr20412-master: Map/Reduce task completed
2019/03/22 14:34:41 rpc.Register: method "CleanupFiles" has 1 input parameters; needs exactly three
2019/03/22 14:34:41 rpc.Register: method "Lock" has 1 input parameters; needs exactly three
2019/03/22 14:34:41 rpc.Register: method "Unlock" has 1 input parameters; needs exactly three
2019/03/22 14:34:41 rpc.Register: method "Wait" has 1 input parameters; needs exactly three
2019/03/22 14:34:41 rpc.Register: method "Lock" has 1 input parameters; needs exactly three
2019/03/22 14:34:41 rpc.Register: method "Unlock" has 1 input parameters; needs exactly three
/var/tmp/824-1000/mr20412-master: Starting Map/Reduce task test
Schedule: 20 mapPhase tasks (10 I/Os)
2019/03/22 14:34:41 rpc.Register: method "Lock" has 1 input parameters; needs exactly three
2019/03/22 14:34:41 rpc.Register: method "Unlock" has 1 input parameters; needs exactly three
/var/tmp/824-1000/mr20412-worker1: given mapPhase task #0 on file 824-mrinput-0.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #1 on file 824-mrinput-1.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #1 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #2 on file 824-mrinput-2.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #2 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #3 on file 824-mrinput-3.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #3 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #4 on file 824-mrinput-4.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #4 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #5 on file 824-mrinput-5.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #5 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #6 on file 824-mrinput-6.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #6 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #7 on file 824-mrinput-7.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #7 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #8 on file 824-mrinput-8.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #8 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #9 on file 824-mrinput-9.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #9 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #10 on file 824-mrinput-10.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #10 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #11 on file 824-mrinput-11.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #11 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #12 on file 824-mrinput-12.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #12 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #13 on file 824-mrinput-13.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #13 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #14 on file 824-mrinput-14.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #14 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #15 on file 824-mrinput-15.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #15 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #16 on file 824-mrinput-16.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #16 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #17 on file 824-mrinput-17.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #17 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #18 on file 824-mrinput-18.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #18 done
/var/tmp/824-1000/mr20412-worker0: given mapPhase task #19 on file 824-mrinput-19.txt (nios: 10)
/var/tmp/824-1000/mr20412-worker0: mapPhase task #19 done
/var/tmp/824-1000/mr20412-worker1: mapPhase task #0 done
Schedule: mapPhase done
Schedule: 10 reducePhase tasks (20 I/Os)
/var/tmp/824-1000/mr20412-worker1: given reducePhase task #1 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker0: given reducePhase task #0 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker0: reducePhase task #0 done
/var/tmp/824-1000/mr20412-worker1: reducePhase task #1 done
/var/tmp/824-1000/mr20412-worker0: given reducePhase task #2 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker1: given reducePhase task #3 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker0: reducePhase task #2 done
/var/tmp/824-1000/mr20412-worker0: given reducePhase task #4 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker1: reducePhase task #3 done
/var/tmp/824-1000/mr20412-worker1: given reducePhase task #5 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker0: reducePhase task #4 done
/var/tmp/824-1000/mr20412-worker0: given reducePhase task #6 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker1: reducePhase task #5 done
/var/tmp/824-1000/mr20412-worker1: given reducePhase task #7 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker0: reducePhase task #6 done
/var/tmp/824-1000/mr20412-worker0: given reducePhase task #8 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker1: reducePhase task #7 done
/var/tmp/824-1000/mr20412-worker1: given reducePhase task #9 on file (nios: 20)
/var/tmp/824-1000/mr20412-worker0: reducePhase task #8 done
/var/tmp/824-1000/mr20412-worker1: reducePhase task #9 done
Schedule: reducePhase done
Merge: read mrtmp.test-res-0
Merge: read mrtmp.test-res-1
Merge: read mrtmp.test-res-2
Merge: read mrtmp.test-res-3
Merge: read mrtmp.test-res-4
Merge: read mrtmp.test-res-5
Merge: read mrtmp.test-res-6
Merge: read mrtmp.test-res-7
Merge: read mrtmp.test-res-8
Merge: read mrtmp.test-res-9
/var/tmp/824-1000/mr20412-master: Map/Reduce task completed
PASS
ok _/home/master/study/6.824/src/mapreduce 2.053s
schedule.go
package mapreduce import ( "fmt" "sync" "strings" "time" ) // // schedule() starts and waits for all tasks in the given phase (mapPhase // or reducePhase). the mapFiles argument holds the names of the files that // are the inputs to the map phase, one per map task. nReduce is the // number of reduce tasks. the registerChan argument yields a stream // of registered workers; each item is the worker's RPC address, // suitable for passing to call(). registerChan will yield all // existing registered workers (if any) and new ones as they register. // func schedule(jobName string, mapFiles []string, nReduce int, phase jobPhase, registerChan chan string) { var ntasks int var n_other int // number of inputs (for reduce) or outputs (for map) switch phase { case mapPhase: ntasks = len(mapFiles) n_other = nReduce case reducePhase: ntasks = nReduce n_other = len(mapFiles) } fmt.Printf("Schedule: %v %v tasks (%d I/Os)\n", ntasks, phase, n_other) // All ntasks tasks have to be scheduled on workers. Once all tasks // have completed successfully, schedule() should return. // // Your code here (Part III, Part IV). // usableWorkers := make(chan string) waitGroup := new(sync.WaitGroup) i := 0 for { var work string; select { case work =<- usableWorkers: //fmt.Printf("usable worker: %v \n", work) case work =<- registerChan: //fmt.Printf("usable worker: %v \n", work) //default: //fmt.Printf("not usable worker wait for next loop ...\n") } if strings.TrimSpace(work) != "" { waitGroup.Add(1) go func(index int) { rqArs := new(DoTaskArgs) rqArs.JobName=jobName if phase == mapPhase { rqArs.File = mapFiles[index] // only for map, the input file } rqArs.Phase=phase // are we in mapPhase or reducePhase? rqArs.TaskNumber=index // this task's index in the current phase // NumOtherPhase is the total number of tasks in other phase; mappers // need this to compute the number of output bins, and reducers needs // this to know how many input files to collect. rqArs.NumOtherPhase=n_other defer func(){ waitGroup.Done() usableWorkers <- work }() call(work, "Worker.DoTask", rqArs, nil) }(i) i++ } else { time.After(time.Second) } if i == ntasks { break } } waitGroup.Wait() fmt.Printf("Schedule: %v done\n", phase) }
