监控GPU显卡，空闲自动抢占跑脚本

代码

监控脚本

MonitorGpu.py，该脚本会自动监控GPU内存和功耗情况，在满足条件情况下，可以自动运行脚本TestCuda.py

import os
import sys
import time
import platform

cmd = 'python TestCuda.py'

def gpu_info():
    system = platform.system()
    if system == "Windows":
        gpu_status = os.popen('nvidia-smi | findstr %').read().split('|')
    else:  # if system == "Linux"
        gpu_status = os.popen('nvidia-smi | grep %').read().split('|')
    gpu_memory = int(gpu_status[2].split('/')[0].split('M')[0].strip())
    gpu_power = int(gpu_status[1].split('   ')[-1].split('/')[0].split('W')[0].strip())
    return gpu_power, gpu_memory


def narrow_setup(interval=2):
    gpu_power, gpu_memory = gpu_info()
    i = 0
    while gpu_memory > 1000 or gpu_power > 20:  # set waiting condition
        gpu_power, gpu_memory = gpu_info()
        i = i % 5
        symbol = 'monitoring ' + '>' * i + ' ' * (10 - i - 1) + '|'
        gpu_power_str = 'gpu power%d W |' % gpu_power
        gpu_memory_str = 'gpu memory%d MiB |' % gpu_memory
        sys.stdout.write('\r' + gpu_memory_str + ' ' + gpu_power_str + ' ' + symbol)
        sys.stdout.flush()
        time.sleep(interval)
        i += 1
    print('n' + cmd)
    os.system(cmd)


if __name__ == '__main__':
    narrow_setup()

TestCuda.py

print("You can use cuda now!")

参考链接

https://www.zhihu.com/question/312964005

https://blog.csdn.net/u013241583/article/details/90174967