167 lines
6.4 KiB
Python
167 lines
6.4 KiB
Python
# Superfans GPU controller
|
|
#
|
|
# author: Domen Tabernik
|
|
# 2019
|
|
|
|
import time, superfans, subprocess, signal, sys, json
|
|
|
|
class GracefulKiller:
|
|
kill_now = False
|
|
def __init__(self):
|
|
signal.signal(signal.SIGINT, self.exit_gracefully)
|
|
signal.signal(signal.SIGTERM, self.exit_gracefully)
|
|
|
|
def exit_gracefully(self,signum, frame):
|
|
self.kill_now = True
|
|
|
|
def enable_persistance_nvidia():
|
|
cmd = 'nvidia-smi -pm 1'
|
|
s = subprocess.check_output(cmd + " 2>&1", shell=True)
|
|
|
|
def retrieve_nvidia_gpu_temperature():
|
|
cmd = 'nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader'
|
|
|
|
s = subprocess.check_output(cmd + " 2>&1", shell=True)
|
|
if len(s) <= 0:
|
|
return False
|
|
s = s.decode('ascii')
|
|
|
|
out = [int(x.strip()) for x in s.split("\n") if len(x.strip()) > 0]
|
|
if out:
|
|
return out
|
|
else:
|
|
return False
|
|
|
|
def superfans_gpu_controller(fan_settings, FAN_INCREASED_MIN_TIME=120, sleep_sec=2, gpu_moving_avg_num=4, fan_target_eps=2.0):
|
|
"""
|
|
Controller function that monitors GPU temperature in constant loop and adjusts FAN speeds based on provided `fan_settings`.
|
|
After the loop the default preset is restored.
|
|
|
|
:param fan_settings: dictionary that maps the temperature in deg C to % of fan speed
|
|
:param FAN_INCREASED_MIN_TIME: minimal time before a fan speed is again reduced (based on previous change) default=120
|
|
:param sleep_sec: loop sleep time (default=2 sec)
|
|
:param gpu_moving_avg_num: moving average for GPU i.e. the number of last measurements that are averaged (default=5)
|
|
:param fan_target_eps: tolerance of fan target w.r.t. the the actual value in deg C (default=1.0)
|
|
:return:
|
|
"""
|
|
superfan_config = dict(hostname= 'localhost')
|
|
|
|
# convert fan_settings keys from string to int
|
|
fan_settings = { int(k): fan_settings[k] for k in sorted(fan_settings.keys()) }
|
|
|
|
# save default present before changing anything
|
|
default_preset = superfans.get_preset(superfan_config)
|
|
print('Started fan control using GPU temperature.')
|
|
print('Using settings:')
|
|
for k in sorted(fan_settings.keys()):
|
|
print('\t%d C = %d ' % (k, fan_settings[k]) + "%")
|
|
print('\n')
|
|
|
|
#open file for cpu temperature
|
|
cpu_t_f = open("/sys/class/thermal/thermal_zone0/temp", "r")
|
|
|
|
# put GPUs into persistance mode so that nvidia-smi will retern immediately
|
|
enable_persistance_nvidia()
|
|
|
|
#try:
|
|
for gege in [0]:
|
|
FAN_MEMBERS = superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_CPU1]
|
|
|
|
# GPU moving average
|
|
previous_target_fan = 0
|
|
previous_update_time = None
|
|
|
|
prev_GPU_temp = []
|
|
|
|
# ensure correct ending when SIGINT and SIGTERM are received
|
|
k = GracefulKiller()
|
|
while not k.kill_now:
|
|
|
|
# get GPU temperature
|
|
GPU_temp = retrieve_nvidia_gpu_temperature()
|
|
|
|
prev_GPU_temp.append(GPU_temp)
|
|
|
|
# continue until we have enough sampels for moving average
|
|
if len(prev_GPU_temp) < gpu_moving_avg_num:
|
|
continue
|
|
|
|
# retain last 5 mesurements
|
|
prev_GPU_temp = prev_GPU_temp[-gpu_moving_avg_num:]
|
|
mean_GPU_temp = prev_GPU_temp[0]
|
|
for gpu_temp in prev_GPU_temp[1:]:
|
|
mean_GPU_temp = [x+y for x,y in zip(gpu_temp, mean_GPU_temp)]
|
|
|
|
mean_GPU_temp = [x/len(prev_GPU_temp) for x in mean_GPU_temp]
|
|
|
|
max_gpu_temp = max(mean_GPU_temp)
|
|
|
|
#read cpu temperature
|
|
cpu_t_f.seek(0,0)
|
|
cpu_temp = int(cpu_t_f.read()) / 1000 #°C
|
|
|
|
max_temp = max(max_gpu_temp, cpu_temp)
|
|
|
|
for key_temp in sorted(fan_settings.keys())[::-1]:
|
|
if key_temp <= max_temp:
|
|
target_fan = fan_settings[key_temp]
|
|
break
|
|
|
|
current_fan_levels = superfans.get_fan(superfan_config, FAN_MEMBERS)
|
|
current_update_time = time.time()
|
|
#diff_cpu1_fan = [abs(current_fan_levels[FAN] - target_fan) for FAN in superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_CPU1] if FAN in current_fan_levels and current_fan_levels[FAN] > 0]
|
|
|
|
#print(list(superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_CPU1]))
|
|
|
|
#print(list(current_fan_levels[FAN] for FAN in superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_CPU1]))
|
|
|
|
# TODO: ignore outlier FANs in case they are faulty
|
|
|
|
disbale_update = False
|
|
|
|
if previous_update_time is not None and previous_target_fan is not None:
|
|
has_enough_time_elapsed = current_update_time - previous_update_time > FAN_INCREASED_MIN_TIME
|
|
is_level_down_change = target_fan < previous_target_fan
|
|
disbale_update = True if is_level_down_change and not has_enough_time_elapsed else False
|
|
|
|
if not disbale_update:
|
|
# Allow for 1% difference in target
|
|
update_cpu1_fan = abs(previous_target_fan - target_fan) > fan_target_eps
|
|
|
|
if update_cpu1_fan:
|
|
superfans.set_fan(superfan_config, target_fan, superfans.FAN_ZONE_CPU1)
|
|
|
|
if update_cpu1_fan:
|
|
print('\tCurrent GPU measurements (in C): %s' % ','.join(map(str,GPU_temp)))
|
|
print('\tMoving average GPU measurements (in C): %s (max=%d)' % (','.join(map(str,mean_GPU_temp)),max_gpu_temp))
|
|
print('\tCPU measurement (in C): %.1f ' % cpu_temp)
|
|
print('\tTarget FAN speed: %d C => FAN %d %% (difference: CPU1 fan = %.2f)' % (max_temp, target_fan, -1)) #max(diff_cpu1_fan)
|
|
print('\n\n')
|
|
|
|
previous_target_fan = target_fan
|
|
previous_update_time = current_update_time
|
|
|
|
time.sleep(sleep_sec)
|
|
#finally:
|
|
# revert back to default preset before finishing
|
|
# superfans.set_preset(superfan_config, default_preset)
|
|
# print('Reverted back to system default.')
|
|
|
|
def main():
|
|
if len(sys.argv) != 2:
|
|
print('Invalid number of arguments: missing configuration file!!')
|
|
print('')
|
|
print(' Usage: %s [PATH_TO_JSON_CONFIG]' % sys.argv[0])
|
|
print('')
|
|
print(' Configuration file in JSON format should include "fan_settings" = {[in deg C]: [% fan], ...} ')
|
|
print
|
|
exit(0)
|
|
|
|
with open(sys.argv[1]) as cfg_file:
|
|
cfg = json.load(cfg_file)
|
|
|
|
superfans_gpu_controller(cfg['fan_settings'])
|
|
|
|
if __name__ == "__main__":
|
|
main()
|