# Superfans GPU controller # # author: Domen Tabernik # 2019 import time, superfans, subprocess, signal, sys, json class GracefulKiller: kill_now = False def __init__(self): signal.signal(signal.SIGINT, self.exit_gracefully) signal.signal(signal.SIGTERM, self.exit_gracefully) def exit_gracefully(self,signum, frame): self.kill_now = True def enable_persistance_nvidia(): cmd = 'nvidia-smi -pm 1' s = subprocess.check_output(cmd + " 2>&1", shell=True) def retrieve_nvidia_gpu_temperature(): cmd = 'nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader' s = subprocess.check_output(cmd + " 2>&1", shell=True) if len(s) <= 0: return False s = s.decode('ascii') out = [int(x.strip()) for x in s.split("\n") if len(x.strip()) > 0] if out: return out else: return False def superfans_gpu_controller(fan_settings, FAN_INCREASED_MIN_TIME=120, sleep_sec=2, gpu_moving_avg_num=4, fan_target_eps=2.0): """ Controller function that monitors GPU temperature in constant loop and adjusts FAN speeds based on provided `fan_settings`. After the loop the default preset is restored. :param fan_settings: dictionary that maps the temperature in deg C to % of fan speed :param FAN_INCREASED_MIN_TIME: minimal time before a fan speed is again reduced (based on previous change) default=120 :param sleep_sec: loop sleep time (default=2 sec) :param gpu_moving_avg_num: moving average for GPU i.e. the number of last measurements that are averaged (default=5) :param fan_target_eps: tolerance of fan target w.r.t. the the actual value in deg C (default=1.0) :return: """ superfan_config = dict(hostname= 'localhost') # convert fan_settings keys from string to int fan_settings = { int(k): fan_settings[k] for k in sorted(fan_settings.keys()) } # save default present before changing anything default_preset = superfans.get_preset(superfan_config) print('Started fan control using GPU temperature.') print('Using settings:') for k in sorted(fan_settings.keys()): print('\t%d C = %d ' % (k, fan_settings[k]) + "%") print('\n') #open file for cpu temperature cpu_t_f = open("/sys/class/thermal/thermal_zone0/temp", "r") # put GPUs into persistance mode so that nvidia-smi will retern immediately enable_persistance_nvidia() #try: for gege in [0]: FAN_MEMBERS = superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_CPU1] # GPU moving average previous_target_fan = 0 previous_update_time = None prev_GPU_temp = [] # ensure correct ending when SIGINT and SIGTERM are received k = GracefulKiller() while not k.kill_now: # get GPU temperature GPU_temp = retrieve_nvidia_gpu_temperature() prev_GPU_temp.append(GPU_temp) # continue until we have enough sampels for moving average if len(prev_GPU_temp) < gpu_moving_avg_num: continue # retain last 5 mesurements prev_GPU_temp = prev_GPU_temp[-gpu_moving_avg_num:] mean_GPU_temp = prev_GPU_temp[0] for gpu_temp in prev_GPU_temp[1:]: mean_GPU_temp = [x+y for x,y in zip(gpu_temp, mean_GPU_temp)] mean_GPU_temp = [x/len(prev_GPU_temp) for x in mean_GPU_temp] max_gpu_temp = max(mean_GPU_temp) #read cpu temperature cpu_t_f.seek(0,0) cpu_temp = int(cpu_t_f.read()) / 1000 #°C max_temp = max(max_gpu_temp, cpu_temp) for key_temp in sorted(fan_settings.keys())[::-1]: if key_temp <= max_temp: target_fan = fan_settings[key_temp] break current_fan_levels = superfans.get_fan(superfan_config, FAN_MEMBERS) current_update_time = time.time() #diff_cpu1_fan = [abs(current_fan_levels[FAN] - target_fan) for FAN in superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_CPU1] if FAN in current_fan_levels and current_fan_levels[FAN] > 0] #print(list(superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_CPU1])) #print(list(current_fan_levels[FAN] for FAN in superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_CPU1])) # TODO: ignore outlier FANs in case they are faulty disbale_update = False if previous_update_time is not None and previous_target_fan is not None: has_enough_time_elapsed = current_update_time - previous_update_time > FAN_INCREASED_MIN_TIME is_level_down_change = target_fan < previous_target_fan disbale_update = True if is_level_down_change and not has_enough_time_elapsed else False if not disbale_update: # Allow for 1% difference in target update_cpu1_fan = abs(previous_target_fan - target_fan) > fan_target_eps if update_cpu1_fan: superfans.set_fan(superfan_config, target_fan, superfans.FAN_ZONE_CPU1) if update_cpu1_fan: print('\tCurrent GPU measurements (in C): %s' % ','.join(map(str,GPU_temp))) print('\tMoving average GPU measurements (in C): %s (max=%d)' % (','.join(map(str,mean_GPU_temp)),max_gpu_temp)) print('\tCPU measurement (in C): %.1f ' % cpu_temp) print('\tTarget FAN speed: %d C => FAN %d %% (difference: CPU1 fan = %.2f)' % (max_temp, target_fan, -1)) #max(diff_cpu1_fan) print('\n\n') previous_target_fan = target_fan previous_update_time = current_update_time time.sleep(sleep_sec) #finally: # revert back to default preset before finishing # superfans.set_preset(superfan_config, default_preset) # print('Reverted back to system default.') def main(): if len(sys.argv) != 2: print('Invalid number of arguments: missing configuration file!!') print('') print(' Usage: %s [PATH_TO_JSON_CONFIG]' % sys.argv[0]) print('') print(' Configuration file in JSON format should include "fan_settings" = {[in deg C]: [% fan], ...} ') print exit(0) with open(sys.argv[1]) as cfg_file: cfg = json.load(cfg_file) superfans_gpu_controller(cfg['fan_settings']) if __name__ == "__main__": main()