Initial commit (working version).
This commit is contained in:
Domen Tabernik
commit
1f8148346f
|
|
@ -0,0 +1,21 @@
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2018 Chris Putnam
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
# NVIDIA GPU-based FAN controller for SUPERMICRO server
|
||||||
|
|
||||||
|
This controller enables automatic adjustments of FANs in SUPERMICRO servers based on GPU temperature. Only NVIDIA GPUs are supported since the tool uses nvidia-smi to parse the GPU temperature. FANs are controlled through IPMI tool (`impitool`) using the modified superfans (https://github.com/putnam/superfans/blob/master/superfans) script.
|
||||||
|
|
||||||
|
# Requirements
|
||||||
|
|
||||||
|
* Linux based only (
|
||||||
|
* Python 2.7
|
||||||
|
* nvidia drivers/tools (nvidia-smi)
|
||||||
|
* IPMI tool (impitool) with loaded
|
||||||
|
|
||||||
|
|
||||||
|
# Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python superfans_gpu_controller.py
|
||||||
|
```
|
||||||
|
|
||||||
|
|
@ -0,0 +1,233 @@
|
||||||
|
# superfans
|
||||||
|
# https://github.com/putnam/superfans
|
||||||
|
#
|
||||||
|
# 2019: modified by Domen Tabernik
|
||||||
|
#
|
||||||
|
|
||||||
|
import os, sys, subprocess, time, shutil, shlex
|
||||||
|
|
||||||
|
# list of FAN preset settings
|
||||||
|
FAN_PRESET_STANDARD=0
|
||||||
|
FAN_PRESET_FULL=1
|
||||||
|
FAN_PRESET_OPTIMAL=2
|
||||||
|
FAN_PRESET_HEAVYIO=4
|
||||||
|
FAN_PRESETS=[FAN_PRESET_STANDARD, FAN_PRESET_FULL, FAN_PRESET_OPTIMAL, FAN_PRESET_HEAVYIO]
|
||||||
|
#FAN_PRESETS_STR={
|
||||||
|
# 'standard' : FAN_PRESET_STANDARD,
|
||||||
|
# 'full' : FAN_PRESET_FULL,
|
||||||
|
# 'optimal' : FAN_PRESET_OPTIMAL,
|
||||||
|
# 'heavyio' : FAN_PRESET_HEAVYIO
|
||||||
|
#}
|
||||||
|
FAN_PRESETS_DESC={
|
||||||
|
FAN_PRESET_STANDARD : "Standard (Temp controlled, target 50%)",
|
||||||
|
FAN_PRESET_FULL : "Full (All fans at 100%)",
|
||||||
|
FAN_PRESET_OPTIMAL : "Optimal (Temp controlled, target 30%)",
|
||||||
|
FAN_PRESET_HEAVYIO : "Heavy IO (Temp controlled, CPU target 50%; Peripherals target 75%"
|
||||||
|
}
|
||||||
|
|
||||||
|
# list of FAN zones
|
||||||
|
FAN_ZONE_CPU1=0 # marked as FAN10 for CPU1 (right one)
|
||||||
|
FAN_ZONE_CPU2=1 # marked as FAN9 for CPU2 (left one)
|
||||||
|
FAN_ZONE_SYS2=2 # marked as FAN1-4 (right ones)
|
||||||
|
FAN_ZONE_SYS1=3 # marked as FAN5-8 (left ones)
|
||||||
|
FAN_ZONES=[FAN_ZONE_CPU1, FAN_ZONE_CPU2, FAN_ZONE_SYS2, FAN_ZONE_SYS1]
|
||||||
|
FAN_ZONES_STR={
|
||||||
|
FAN_ZONE_CPU1:'cpu1',
|
||||||
|
FAN_ZONE_CPU2:'cpu2',
|
||||||
|
FAN_ZONE_SYS2:'system2',
|
||||||
|
FAN_ZONE_SYS1:'system1',
|
||||||
|
}
|
||||||
|
|
||||||
|
# list of FANs and zone member association
|
||||||
|
FAN1 ='FAN1'
|
||||||
|
FAN2 ='FAN2'
|
||||||
|
FAN3 ='FAN3'
|
||||||
|
FAN4 ='FAN4'
|
||||||
|
FAN5 ='FAN5'
|
||||||
|
FAN6 ='FAN6'
|
||||||
|
FAN7 ='FAN7'
|
||||||
|
FAN8 ='FAN8'
|
||||||
|
FAN9 ='FAN9'
|
||||||
|
FAN10 ='FAN10'
|
||||||
|
|
||||||
|
FAN_ZONES_MEMBERS= {
|
||||||
|
FAN_ZONE_CPU1:FAN10,
|
||||||
|
FAN_ZONE_CPU2:FAN9,
|
||||||
|
FAN_ZONE_SYS2:[FAN1,FAN2,FAN3,FAN4],
|
||||||
|
FAN_ZONE_SYS1:[FAN5,FAN6,FAN7,FAN8],
|
||||||
|
}
|
||||||
|
|
||||||
|
# based on observations on SUPERMICRO_4029GP_TRT2 the
|
||||||
|
# SYS1 and SYS2 fans use the following linear equations to
|
||||||
|
# convert from RPM to % value
|
||||||
|
def SUPERMICRO_4029GP_TRT2_RPM_to_percent(rpm):
|
||||||
|
return rpm * 0.0098 - 11.5479
|
||||||
|
|
||||||
|
def set_fan_with_full_preset(config, speed, zone):
|
||||||
|
"""
|
||||||
|
Set fan speed to a fixed %.
|
||||||
|
Some chassis implement separate fan "zones" named CPU and Peripheral. To target specific zones, use the --zone option.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Make sure fans are on Full setting, or else this won't stick for long
|
||||||
|
s = get_preset(config)
|
||||||
|
if s is False:
|
||||||
|
print(time.ctime() +": Unable to get current fan status; exiting")
|
||||||
|
return False
|
||||||
|
|
||||||
|
if s != FAN_PRESET_FULL:
|
||||||
|
print(time.ctime() +": The fan controller is currently not set to Full mode (required for manual fan settings, which will otherwise be adjusted by the BMC within minutes); setting it now.")
|
||||||
|
set_preset(config, preset='full')
|
||||||
|
print(time.ctime() +": Waiting 5 seconds to let fans spin up...")
|
||||||
|
time.sleep(5)
|
||||||
|
|
||||||
|
ok = True
|
||||||
|
if zone == 'all' or zone == 'cpu':
|
||||||
|
ok = ipmi_raw_cmd('0x30 0x70 0x66 0x01 0x00 0x%02x' % speed, **config)
|
||||||
|
if ok and (zone == 'all' or zone == 'periph'):
|
||||||
|
ok = ipmi_raw_cmd('0x30 0x70 0x66 0x01 0x01 0x%02x' % speed, **config)
|
||||||
|
|
||||||
|
if ok:
|
||||||
|
print(time.ctime() +": Set %s fans on %s to %d%%." % (zone, config['hostname'], speed))
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
print(time.ctime() +": Unable to update fans.")
|
||||||
|
return False
|
||||||
|
|
||||||
|
def set_fan(config, speed, zone):
|
||||||
|
"""
|
||||||
|
Set fan speed to a fixed %.
|
||||||
|
Will be changed by Server if not in FULL preset (need to periodically call this)
|
||||||
|
"""
|
||||||
|
|
||||||
|
ok = ipmi_raw_cmd('0x30 0x70 0x66 0x01 0x%02x 0x%02x' % (zone, speed), **config)
|
||||||
|
|
||||||
|
if ok:
|
||||||
|
print(time.ctime() +": Set %s fans on %s to %d%%." % (FAN_ZONES_STR[zone], config['hostname'], speed))
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
print(time.ctime() +": Unable to update fans.")
|
||||||
|
return False
|
||||||
|
|
||||||
|
def get_fan(config, fan):
|
||||||
|
"""
|
||||||
|
Get fan speed in % (for one or more fans).
|
||||||
|
"""
|
||||||
|
|
||||||
|
fan_status_list = ipmi_fan_status(**config)
|
||||||
|
|
||||||
|
if type(fan) == list:
|
||||||
|
return_list = {}
|
||||||
|
for f in fan:
|
||||||
|
if fan_status_list.has_key(f):
|
||||||
|
return_list[f] = SUPERMICRO_4029GP_TRT2_RPM_to_percent(fan_status_list[f])
|
||||||
|
return return_list
|
||||||
|
elif fan_status_list.has_key(fan):
|
||||||
|
return SUPERMICRO_4029GP_TRT2_RPM_to_percent(fan_status_list[fan])
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _set_preset(config):
|
||||||
|
"""
|
||||||
|
Retrieves fan controller preset & fan speed.
|
||||||
|
"""
|
||||||
|
status = get_preset(config)
|
||||||
|
if status is False:
|
||||||
|
return False
|
||||||
|
if status in FAN_PRESETS:
|
||||||
|
s = FAN_PRESETS_DESC[status]
|
||||||
|
else:
|
||||||
|
s = "Unknown status code %d" % status
|
||||||
|
# manual fan ctl get(0)/set(1) cpu(0)/periph(1) duty(0-0x64)
|
||||||
|
# 0x30 0x70 0x66 0x00 0x00 0x64
|
||||||
|
fan_speed = ipmi_raw_cmd('0x30 0x70 0x66 0x00 0x00', **config)
|
||||||
|
if fan_speed is False:
|
||||||
|
return False
|
||||||
|
fan_speed2 = ipmi_raw_cmd('0x30 0x70 0x66 0x00 0x01', **config)
|
||||||
|
if fan_speed2 is False:
|
||||||
|
return False
|
||||||
|
|
||||||
|
print(time.ctime() +": Preset: %s" % s)
|
||||||
|
print(time.ctime() +": Current fan speed (CPU Zone): %d%%" % int(fan_speed, 16))
|
||||||
|
print(time.ctime() +": Current fan speed (Peripheral zone): %d%%" % int(fan_speed2, 16))
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def set_preset(config, preset):
|
||||||
|
if preset not in FAN_PRESETS:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if ipmi_raw_cmd("0x30 0x45 0x01 0x0%d" % preset, **config):
|
||||||
|
print(time.ctime() +": Updated preset on %s." % config['hostname'])
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
def ipmi_raw_cmd(raw_cmd, hostname = 'localhost', username=None, password=None, use_env=False):
|
||||||
|
|
||||||
|
if hostname == 'localhost':
|
||||||
|
if os.geteuid() != 0:
|
||||||
|
print(time.ctime() +": In order to communicate with the kernel's IPMI module, you must be root.")
|
||||||
|
sys.exit(1)
|
||||||
|
cmd = 'ipmitool raw %s' % raw_cmd
|
||||||
|
else:
|
||||||
|
if use_env:
|
||||||
|
cmd_pass = '-E'
|
||||||
|
else:
|
||||||
|
cmd_pass = '-P %s' % shlex.quote(password)
|
||||||
|
cmd = 'ipmitool -I lanplus -U %s %s -H %s raw %s' % (shlex.quote(username), cmd_pass, hostname, raw_cmd)
|
||||||
|
|
||||||
|
try:
|
||||||
|
s = subprocess.check_output(cmd + " 2>&1", shell=True)
|
||||||
|
except subprocess.CalledProcessError, ex:
|
||||||
|
print(time.ctime() +": Error: Problem running ipmitool")
|
||||||
|
print(time.ctime() +": Command: %s" % cmd)
|
||||||
|
print(time.ctime() +": Return code: %d" % ex)
|
||||||
|
return False
|
||||||
|
|
||||||
|
out = s.strip()
|
||||||
|
if out:
|
||||||
|
return out
|
||||||
|
else:
|
||||||
|
return True
|
||||||
|
|
||||||
|
def ipmi_fan_status(hostname = 'localhost', username=None, password=None, use_env=False):
|
||||||
|
cmd = 'ipmitool sensor | grep FAN'
|
||||||
|
|
||||||
|
if hostname == 'localhost':
|
||||||
|
if os.geteuid() != 0:
|
||||||
|
print(time.ctime() +": In order to communicate with the kernel's IPMI module, you must be root.")
|
||||||
|
sys.exit(1)
|
||||||
|
cmd = 'ipmitool sensor | grep FAN '
|
||||||
|
else:
|
||||||
|
if use_env:
|
||||||
|
cmd_pass = '-E'
|
||||||
|
else:
|
||||||
|
cmd_pass = '-P %s' % shlex.quote(password)
|
||||||
|
cmd = 'ipmitool -I lanplus -U %s %s -H %s sensor | grep FAN' % (shlex.quote(username), cmd_pass, hostname)
|
||||||
|
try:
|
||||||
|
s = subprocess.check_output(cmd + " 2>&1", shell=True)
|
||||||
|
except subprocess.CalledProcessError, ex:
|
||||||
|
print(time.ctime() +": Error: Problem running ipmitool")
|
||||||
|
print(time.ctime() +": Command: %s" % cmd)
|
||||||
|
print(time.ctime() +": Return code: %d" % ex)
|
||||||
|
return False
|
||||||
|
|
||||||
|
fan_status_return = {}
|
||||||
|
for fan_str in s.split("\n"):
|
||||||
|
if len(fan_str.strip()) > 0:
|
||||||
|
fan_stat = fan_str.split("|")
|
||||||
|
fan_name = fan_stat[0].strip()
|
||||||
|
fan_rpm = float(fan_stat[1].strip())
|
||||||
|
fan_status_return[fan_name] = fan_rpm
|
||||||
|
return fan_status_return
|
||||||
|
|
||||||
|
def get_preset(config):
|
||||||
|
try:
|
||||||
|
s = ipmi_raw_cmd('0x30 0x45 0x00', **config)
|
||||||
|
if s is False:
|
||||||
|
return False
|
||||||
|
return int(s)
|
||||||
|
except:
|
||||||
|
return False
|
||||||
|
|
@ -0,0 +1,130 @@
|
||||||
|
# Superfans GPU controller
|
||||||
|
#
|
||||||
|
# author: Domen Tabernik
|
||||||
|
# 2019
|
||||||
|
|
||||||
|
import time, superfans, subprocess
|
||||||
|
|
||||||
|
def retrieve_nvidia_gpu_temperature():
|
||||||
|
cmd = 'nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader'
|
||||||
|
|
||||||
|
s = subprocess.check_output(cmd + " 2>&1", shell=True)
|
||||||
|
if len(s) <= 0:
|
||||||
|
return False
|
||||||
|
|
||||||
|
out = [int(x.strip()) for x in s.split("\n") if len(x.strip()) > 0]
|
||||||
|
if out:
|
||||||
|
return out
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def superfans_gpu_controller(fan_settings, FAN_INCREASED_MIN_TIME=120, sleep_sec=2, gpu_moving_avg_num=5, fan_target_eps=1.0):
|
||||||
|
"""
|
||||||
|
Controller function that monitors GPU temperature in constant loop and adjusts FAN speeds based on provided `fan_settings`.
|
||||||
|
After the loop the default preset is restored.
|
||||||
|
|
||||||
|
:param fan_settings: dictionary that maps the temperature in deg C to % of fan speed
|
||||||
|
:param FAN_INCREASED_MIN_TIME: minimal time before a fan speed is again reduced (based on previous change) default=120
|
||||||
|
:param sleep_sec: loop sleep time (default=2 sec)
|
||||||
|
:param gpu_moving_avg_num: moving average for GPU i.e. the number of last measurements that are averaged (default=5)
|
||||||
|
:param fan_target_eps: tolerance of fan target w.r.t. the the actual value in deg C (default=1.0)
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
superfan_config = dict(hostname= 'localhost')
|
||||||
|
|
||||||
|
# save default present before changing anything
|
||||||
|
default_preset = superfans.get_preset(superfan_config)
|
||||||
|
print(time.ctime() + ': Started fan control using GPU temperature.')
|
||||||
|
print(time.ctime() + ': Using settings:')
|
||||||
|
for k in sorted(fan_settings.keys()):
|
||||||
|
print(time.ctime() + ': \t%d C = %d ' % (k, fan_settings[k]) + "%")
|
||||||
|
print(time.ctime() + ':')
|
||||||
|
try:
|
||||||
|
|
||||||
|
FAN_MEMBERS = superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_SYS1] + \
|
||||||
|
superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_SYS2]
|
||||||
|
|
||||||
|
# GPU moving average
|
||||||
|
previous_target_fan = None
|
||||||
|
previous_update_time = None
|
||||||
|
|
||||||
|
prev_GPU_temp = []
|
||||||
|
|
||||||
|
while True:
|
||||||
|
|
||||||
|
# get GPU temperature
|
||||||
|
GPU_temp = retrieve_nvidia_gpu_temperature()
|
||||||
|
|
||||||
|
prev_GPU_temp.append(GPU_temp)
|
||||||
|
|
||||||
|
# continue until we have enough sampels for moving average
|
||||||
|
if len(prev_GPU_temp) < gpu_moving_avg_num:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# retain last 5 mesurements
|
||||||
|
prev_GPU_temp = prev_GPU_temp[-gpu_moving_avg_num:]
|
||||||
|
mean_GPU_temp = prev_GPU_temp[0]
|
||||||
|
for gpu_temp in prev_GPU_temp[1:]:
|
||||||
|
mean_GPU_temp = [x+y for x,y in zip(gpu_temp, mean_GPU_temp)]
|
||||||
|
|
||||||
|
mean_GPU_temp = [x/len(prev_GPU_temp) for x in mean_GPU_temp]
|
||||||
|
|
||||||
|
max_gpu_temp = max(mean_GPU_temp)
|
||||||
|
|
||||||
|
for key_temp in sorted(fan_settings.keys())[::-1]:
|
||||||
|
if key_temp <= max_gpu_temp:
|
||||||
|
target_fan = fan_settings[key_temp]
|
||||||
|
break
|
||||||
|
|
||||||
|
|
||||||
|
current_fan_levels = superfans.get_fan(superfan_config, FAN_MEMBERS)
|
||||||
|
current_update_time = time.time()
|
||||||
|
diff_sys1_fan = [abs(current_fan_levels[FAN] - target_fan) for FAN in superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_SYS1]]
|
||||||
|
diff_sys2_fan = [abs(current_fan_levels[FAN] - target_fan) for FAN in superfans.FAN_ZONES_MEMBERS[superfans.FAN_ZONE_SYS2]]
|
||||||
|
|
||||||
|
disbale_update = False
|
||||||
|
|
||||||
|
if previous_update_time is not None and previous_target_fan is not None:
|
||||||
|
has_enough_time_elapsed = current_update_time - previous_update_time > FAN_INCREASED_MIN_TIME
|
||||||
|
is_level_down_change = target_fan < previous_target_fan
|
||||||
|
disbale_update = True if is_level_down_change and not has_enough_time_elapsed else False
|
||||||
|
|
||||||
|
if not disbale_update:
|
||||||
|
# Allow for 1% difference in target
|
||||||
|
update_sys1_fan = any([d > fan_target_eps for d in diff_sys1_fan])
|
||||||
|
update_sys2_fan = any([d > fan_target_eps for d in diff_sys2_fan])
|
||||||
|
if update_sys1_fan:
|
||||||
|
superfans.set_fan(superfan_config, target_fan, superfans.FAN_ZONE_SYS1)
|
||||||
|
|
||||||
|
if update_sys2_fan:
|
||||||
|
superfans.set_fan(superfan_config, target_fan, superfans.FAN_ZONE_SYS2)
|
||||||
|
|
||||||
|
if update_sys1_fan or update_sys2_fan:
|
||||||
|
print(time.ctime() + ': \tCurrent GPU measurements: %s' % ','.join(map(str,GPU_temp)))
|
||||||
|
print(time.ctime() + ': \tMoving average GPU measurements: %s' % ','.join(map(str,mean_GPU_temp)))
|
||||||
|
print(time.ctime() + ': \tTarget difference: SYS1 fan = %f; SYS2 fan = %f' % (max(diff_sys1_fan), max(diff_sys2_fan)))
|
||||||
|
print(time.ctime() + ':')
|
||||||
|
|
||||||
|
|
||||||
|
previous_target_fan = target_fan
|
||||||
|
previous_update_time = current_update_time
|
||||||
|
|
||||||
|
time.sleep(sleep_sec)
|
||||||
|
finally:
|
||||||
|
# revert back to default preset before finishing
|
||||||
|
superfans.set_preset(superfan_config, default_preset)
|
||||||
|
print(time.ctime() + ': Reverted back to system default.')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# fan settings = {[in deg C]: [% fan], ...}
|
||||||
|
fan_settings = {0: 25,
|
||||||
|
60: 30,
|
||||||
|
70: 36,
|
||||||
|
80: 40,
|
||||||
|
85: 45,
|
||||||
|
90: 50}
|
||||||
|
|
||||||
|
superfans_gpu_controller(fan_settings)
|
||||||
Loading…
Reference in New Issue