/*****************************************************************************\
 *  power_save.c - support node power saving mode. Nodes which have been
 *  idle for an extended period of time will be placed into a power saving
 *  mode by running an arbitrary script. This script can lower the voltage
 *  or frequency of the nodes or can completely power the nodes off.
 *  When the node is restored to normal operation, another script will be
 *  executed. Many parameters are available to control this mode of operation.
 *****************************************************************************
 *  Copyright (C) 2007 The Regents of the University of California.
 *  Copyright (C) 2008-2009 Lawrence Livermore National Security.
 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
 *  Written by Morris Jette <jette1@llnl.gov>
 *  CODE-OCEC-09-009. All rights reserved.
 *
 *  This file is part of Slurm, a resource management program.
 *  For details, see <https://slurm.schedmd.com/>.
 *  Please also read the included file: DISCLAIMER.
 *
 *  Slurm is free software; you can redistribute it and/or modify it under
 *  the terms of the GNU General Public License as published by the Free
 *  Software Foundation; either version 2 of the License, or (at your option)
 *  any later version.
 *
 *  In addition, as a special exception, the copyright holders give permission
 *  to link the code of portions of this program with the OpenSSL library under
 *  certain conditions as described in each individual source file, and
 *  distribute linked combinations including the two. You must obey the GNU
 *  General Public License in all respects for all of the code used other than
 *  OpenSSL. If you modify file(s) with this exception, you may extend this
 *  exception to your version of the file(s), but you are not obligated to do
 *  so. If you do not wish to do so, delete this exception statement from your
 *  version.  If you delete this exception statement from all source files in
 *  the program, then also delete it here.
 *
 *  Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
 *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 *  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 *  details.
 *
 *  You should have received a copy of the GNU General Public License along
 *  with Slurm; if not, write to the Free Software Foundation, Inc.,
 *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
\*****************************************************************************/

#include "config.h"

#define _GNU_SOURCE

#include <limits.h>	/* For LONG_MIN, LONG_MAX */
#include <signal.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>

#include "src/common/bitstring.h"
#include "src/common/data.h"
#include "src/common/env.h"
#include "src/common/fd.h"
#include "src/common/fetch_config.h"
#include "src/common/hostlist.h"
#include "src/common/list.h"
#include "src/common/macros.h"
#include "src/common/power_action.h"
#include "src/common/read_config.h"
#include "src/common/threadpool.h"
#include "src/common/xmalloc.h"
#include "src/common/xstring.h"

#include "src/interfaces/accounting_storage.h"
#include "src/interfaces/auth.h"
#include "src/interfaces/node_features.h"
#include "src/interfaces/serializer.h"

#include "src/slurmctld/agent.h"
#include "src/slurmctld/job_scheduler.h"
#include "src/slurmctld/locks.h"
#include "src/slurmctld/node_scheduler.h"
#include "src/slurmctld/power_save.h"
#include "src/slurmctld/slurmctld.h"
#include "src/slurmctld/slurmscriptd.h"
#include "src/slurmctld/state_save.h"
#include "src/slurmctld/trigger_mgr.h"

/* avoid magic numbers */
#define MAX_NODE_RATE (60000 /*millisecond*/ * 1 /*node/millisecond*/)

static pthread_t power_thread = 0;
static pthread_cond_t power_cond = PTHREAD_COND_INITIALIZER;
static pthread_mutex_t power_mutex = PTHREAD_MUTEX_INITIALIZER;
bool power_save_config = false;
bool power_save_enabled = false;
bool power_save_started = false;
bool power_save_debug = false;

int suspend_rate, resume_rate, max_timeout;
time_t last_log = (time_t) 0;
uint16_t slurmd_timeout;
static bool idle_on_node_suspend = false;
static uint16_t power_save_interval = 10;
static uint16_t power_save_min_interval = 0;

static list_t *power_action_list = NULL;

list_t *resume_job_list = NULL;

power_action_t *reboot_prog = NULL;
power_action_t *resume_fail_prog = NULL;
power_action_t *resume_prog = NULL;
power_action_t *suspend_prog = NULL;

typedef struct {
	power_action_t *action;
	char *features;
	hostlist_t *hostlist;
} power_script_args_t;

/* Group of nodes to suspend with the same PowerAction */
typedef struct {
	power_action_t *action;
	char *action_name;
	bitstr_t *bitmap;
	bool is_default;
	char *env_name;
	uint16_t protocol_version; /* protocol version to use */
} action_entry_t;

typedef struct {
	bool inited;
	uint64_t last_update;
	uint32_t max_tokens;
	uint32_t refill_count;
	uint32_t refill_period_msec;
	uint32_t tokens;
} rl_config_t;

typedef struct exc_node_partital {
	int exc_node_cnt;
	bitstr_t *exc_node_cnt_bitmap;
} exc_node_partital_t;
list_t *partial_node_list = NULL;

bitstr_t *exc_node_bitmap = NULL;

/* Possible SuspendExcStates */
static bool suspend_exc_down;
static uint32_t suspend_exc_state_flags;

static void _clear_power_config(void);
static void _do_power_work(time_t now, bool auto_power_save_enabled);
static int _do_resume_action(void *entry, void *payload);
static int _do_power_action(void *entry, void *payload);
static int _do_power_action_reboot(void *entry, void *payload);
static int   _init_power_config(void);
static void *_power_save_thread(void *arg);
static uint64_t _timespec_to_msec(struct timespec *tv);

static void _rl_init(rl_config_t *config,
		     uint32_t refill_count,
		     uint32_t max_tokens,
		     uint32_t refill_period_msec,
		     uint32_t start_tokens);
static uint32_t _rl_get_tokens(rl_config_t *config);
static void _rl_spend_token(rl_config_t *config);

static rl_config_t resume_rl_config, suspend_rl_config;

static void _free_action_entry(void *p)
{
	action_entry_t *e = p;
	xfree(e->action_name);
	/*
	 * e->action points at shared power_action_t in power_action_list and is
	 * never owned.
	 */
	FREE_NULL_BITMAP(e->bitmap);
	xfree(e->env_name);
	xfree(e);
}

static void _exc_node_part_free(void *x)
{
	exc_node_partital_t *ext_part_struct = (exc_node_partital_t *) x;
	FREE_NULL_BITMAP(ext_part_struct->exc_node_cnt_bitmap);
	xfree(ext_part_struct);
}

static int _parse_exc_nodes(void)
{
	int rc = SLURM_SUCCESS;
	char *save_ptr = NULL, *sep, *tmp, *tok, *node_cnt_str;
	hostlist_t *hostlist = NULL;

	/* Shortcut if ":<node_cnt>" is not used */
	sep = strchr(slurm_conf.suspend_exc_nodes, ':');
	if (!sep) {
		hostlist = nodespec_to_hostlist(slurm_conf.suspend_exc_nodes,
						false, NULL);
		rc = hostlist2bitmap(hostlist, false, &exc_node_bitmap);
		FREE_NULL_HOSTLIST(hostlist);
		return rc;
	}

	FREE_NULL_LIST(partial_node_list);
	partial_node_list = list_create(_exc_node_part_free);
	tmp = xstrdup(slurm_conf.suspend_exc_nodes);
	tok = strtok_r(tmp, ",", &save_ptr);
	while (tok) {
		bitstr_t *exc_node_cnt_bitmap = NULL;
		long ext_node_cnt = 0;
		exc_node_partital_t *ext_part_struct;

		if ((node_cnt_str = xstrstr(tok, ":"))) {
			*node_cnt_str = '\0';
			ext_node_cnt = strtol(node_cnt_str + 1, NULL, 10);
		}
		hostlist = nodespec_to_hostlist(tok, false, NULL);
		rc = hostlist2bitmap(hostlist, false, &exc_node_cnt_bitmap);
		FREE_NULL_HOSTLIST(hostlist);

		if (!ext_node_cnt) {
			ext_node_cnt = bit_set_count(exc_node_cnt_bitmap);
		}
		if (bit_set_count(exc_node_cnt_bitmap)) {
			ext_part_struct = xmalloc(sizeof(exc_node_partital_t));
			ext_part_struct->exc_node_cnt = (int) ext_node_cnt;
			ext_part_struct->exc_node_cnt_bitmap =
				exc_node_cnt_bitmap;
			list_append(partial_node_list, ext_part_struct);
		} else
			FREE_NULL_BITMAP(exc_node_cnt_bitmap);
		tok = strtok_r(NULL, ",", &save_ptr);
	}
	xfree(tmp);
	if (list_is_empty(partial_node_list))
		FREE_NULL_LIST(partial_node_list);

	return rc;
}

/*
 * Print elements of the excluded nodes with counts
 */
static int _list_part_node_lists(void *x, void *arg)
{
	exc_node_partital_t *ext_part_struct = (exc_node_partital_t *) x;
	char *tmp = bitmap2node_name(ext_part_struct->exc_node_cnt_bitmap);
	log_flag(POWER, "exclude %d nodes from %s",
		 ext_part_struct->exc_node_cnt, tmp);
	xfree(tmp);
	return 0;

}

static void _parse_exc_states(void)
{
	char *buf, *tok, *saveptr;
	/* Flags in _node_state_suspendable() are already excluded */
	uint32_t excludable_state_flags = NODE_STATE_CLOUD |
					  NODE_STATE_DRAIN |
					  NODE_STATE_DYNAMIC_FUTURE |
					  NODE_STATE_DYNAMIC_NORM |
					  NODE_STATE_EXTERNAL |
					  NODE_STATE_FAIL |
					  NODE_STATE_INVALID_REG |
					  NODE_STATE_MAINT |
					  NODE_STATE_NO_RESPOND |
					  NODE_STATE_PLANNED |
					  NODE_STATE_RES;

	buf = xstrdup(slurm_conf.suspend_exc_states);
	for (tok = strtok_r(buf, ",", &saveptr); tok;
	     tok = strtok_r(NULL, ",", &saveptr)) {
		uint32_t flag = 0;

		/* Base node states */
		if (!xstrncasecmp(tok, "DOWN", MAX(strlen(tok), 2))){
			suspend_exc_down = true;
			continue;
		}

		/* Flag node states */
		flag = parse_node_state_flag(tok);
		if (flag & excludable_state_flags) {
			suspend_exc_state_flags |= flag;
			continue;
		}

		error("Invalid SuspendExcState %s", tok);
	}
	xfree(buf);

	if (power_save_debug) {
		char *exc_states_str =
			node_state_string_complete(suspend_exc_state_flags);
		log_flag(POWER, "suspend_exc_down=%d suspend_exc_state_flags=%s",
			 suspend_exc_down, exc_states_str);
		xfree(exc_states_str);
	}
}

/*
 * Is it possible to suspend this node
 */
static bool _node_state_suspendable(node_record_t *node_ptr)
{
	/* Must have idle or down base state */
	if (!IS_NODE_IDLE(node_ptr) && !IS_NODE_DOWN(node_ptr))
		return false;

	/* Must not have these flags */
	if (IS_NODE_COMPLETING(node_ptr) ||
	    IS_NODE_POWERING_UP(node_ptr) ||
	    IS_NODE_POWERING_DOWN(node_ptr) ||
	    IS_NODE_REBOOT_ISSUED(node_ptr) ||
	    IS_NODE_REBOOT_REQUESTED(node_ptr))
	    return false;

	return true;
}

/*
 * Should this node be suspended after SuspendTime has elapsed
 */
static bool _node_state_should_suspend(node_record_t *node_ptr)
{
	/* SuspendExcStates */
	if (suspend_exc_down && IS_NODE_DOWN(node_ptr))
		return false;
	if (suspend_exc_state_flags & node_ptr->node_state)
		return false;

	return true;
}

/*
 * Is the node in an "active" state, meaning that it is powered up and
 * idle or allocated
 */
static bool _node_state_active(node_record_t *node_ptr)
{
	/* inactive if not one of these */
	if (!IS_NODE_ALLOCATED(node_ptr) &&
	    !IS_NODE_IDLE(node_ptr)) {
		return false;
	}

	/* inactive if any of these */
	if (IS_NODE_POWERING_DOWN(node_ptr) ||
	    IS_NODE_POWERING_UP(node_ptr) ||
	    IS_NODE_POWERED_DOWN(node_ptr) ||
	    IS_NODE_DRAIN(node_ptr) ||
	    (node_ptr->sus_job_cnt > 0)) {
		return false;
	}
	/* powering up or completing included here */
	/* active */
	return true;
}

/*
 * Select the nodes specific nodes to be excluded from consideration for
 * suspension based upon the node states and specified count. Active
 * (powered up and idle or allocated) and suspendable nodes are
 * counted when fulfilling the exclude count.
 */
static int _pick_exc_nodes(void *x, void *arg)
{
	bitstr_t **orig_exc_nodes = (bitstr_t **) arg;
	exc_node_partital_t *ext_part_struct = (exc_node_partital_t *) x;
	bitstr_t *exc_node_cnt_bitmap;
	bitstr_t *suspendable_bitmap = NULL;
	bitstr_t *active_bitmap = NULL;
	int avail_node_cnt, exc_node_cnt, active_count;
	node_record_t *node_ptr = NULL;
	hostlist_t *active_hostlist, *suspend_hostlist;
	char *suspend_str = NULL, *active_str = NULL;

	exc_node_cnt_bitmap = ext_part_struct->exc_node_cnt_bitmap;
	exc_node_cnt = ext_part_struct->exc_node_cnt;

	avail_node_cnt = bit_set_count(exc_node_cnt_bitmap);
	if (exc_node_cnt >= avail_node_cnt) {
		/* Exclude all nodes in this set */
		exc_node_cnt_bitmap = bit_copy(exc_node_cnt_bitmap);
	} else {
		/* gather suspendable nodes */
		/* count active but not suspendable */
		active_count = 0;
		suspendable_bitmap = bit_alloc(bit_size(exc_node_cnt_bitmap));
		active_bitmap = bit_alloc(bit_size(exc_node_cnt_bitmap));

		for (int i = 0;
		     (node_ptr = next_node_bitmap(exc_node_cnt_bitmap, &i));
		     i++) {
			/*
			 * a powered down node is technically suspendable, but
			 * it should not count toward suspendable nodes here
			 */
			if (_node_state_suspendable(node_ptr) &&
			    !IS_NODE_POWERED_DOWN(node_ptr)) {
				bit_set(suspendable_bitmap, i);
			} else if (_node_state_active(node_ptr)) {
				bit_set(active_bitmap, i);
				active_count++;
			}
		}

		if (power_save_debug && (get_log_level() >= LOG_LEVEL_DEBUG)) {
			active_hostlist = bitmap2hostlist(active_bitmap);
			active_str = slurm_hostlist_ranged_string_xmalloc(
				active_hostlist);
			suspend_hostlist = bitmap2hostlist(suspendable_bitmap);
			suspend_str = slurm_hostlist_ranged_string_xmalloc(
				suspend_hostlist);

			log_flag(POWER, "avoid %d nodes: active: %d (%s), suspendable: (%s)",
			         exc_node_cnt, active_count, active_str,
				 suspend_str);
			FREE_NULL_HOSTLIST(active_hostlist);
			FREE_NULL_HOSTLIST(suspend_hostlist);
			xfree(active_str);
			xfree(suspend_str);
		}

		/* Exclude any remaining suspendable nodes */
		exc_node_cnt -= active_count;
		if (exc_node_cnt > 0) {
			bit_pick_firstn(suspendable_bitmap, exc_node_cnt);
		} else {
			bit_clear_all(suspendable_bitmap);
		}

		exc_node_cnt_bitmap = suspendable_bitmap;
		FREE_NULL_BITMAP(active_bitmap);
	}

	if (*orig_exc_nodes == NULL) {
		*orig_exc_nodes = exc_node_cnt_bitmap;
	} else {
		bit_or(*orig_exc_nodes, exc_node_cnt_bitmap);
		FREE_NULL_BITMAP(exc_node_cnt_bitmap);
	}

	return 0;
}

static bool _node_needs_reboot(node_record_t *node_ptr)
{
	bool want_reboot = false;
	if (!IS_NODE_REBOOT_REQUESTED(node_ptr))
		return false;
	if (IS_NODE_REBOOT_ISSUED(node_ptr)) {
		debug2("%s: Still waiting for boot of node %s",
		       __func__, node_ptr->name);
		return false;
	}
	if (IS_NODE_COMPLETING(node_ptr)) {
		return false;
	}
	/* only active idle nodes, don't reboot
	 * nodes that are idle but have suspended
	 * jobs on them
	 */
	if (IS_NODE_IDLE(node_ptr) && !IS_NODE_NO_RESPOND(node_ptr) &&
	    !IS_NODE_POWERING_UP(node_ptr) && node_ptr->sus_job_cnt == 0)
		want_reboot = true;
	else if (IS_NODE_FUTURE(node_ptr) &&
		 (node_ptr->last_response == (time_t) 0))
		want_reboot = true; /* system just restarted */
	else if (IS_NODE_DOWN(node_ptr))
		want_reboot = true;
	else
		want_reboot = false;
	if (!want_reboot) {
		return false;
	}
	return true;
}

static int _find_action_entry(void *action_entry_ptr, void *key)
{
	action_entry_t *action_entry = (action_entry_t *) action_entry_ptr;
	return !xstrcasecmp(action_entry->action_name, (char *) key);
}

static list_t *_build_power_action_list(bitstr_t *node_bitmap,
					char *default_action,
					bool force_default)
{
	node_record_t *node_ptr = NULL;
	action_entry_t *entry = NULL;
	list_t *action_list = list_create(_free_action_entry);
	bool is_default = false;
	for (int i = 0; (node_ptr = next_node_bitmap(node_bitmap, &i)); i++) {
		char *key = NULL;
		if (node_ptr->power_action_name) {
			if (force_default) {
				debug3("%s: force default power action '%s' for node %s",
				       __func__, default_action ? default_action : "(none)",
				       node_ptr->name);
				key = xstrdup(default_action);
				is_default = true;
				xfree(node_ptr->power_action_name);
			} else {
				key = node_ptr->power_action_name;
				node_ptr->power_action_name = NULL;
			}
		} else if (default_action) {
			key = xstrdup(default_action);
			is_default = true;
		}
		if (!key) {
			error("power_save: no power action found for node %s (default_action=%s)",
			      node_ptr->name,
			      default_action ? default_action : "(none)");
			continue;
		}
		entry = list_find_first(action_list, _find_action_entry, key);
		if (!entry) {
			entry = xmalloc(sizeof(action_entry_t));
			entry->action = NULL;
			entry->action_name = xstrdup(key);
			entry->is_default = is_default;
			entry->bitmap = bit_alloc(node_record_count);
			entry->protocol_version = SLURM_PROTOCOL_VERSION;
			list_append(action_list, entry);
			debug3("power action entry created for %s",
			       key);
		} else {
			debug3("power action entry found for %s",
			       key);
		}
		bit_set(entry->bitmap, node_ptr->index);
		if (node_ptr->protocol_version < entry->protocol_version) {
			debug2("%s: in order to run PowerAction %s, node %s requires downgrading protocol version from %hu to %hu",
				__func__, key, node_ptr->name,
			       entry->protocol_version,
			       node_ptr->protocol_version);
			entry->protocol_version = node_ptr->protocol_version;
		}
		xfree(key);
	}
	return action_list;
}

/* Perform any power change work to nodes */
static void _do_power_work(time_t now, bool auto_power_save_enabled)
{
	int i, susp_total = 0;
	uint32_t susp_state;
	bitstr_t *avoid_node_bitmap = NULL, *failed_node_bitmap = NULL;
	bitstr_t *wake_node_bitmap = NULL, *sleep_node_bitmap = NULL;
	bitstr_t *reboot_node_bitmap = NULL;
	node_record_t *node_ptr;
	data_t *resume_json_data = NULL;
	data_t *jobs_data = NULL;
	list_itr_t *resume_job_iter = NULL;
	bitstr_t *job_power_node_bitmap;
	uint32_t *job_id_ptr;
	bool nodes_updated = false;

	/* Identify nodes to avoid considering for suspend */
	if (partial_node_list) {
		(void) list_for_each(partial_node_list, _pick_exc_nodes,
				     &avoid_node_bitmap);
	}
	if (exc_node_bitmap) {
		if (avoid_node_bitmap)
			bit_or(avoid_node_bitmap, exc_node_bitmap);
		else
			avoid_node_bitmap = bit_copy(exc_node_bitmap);
	}

	if (avoid_node_bitmap && power_save_debug &&
	    (get_log_level() >= LOG_LEVEL_DEBUG)) {
		char *tmp = bitmap2node_name(avoid_node_bitmap);
		debug("avoid nodes %s", tmp);
		xfree(tmp);
	}

	/*
	 * Build job to node mapping for json output
	 * all_nodes = all nodes that need to be resumed this iteration
	 * jobs[] - list of job to node mapping of nodes that the job needs to
	 * be resumed for job. Multiple jobs can request the same nodes. Report
	 * all jobs to node mapping for this iteration.
	 * e.g.
	 * {
	 * all_nodes: n[1-3]
	 * jobs: [{job_id:123, nodes:n[1-3]}, {job_id:124, nodes:n[1-3]}]
	 * }
	 */
	resume_json_data = data_set_dict(data_new());
	jobs_data = data_set_list(data_key_set(resume_json_data, "jobs"));

	job_power_node_bitmap = bit_alloc(node_record_count);

	resume_job_iter = list_iterator_create(resume_job_list);
	while ((job_id_ptr = list_next(resume_job_iter))) {
		char *nodes, *node_bitmap;
		job_record_t *job_ptr;
		data_t *job_node_data;
		bitstr_t *need_resume_bitmap, *to_resume_bitmap;

		if ((resume_rate > 0) && (!_rl_get_tokens(&resume_rl_config))) {
			log_flag(POWER, "resume rate reached");
			break;
		}

		if (!(job_ptr = find_job_record(*job_id_ptr))) {
			log_flag(POWER, "%u needed resuming but is gone now",
				 *job_id_ptr);
			list_delete_item(resume_job_iter);
			continue;
		}
		if (!IS_JOB_CONFIGURING(job_ptr)) {
			log_flag(POWER, "%pJ needed resuming but isn't configuring anymore",
				 job_ptr);
			list_delete_item(resume_job_iter);
			continue;
		}
		if (!bit_overlap_any(job_ptr->node_bitmap,
		                     power_down_node_bitmap)) {
			log_flag(POWER, "%pJ needed resuming but nodes aren't power_save anymore",
				 job_ptr);
			list_delete_item(resume_job_iter);
			continue;
		}

		to_resume_bitmap = bit_alloc(node_record_count);

		need_resume_bitmap = bit_copy(job_ptr->node_bitmap);
		bit_and(need_resume_bitmap, power_down_node_bitmap);

		for (int i = 0; next_node_bitmap(need_resume_bitmap, &i); i++) {
			if ((resume_rate == 0) ||
			    (_rl_get_tokens(&resume_rl_config))) {
				_rl_spend_token(&resume_rl_config);
				bit_set(job_power_node_bitmap, i);
				bit_set(to_resume_bitmap, i);
				bit_clear(need_resume_bitmap, i);
			}
		}

		job_node_data = data_set_dict(data_list_append(jobs_data));
		data_set_string(data_key_set(job_node_data, "exclusive"),
				job_exclusive_display_string(
					get_job_exclusive_display_value(
						job_ptr)));
		data_set_string(data_key_set(job_node_data, "extra"),
				job_ptr->extra);
		data_set_int(data_key_set(job_node_data, "job_id"),
			     job_ptr->job_id);
		data_set_int(data_key_set(job_node_data, "priority"),
			     job_ptr->priority);
		data_set_string(data_key_set(job_node_data, "features"),
				job_ptr->details->features_use);
		if ((node_bitmap = bitmap2node_name(job_ptr->node_bitmap))) {
			data_set_string_own(data_key_set(job_node_data,
							 "nodes_alloc"),
					    node_bitmap);
		}
		nodes = bitmap2node_name(to_resume_bitmap);
		data_set_string_own(data_key_set(job_node_data, "nodes_resume"),
				    nodes);
		data_set_string(data_key_set(job_node_data, "oversubscribe"),
				job_oversubscribe_string(
					get_job_oversubscribe_value(job_ptr)));
		data_set_string(data_key_set(job_node_data, "partition"),
				job_ptr->part_ptr->name);
		data_set_string(data_key_set(job_node_data, "reservation"),
				job_ptr->resv_name);

		/* No more nodes to power up, remove job from list */
		if (!bit_set_count(need_resume_bitmap)) {
			log_flag(POWER, "no more nodes to resume for job %pJ",
				 job_ptr);
			list_delete_item(resume_job_iter);
		} else if (power_save_debug) {
			char *still_needed_nodes =
				bitmap2node_name(need_resume_bitmap);
			log_flag(POWER, "%s still left to boot for %pJ",
				 still_needed_nodes, job_ptr);
			xfree(still_needed_nodes);
		}

		FREE_NULL_BITMAP(need_resume_bitmap);
		FREE_NULL_BITMAP(to_resume_bitmap);
	}

	/* Build bitmaps identifying each node which should change state */
	for (i = 0; (node_ptr = next_node(&i)); i++) {
		bool has_job_to_resume;
		bool not_rate_limited;
		bool no_suspended_jobs;
		bool suspend_time_exceeded;
		bool not_avoid_node;

		susp_state = IS_NODE_POWERED_DOWN(node_ptr);

		if (susp_state)
			susp_total++;

		/* Resume nodes as appropriate */
		has_job_to_resume =
			(auto_power_save_enabled &&
			 bit_test(job_power_node_bitmap, node_ptr->index));
		not_rate_limited =
			(resume_rate == 0 || _rl_get_tokens(&resume_rl_config));
		if (has_job_to_resume || (susp_state && not_rate_limited &&
					  !IS_NODE_POWERING_DOWN(node_ptr) &&
					  IS_NODE_POWER_UP(node_ptr))) {
			if (wake_node_bitmap == NULL) {
				wake_node_bitmap = bit_alloc(node_record_count);
			}
			if (!(bit_test(job_power_node_bitmap,
				       node_ptr->index)))
				_rl_spend_token(&resume_rl_config);
			node_ptr->node_state &= (~NODE_STATE_POWER_UP);
			node_ptr->node_state &= (~NODE_STATE_POWERED_DOWN);
			node_ptr->node_state |=   NODE_STATE_POWERING_UP;
			node_ptr->node_state |=   NODE_STATE_NO_RESPOND;
			bit_clear(power_down_node_bitmap, node_ptr->index);
			bit_set(power_up_node_bitmap, node_ptr->index);
			node_ptr->boot_req_time = now;
			bit_set(booting_node_bitmap, node_ptr->index);
			bit_set(wake_node_bitmap,    node_ptr->index);

			bit_clear(job_power_node_bitmap, node_ptr->index);
			if (IS_NODE_DRAIN(node_ptr) || IS_NODE_DOWN(node_ptr))
				clusteracct_storage_g_node_down(
					acct_db_conn, node_ptr, now,
					node_ptr->reason, node_ptr->reason_uid);
			else
				clusteracct_storage_g_node_up(acct_db_conn,
							      node_ptr, now);
			nodes_updated = true;
		}

		/* Suspend nodes as appropriate */
		not_rate_limited = (suspend_rate == 0 ||
				    _rl_get_tokens(&suspend_rl_config));
		no_suspended_jobs = (node_ptr->sus_job_cnt == 0);
		suspend_time_exceeded = (auto_power_save_enabled &&
					 (node_ptr->last_busy != 0) &&
					 (node_ptr->last_busy <
					  (now - node_ptr->suspend_time)));
		not_avoid_node =
			(avoid_node_bitmap == NULL ||
			 (!bit_test(avoid_node_bitmap, node_ptr->index)));
		if (_node_state_suspendable(node_ptr) && not_rate_limited &&
		    no_suspended_jobs &&
		    (IS_NODE_POWER_DOWN(node_ptr) ||
		     (suspend_time_exceeded &&
		      _node_state_should_suspend(node_ptr) &&
		      not_avoid_node))) {
			if (sleep_node_bitmap == NULL) {
				sleep_node_bitmap =
					bit_alloc(node_record_count);
			}

			/* Clear power_down_asap */
			if (IS_NODE_POWER_DOWN(node_ptr) &&
			    IS_NODE_DRAIN(node_ptr)) {
				node_ptr->node_state &= (~NODE_STATE_DRAIN);
			}

			_rl_spend_token(&suspend_rl_config);
			node_ptr->node_state |= NODE_STATE_POWERING_DOWN;
			node_ptr->node_state &= (~NODE_STATE_POWER_DOWN);
			node_ptr->node_state &= (~NODE_STATE_POWERED_DOWN);
			node_ptr->node_state &= (~NODE_STATE_NO_RESPOND);
			bit_set(power_down_node_bitmap, node_ptr->index);
			bit_clear(power_up_node_bitmap, node_ptr->index);
			bit_set(sleep_node_bitmap,   node_ptr->index);

			/* Don't allocate until after SuspendTimeout */
			bit_clear(avail_node_bitmap, node_ptr->index);
			node_ptr->power_save_req_time = now;

			if (idle_on_node_suspend) {
				if (IS_NODE_DOWN(node_ptr)) {
					trigger_node_up(node_ptr);
				}

				node_ptr->node_state = NODE_STATE_IDLE |
						       (node_ptr->node_state &
							NODE_STATE_FLAGS);
				node_ptr->node_state &= (~NODE_STATE_DRAIN);
				node_ptr->node_state &= (~NODE_STATE_FAIL);
			}
			nodes_updated = true;
		}

		if (IS_NODE_POWERING_DOWN(node_ptr) &&
		    ((node_ptr->power_save_req_time + node_ptr->suspend_timeout)
		     < now)) {
			node_ptr->node_state &= (~NODE_STATE_INVALID_REG);
			node_ptr->node_state &= (~NODE_STATE_POWERING_DOWN);
			node_ptr->node_state |= NODE_STATE_POWERED_DOWN;

			if (IS_NODE_CLOUD(node_ptr)) {
				/* Reset hostname and addr to node's name. */
				set_node_comm_name(node_ptr, NULL,
						   node_ptr->name);
			}

			if (!IS_NODE_DOWN(node_ptr) &&
			    !IS_NODE_DRAIN(node_ptr) &&
			    !IS_NODE_FAIL(node_ptr))
				make_node_avail(node_ptr);

			node_ptr->last_busy = 0;
			node_ptr->power_save_req_time = 0;
			node_mgr_reset_node_stats(node_ptr);

			reset_node_active_features(node_ptr);
			reset_node_instance(node_ptr);
			reset_node_topology(node_ptr);

			clusteracct_storage_g_node_down(
				acct_db_conn, node_ptr, now,
				"Powered down after SuspendTimeout",
				node_ptr->reason_uid);
			nodes_updated = true;
		}

		/*
		 * Down nodes as if not resumed by ResumeTimeout
		 */
		if (bit_test(booting_node_bitmap, node_ptr->index) &&
		    (node_ptr->resume_timeout != INFINITE16) &&
		    (now >
		     (node_ptr->boot_req_time + node_ptr->resume_timeout)) &&
		    IS_NODE_POWERING_UP(node_ptr) &&
		    IS_NODE_NO_RESPOND(node_ptr)) {
			info("node %s not resumed by ResumeTimeout(%d), setting DOWN and POWERED_DOWN",
			     node_ptr->name, node_ptr->resume_timeout);
			node_ptr->node_state &= (~NODE_STATE_DRAIN);
			node_ptr->node_state &= (~NODE_STATE_POWER_DOWN);
			node_ptr->node_state &= (~NODE_STATE_POWERING_UP);
			node_ptr->node_state &= (~NODE_STATE_NO_RESPOND);
			node_ptr->node_state &= (~NODE_STATE_REBOOT_ISSUED);
			node_ptr->node_state |= NODE_STATE_POWERED_DOWN;

			reset_node_active_features(node_ptr);
			reset_node_instance(node_ptr);
			reset_node_topology(node_ptr);

			/*
			 * set_node_down_ptr() will remove the node from the
			 * avail_node_bitmap.
			 *
			 * Call AFTER setting state adding POWERED_DOWN so that
			 * the node is marked as "planned down" in the usage
			 * tables because:
			 * set_node_down_ptr()->_make_node_down()->
			 * clusteracct_storage_g_node_down().
			 */
			set_node_down_ptr(node_ptr, "ResumeTimeout reached");
			bit_set(power_down_node_bitmap, node_ptr->index);
			bit_clear(power_up_node_bitmap, node_ptr->index);
			bit_clear(booting_node_bitmap, node_ptr->index);
			node_ptr->last_busy = 0;
			node_ptr->boot_req_time = 0;
			node_mgr_reset_node_stats(node_ptr);

			if (resume_fail_prog) {
				if (!failed_node_bitmap) {
					failed_node_bitmap =
						bit_alloc(node_record_count);
				}
				bit_set(failed_node_bitmap, node_ptr->index);
			}
			nodes_updated = true;
		}

		/*
		 * Reboot nodes as appropriate
		 */
		if (_node_needs_reboot(node_ptr)) {
			if (reboot_node_bitmap == NULL) {
				reboot_node_bitmap =
					bit_alloc(node_record_count);
			}
			bit_set(reboot_node_bitmap, node_ptr->index);
			/*
                         * node_ptr->node_state &= ~NODE_STATE_MAINT;
                         * The NODE_STATE_MAINT bit will just get set again as
                         * long as the node remains in the maintenance
                         * reservation, so don't clear it here because it won't
                         * do anything.
                         */
			node_ptr->node_state &= NODE_STATE_FLAGS;
			node_ptr->node_state |= NODE_STATE_DOWN;
			node_ptr->node_state &= ~NODE_STATE_REBOOT_REQUESTED;
			node_ptr->node_state |= NODE_STATE_REBOOT_ISSUED;

			bit_clear(avail_node_bitmap, node_ptr->index);
			bit_clear(idle_node_bitmap, node_ptr->index);

			/* Unset this as this node is not in reboot ASAP
                         * anymore. */
			bit_clear(asap_node_bitmap, node_ptr->index);

			node_ptr->boot_req_time = now;

			set_node_reason(node_ptr, "reboot issued", now);

			clusteracct_storage_g_node_down(acct_db_conn, node_ptr,
							now, NULL,
							slurm_conf
								.slurm_user_id);
			nodes_updated = true;
		}
		if (IS_NODE_REBOOT_ISSUED(node_ptr) &&
		    (node_ptr->resume_timeout != INFINITE16) &&
		    node_ptr->boot_req_time &&
		    (now >
		     (node_ptr->boot_req_time + node_ptr->resume_timeout))) {
			/*
			 * Remove states now so that event state shows as DOWN.
			 * Does not remove drain state in case it was set by
			 * scontrol update nodename.
			 */
			node_ptr->node_state &= (~NODE_STATE_POWERING_UP);
			node_ptr->node_state &= (~NODE_STATE_REBOOT_ISSUED);
			node_ptr->boot_req_time = 0;
			set_node_down_ptr(node_ptr, "reboot timed out");

			bit_clear(rs_node_bitmap, node_ptr->index);
		}
	}
	FREE_NULL_BITMAP(avoid_node_bitmap);
	if (power_save_debug && ((now - last_log) > 600) && (susp_total > 0)) {
		log_flag(POWER, "Power save mode: %d nodes", susp_total);
		last_log = now;
	}

	if (sleep_node_bitmap) {
		/* Group nodes by power_down_action (NULL = default SuspendProgram) */
		list_t *sleep_action_list =
			_build_power_action_list(sleep_node_bitmap,
						 suspend_prog ?
							 suspend_prog->name :
							 NULL,
						 false);
		FREE_NULL_BITMAP(sleep_node_bitmap);

		list_for_each(sleep_action_list, _do_power_action, NULL);
		FREE_NULL_LIST(sleep_action_list);
		nodes_updated = true;
	}

	if (wake_node_bitmap) {
		list_t *wake_action_list =
			_build_power_action_list(wake_node_bitmap,
						 resume_prog ?
							 resume_prog->name :
							 NULL,
						 false);
		FREE_NULL_BITMAP(wake_node_bitmap);
		list_for_each(wake_action_list, _do_resume_action,
			      resume_json_data);
		FREE_NULL_LIST(wake_action_list);
		nodes_updated = true;
	}

	if (failed_node_bitmap) {
		list_t *fail_action_list =
			_build_power_action_list(failed_node_bitmap,
						 resume_fail_prog ?
							 resume_fail_prog
								 ->name :
							 NULL,
						 false);
		FREE_NULL_BITMAP(failed_node_bitmap);

		list_for_each(fail_action_list, _do_power_action, NULL);
		FREE_NULL_LIST(fail_action_list);
		nodes_updated = true;
	}

	if (reboot_node_bitmap) {
		list_t *reboot_action_list =
			_build_power_action_list(reboot_node_bitmap,
						 reboot_prog ?
							 reboot_prog->name :
							 NULL,
						 false);
		FREE_NULL_BITMAP(reboot_node_bitmap);
		list_for_each(reboot_action_list, _do_power_action_reboot,
			      NULL);
		FREE_NULL_LIST(reboot_action_list);
	}

	if (nodes_updated)
		last_node_update = time(NULL);

	list_iterator_destroy(resume_job_iter);
	FREE_NULL_DATA(resume_json_data);
	FREE_NULL_BITMAP(job_power_node_bitmap);
}

static int _do_resume_action(void *e, void *payload)
{
	action_entry_t *entry = e;
	data_t *resume_json_data = payload;
	char *json = NULL;
	char *nodes = bitmap2node_name(entry->bitmap);
	if (!nodes) {
		error("power_save: failed bitmap2nodename");
		return SLURM_ERROR;
	}
	data_set_string(data_key_set(resume_json_data, "all_nodes_resume"),
			nodes);
	serialize_g_data_to_string(&json, NULL, resume_json_data,
				   MIME_TYPE_JSON, SER_FLAGS_COMPACT);
	data_key_unset(resume_json_data, "all_nodes_resume");

	entry->env_name = xstrdup("SLURM_RESUME_FILE");
	_do_power_action(entry, json);
	xfree(nodes);
	xfree(json);
	return SLURM_SUCCESS;
}

static int _do_power_action(void *e, void *payload)
{
	action_entry_t *entry = e;
	char *json = payload;
	char *host_str = NULL;
	hostlist_t *hl;
	bitstr_t *bitmap = entry->bitmap;
	power_action_t *action = NULL;

	hl = bitmap2hostlist(bitmap);
	if (!hl) {
		error("power_save: failed bitmap2hostlist");
		return SLURM_ERROR;
	}
	host_str = hostlist_ranged_string_xmalloc(hl);
	action = entry->action;
	if (!action) {
		action = power_action_find(power_action_list,
					   entry->action_name);
		if (!action) {
			error("Invalid PowerAction '%s', skipping nodes %s",
			      entry->action_name, host_str);
			xfree(host_str);
			FREE_NULL_HOSTLIST(hl);
			return SLURM_ERROR;
		}
	}
	if (!power_action_valid_prog(action)) {
		info("Invalid PowerAction '%s', skipping nodes %s",
		      action->name, host_str);
		xfree(host_str);
		FREE_NULL_HOSTLIST(hl);
		return SLURM_ERROR;
	}

	if (action->on_slurmctld) {
		slurmscriptd_run_power_action(action, host_str, NULL, 0,
					      max_timeout, entry->env_name,
					      json);
		log_flag(POWER, "Issued slurmctld power action %s on nodes %s",
			 action->name, host_str);
		FREE_NULL_HOSTLIST(hl);
	} else {
		/* Location=slurmd: send RPC to each slurmd to run the script */
		agent_arg_t *agent_args;
		run_power_action_msg_t *action_msg;

		if (entry->protocol_version < SLURM_26_05_PROTOCOL_VERSION) {
			error("power_save: nodes needing power action %s must be running at least Slurm 26.05",
			      action->name);
			FREE_NULL_HOSTLIST(hl);
			xfree(host_str);
			return SLURM_ERROR;
		}

		action_msg = xmalloc(sizeof(*action_msg));
		action_msg->action_name = xstrdup(action->name);
		action_msg->file_env_name = xstrdup(entry->env_name);
		action_msg->file_content = xstrdup(json);

		agent_args = xmalloc(sizeof(*agent_args));
		agent_args->msg_type = REQUEST_RUN_POWER_ACTION;
		agent_args->retry = 0;
		agent_args->hostlist = hl;
		agent_args->node_count = hostlist_count(hl);
		agent_args->msg_args = action_msg;
		agent_args->protocol_version = entry->protocol_version ?
						       entry->protocol_version :
						       SLURM_PROTOCOL_VERSION;
		set_agent_arg_r_uid(agent_args, SLURM_AUTH_UID_ANY);

		agent_queue_request(agent_args);
		log_flag(POWER, "Issued slurmd power action %s on nodes %s",
			 action->name, host_str);
	}
	xfree(host_str);
	return SLURM_SUCCESS;
}

void power_action_reboot(bitstr_t *bitmap, char *features)
{
	list_t *action_list = NULL;

	if (!reboot_prog) {
		error("power_save: reboot requested but RebootProgram is not configured");
		return;
	}
	action_list = _build_power_action_list(bitmap, reboot_prog->name, true);
	list_for_each(action_list, _do_power_action_reboot, xstrdup(features));
	FREE_NULL_LIST(action_list);
}

static int _do_power_action_reboot(void *e, void *payload)
{
	action_entry_t *entry = e;
	agent_arg_t *agent_args;
	char *features = payload;
	char *host_str = NULL;
	hostlist_t *hl;
	power_action_t *action = NULL;

	hl = bitmap2hostlist(entry->bitmap);
	if (!hl) {
		error("power_save: failed bitmap2hostlist");
		xfree(features);
		return SLURM_ERROR;
	}
	host_str = hostlist_ranged_string_xmalloc(hl);

	action = entry->action;
	if (!action) {
		action = power_action_find(power_action_list,
					   entry->action_name);
		if (!action) {
			error("Invalid PowerAction '%s', skipping nodes %s",
			      entry->action_name, host_str);
			FREE_NULL_HOSTLIST(hl);
			xfree(host_str);
			xfree(features);
			return SLURM_ERROR;
		}
	}

	if (action->on_slurmctld) {
		log_flag(POWER, "Issuing slurmctld power action reboot '%s' request for nodes %s",
			 action->name,
			 host_str);
		slurmscriptd_run_power_action(action, host_str, features, 0,
					      max_timeout, NULL, NULL);
		FREE_NULL_HOSTLIST(hl);
	} else {
		reboot_msg_t *reboot_msg;

		if ((entry->protocol_version < SLURM_26_05_PROTOCOL_VERSION) &&
		    !entry->is_default) {
			error("power_save: nodes needing power action %s must be running at least Slurm 26.05",
			      action->name);
			FREE_NULL_HOSTLIST(hl);
			xfree(host_str);
			xfree(features);
			return SLURM_ERROR;
		}
		log_flag(POWER, "Issuing slurmd power action reboot '%s' request for nodes %s",
			action->name,
			host_str);
		agent_args = xmalloc(sizeof(*agent_args));
		agent_args->msg_type = REQUEST_REBOOT_NODES;
		agent_args->retry = 0;
		agent_args->hostlist = hl;
		agent_args->node_count = hostlist_count(hl);
		agent_args->protocol_version = entry->protocol_version ?
						       entry->protocol_version :
						       SLURM_PROTOCOL_VERSION;

		reboot_msg = xmalloc(sizeof(*reboot_msg));
		slurm_init_reboot_msg(reboot_msg, true);
		reboot_msg->power_action_name = xstrdup(action->name);
		reboot_msg->features = features;
		agent_args->msg_args = reboot_msg;

		set_agent_arg_r_uid(agent_args, SLURM_AUTH_UID_ANY);
		agent_queue_request(agent_args);
	}
	xfree(host_str);
	return SLURM_SUCCESS;
}

static void _clear_power_config(void)
{
	FREE_NULL_LIST(power_action_list);

	reboot_prog = NULL;
	resume_fail_prog = NULL;
	resume_prog = NULL;
	suspend_prog = NULL;

	suspend_exc_down = false;
	suspend_exc_state_flags = 0;
	FREE_NULL_BITMAP(exc_node_bitmap);
	FREE_NULL_LIST(partial_node_list);
}

static int _set_partition_options(void *x, void *ignored)
{
	part_record_t *part_ptr = (part_record_t *)x;
	node_record_t *node_ptr;

	if (part_ptr->resume_timeout != NO_VAL16)
		max_timeout = MAX(max_timeout, part_ptr->resume_timeout);

	if (part_ptr->suspend_timeout != NO_VAL16)
		max_timeout = MAX(max_timeout, part_ptr->suspend_timeout);

	for (int i = 0;
	     (node_ptr = next_node_bitmap(part_ptr->node_bitmap, &i)); i++) {
		/*
		 * Only update suspend_time from partition if not already set
		 * at node level (from NodeName configuration)
		 */
		if (node_ptr->suspend_time == NO_VAL)
			node_ptr->suspend_time = part_ptr->suspend_time;
		else if (node_ptr->config_ptr->suspend_time == NO_VAL &&
			 part_ptr->suspend_time != NO_VAL)
			node_ptr->suspend_time = MAX(node_ptr->suspend_time,
						     part_ptr->suspend_time);

		if (node_ptr->resume_timeout == NO_VAL16)
			node_ptr->resume_timeout = part_ptr->resume_timeout;
		else if (part_ptr->resume_timeout != NO_VAL16)
			node_ptr->resume_timeout = MAX(
				node_ptr->resume_timeout,
				part_ptr->resume_timeout);

		if (node_ptr->suspend_timeout == NO_VAL16)
			node_ptr->suspend_timeout = part_ptr->suspend_timeout;
		else if (part_ptr->suspend_timeout != NO_VAL16)
			node_ptr->suspend_timeout = MAX(
				node_ptr->suspend_timeout,
				part_ptr->suspend_timeout);
	}

	return 0;
}

/*
 * Parse settings for excluding nodes, partitions and states from being
 * suspended.
 *
 * This creates node bitmaps. Must be done again when node bitmaps change.
 */
extern void power_save_exc_setup(void)
{
	xassert(verify_lock(CONF_LOCK, READ_LOCK));
	xassert(verify_lock(NODE_LOCK, READ_LOCK));
	xassert(verify_lock(PART_LOCK, READ_LOCK));

	FREE_NULL_BITMAP(exc_node_bitmap);

	if (slurm_conf.suspend_exc_nodes &&
	    (_parse_exc_nodes() != SLURM_SUCCESS))
		error("Invalid SuspendExcNodes %s some nodes may be ignored.",
		      slurm_conf.suspend_exc_nodes);

	if (slurm_conf.suspend_exc_parts) {
		char *tmp = NULL, *one_part = NULL, *part_list = NULL;
		part_record_t *part_ptr = NULL;

		part_list = xstrdup(slurm_conf.suspend_exc_parts);
		one_part = strtok_r(part_list, ",", &tmp);
		while (one_part != NULL) {
			part_ptr = find_part_record(one_part);
			if (!part_ptr) {
				error("Invalid SuspendExcPart %s ignored",
				      one_part);
			} else if (exc_node_bitmap) {
				bit_or(exc_node_bitmap,
				       part_ptr->node_bitmap);
			} else {
				exc_node_bitmap =
					bit_copy(part_ptr->node_bitmap);
			}
			one_part = strtok_r(NULL, ",", &tmp);
		}
		xfree(part_list);
	}

	if (slurm_conf.suspend_exc_states)
		_parse_exc_states();

	if (power_save_debug) {
		if (exc_node_bitmap) {
			char *tmp = bitmap2node_name(exc_node_bitmap);
			log_flag(POWER, "excluded nodes %s", tmp);
			xfree(tmp);
		}
		if (partial_node_list) {
			(void) list_for_each(partial_node_list,
					     _list_part_node_lists, NULL);
		}
	}
}

static void power_save_rl_setup(void)
{
	uint32_t max_tokens, refill_period_msec, effective_max_interval;

	/*
	 * Power save either runs nominally close to power_save_interval
	 * or, at worst, at the minimum rate. Either way, we'll want the
	 * larger value for worst-case scenario in sizing bucket.
	 */
	effective_max_interval = MAX(1,
				     MAX(power_save_interval,
					 power_save_min_interval));

	if (resume_rate) {
		/*
		 * If the rate is high and/or the power save interval is large,
		 * the bucket must be larger to accommodate large token
		 * accumulation between executions of _do_power_work().
		 * units are: (tokens) = ((tokens/min) * seconds) /
		 *	                 (seconds / min)
		 */
		if (resume_rate * effective_max_interval < 60)
			max_tokens = 1;
		else
			max_tokens = resume_rate * effective_max_interval / 60;

		/*
		 * Token refill period is independent of bucket size. We will
		 * add one token every period and they will be spent in each
		 * iteration of _do_power_work(). The minimum period is 1ms,
		 * therefore the max number of nodes updated is 60000 per minute
		 */
		refill_period_msec = MAX_NODE_RATE / resume_rate;

		_rl_init(&resume_rl_config,
			 1,
			 max_tokens,
			 refill_period_msec,
			 0);
	}

	if (suspend_rate) {
		if (suspend_rate * effective_max_interval < 60)
			max_tokens = 1;
		else
			max_tokens = suspend_rate * effective_max_interval / 60;

		refill_period_msec = MAX_NODE_RATE / suspend_rate;

		_rl_init(&suspend_rl_config,
			 1,
			 max_tokens,
			 refill_period_msec,
			 0);
	}
}

static int _print_power_action(void *action_ptr, void *unused)
{
	power_action_t *action = (power_action_t *) action_ptr;
	const char *default_prog_str = "";
	if (action == suspend_prog) {
		default_prog_str = "SuspendProgram";
	} else if (action == resume_prog) {
		default_prog_str = "ResumeProgram";
	} else if (action == resume_fail_prog) {
		default_prog_str = "ResumeFailProgram";
	} else if (action == reboot_prog) {
		default_prog_str = "RebootProgram";
	}
	log_flag(POWER, "PowerAction entry: %s (%s) (%s:%s)",
		 action->name,
		 default_prog_str,
		 action->on_slurmctld ? "slurmctld" :
		 			"slurmd",
		 action->program);
	return SLURM_SUCCESS;
}

/*
 * Initialize power_save module parameters.
 * Return 0 on valid configuration to run power saving,
 * otherwise log the problem and return -1
 */
static int _init_power_config(void)
{
	char *tmp_ptr;
	bool suspend_time_set = false;
	bool auto_power_save_disabled = false;

	last_log = 0;
	suspend_rate = slurm_conf.suspend_rate;
	resume_rate = slurm_conf.resume_rate;
	slurmd_timeout = slurm_conf.slurmd_timeout;
	max_timeout = MAX(slurm_conf.suspend_timeout,
			  slurm_conf.resume_timeout);
	_clear_power_config();
	slurm_conf_power_action_list(&power_action_list);
	suspend_prog = power_action_find_or_create(power_action_list,
						   slurm_conf.suspend_program,
						   POWER_ACTION_SUSPEND);
	resume_prog = power_action_find_or_create(power_action_list,
						  slurm_conf.resume_program,
						  POWER_ACTION_RESUME);
	resume_fail_prog =
		power_action_find_or_create(power_action_list,
					    slurm_conf.resume_fail_program,
					    POWER_ACTION_RESUME_FAIL);
	reboot_prog = power_action_find_or_create(power_action_list,
						  slurm_conf.reboot_program,
						  POWER_ACTION_REBOOT);

	if (reboot_prog && !power_action_valid_prog(reboot_prog)) {
		error("Invalid RebootProgram PowerAction %s (%s)",
			reboot_prog->name, reboot_prog->program);
		reboot_prog = NULL;
	}
	if (suspend_prog && !power_action_valid_prog(suspend_prog)) {
		error("Invalid SuspendProgram PowerAction %s (%s)",
			suspend_prog->name, suspend_prog->program);
		suspend_prog = NULL;
	}
	if (resume_prog && !power_action_valid_prog(resume_prog)) {
		error("Invalid ResumeProgram PowerAction %s (%s)",
			resume_prog->name, resume_prog->program);
		resume_prog = NULL;
	}
	if (resume_fail_prog && !power_action_valid_prog(resume_fail_prog)) {
		error("Invalid ResumeFailProgram PowerAction %s (%s)",
			resume_fail_prog->name, resume_fail_prog->program);
		resume_fail_prog = NULL;
	}

	if (slurm_conf.debug_flags & DEBUG_FLAG_POWER) {
		if (list_count(power_action_list) > 0)
			list_for_each(power_action_list, _print_power_action,
				      NULL);
		else
			log_flag(POWER, "No PowerAction entries");
	}

	if (!suspend_prog || !resume_prog)
		auto_power_save_disabled = true;

	idle_on_node_suspend = xstrcasestr(slurm_conf.slurmctld_params,
					   "idle_on_node_suspend");
	if ((tmp_ptr = xstrcasestr(slurm_conf.slurmctld_params,
				   "power_save_interval="))) {
		power_save_interval =
			strtol(tmp_ptr + strlen("power_save_interval="), NULL,
			       10);
	}
	if ((tmp_ptr = xstrcasestr(slurm_conf.slurmctld_params,
				   "power_save_min_interval="))) {
		power_save_min_interval =
			strtol(tmp_ptr + strlen("power_save_min_interval="),
			       NULL, 10);
	}

	power_save_set_timeouts(&suspend_time_set);

	if (slurm_conf.debug_flags & DEBUG_FLAG_POWER)
		power_save_debug = true;
	else
		power_save_debug = false;

	power_save_exc_setup();
	power_save_rl_setup();

	if (((resume_rate || suspend_rate)) &&
	    ((power_save_interval > 60) || (power_save_min_interval > 60))) {
		error("power save module can not work effectively with interval > 60 seconds");
		auto_power_save_disabled = true;
	}
	if ((suspend_rate > MAX_NODE_RATE) || (resume_rate > MAX_NODE_RATE)) {
		error("power: selected suspend/resume rate exceeds maximum: %d/%d max: %d",
		      suspend_rate, resume_rate, MAX_NODE_RATE);
		auto_power_save_disabled = true;
	}

	if ((slurm_conf.suspend_time == INFINITE) && !suspend_time_set) {
		/* not an error */
		debug("power: SuspendTime < 0");
		auto_power_save_disabled = true;
	}
	if (suspend_rate < 0) {
		error("auto power save module disabled, SuspendRate < 0");
		auto_power_save_disabled = true;
	}
	if (resume_rate < 0) {
		error("auto power save module disabled, ResumeRate < 0");
		auto_power_save_disabled = true;
	}
	if (!suspend_prog) {
		debug("power: No valid SuspendProgram");
		auto_power_save_disabled = true;
	}
	if (!resume_prog) {
		debug("power: No valid ResumeProgram");
		auto_power_save_disabled = true;
	}
	if (auto_power_save_disabled) {
		info("power: automatic power save mode disabled");
	}
	return auto_power_save_disabled ? -1 : 0;
}

extern bool power_save_valid_action_default(power_action_type_t type,
					    char *action_name)
{
	power_action_t *action = NULL;
	bool valid = false;
	slurm_mutex_lock(&power_mutex);
	while (!power_save_config) {
		slurm_cond_wait(&power_cond, &power_mutex);
	}
	if (action_name) {
		action = power_action_find(power_action_list, action_name);
	} else {
		switch (type) {
		case POWER_ACTION_RESUME:
			action = resume_prog;
			break;
		case POWER_ACTION_SUSPEND:
			action = suspend_prog;
			break;
		case POWER_ACTION_RESUME_FAIL:
			action = resume_fail_prog;
			break;
		case POWER_ACTION_REBOOT:
			action = reboot_prog;
			break;
		case POWER_ACTION_NONE:
			action = NULL;
			break;
		default:
			error("Invalid power action type %d", type);
			action = NULL;
			break;
		}
	}
	if (!action) {
		valid = false;
	} else if (type == POWER_ACTION_RESUME && !action->on_slurmctld) {
		error("Invalid power up action %s, cannot resume from slurmd",
		      (action_name ? action_name : action->name));
		valid = false;
	} else if (power_action_valid_prog(action)) {
		valid = true;
	}
	slurm_mutex_unlock(&power_mutex);
	return valid;
}

extern void config_power_mgr(void)
{
	slurm_mutex_lock(&power_mutex);
	if (_init_power_config()) {
		if (power_save_enabled) {
			/* transition from enabled to disabled */
			info("automatic power_save mode has been disabled due to configuration changes");
		}
		power_save_enabled = false;
	} else {
		power_save_enabled = true;
	}
	power_save_config = true;
	slurm_cond_signal(&power_cond);
	slurm_mutex_unlock(&power_mutex);
}

extern void config_power_mgr_fini(void)
{
	slurm_mutex_lock(&power_mutex);
	power_save_config = false;
	_clear_power_config();
	slurm_mutex_unlock(&power_mutex);
}

extern void power_save_init(void)
{
	slurm_mutex_lock(&power_mutex);
	if (power_save_started) {
		slurm_mutex_unlock(&power_mutex);
		return;
	}
	power_save_started = true;

	slurm_thread_create("powersave", &power_thread, _power_save_thread,
			    NULL);
	slurm_mutex_unlock(&power_mutex);
}

/* Report if node power saving is enabled */
extern bool power_save_test(void)
{
	bool rc;

	slurm_mutex_lock(&power_mutex);
	while (!power_save_config) {
		slurm_cond_wait(&power_cond, &power_mutex);
	}
	rc = power_save_enabled;
	slurm_mutex_unlock(&power_mutex);

	return rc;
}

/* Free module's allocated memory */
extern void power_save_fini(void)
{
	slurm_cond_signal(&power_cond);
	slurm_thread_join(power_thread);

	slurm_mutex_lock(&power_mutex);
	if (power_save_started) {     /* Already running */
		power_save_started = false;
	}
	FREE_NULL_LIST(resume_job_list);
	slurm_mutex_unlock(&power_mutex);
}

static int _build_resume_job_list(void *object, void *arg)
{
	job_record_t *job_ptr = (job_record_t *)object;

	if (IS_JOB_CONFIGURING(job_ptr) &&
	    bit_overlap_any(job_ptr->node_bitmap,
			    power_down_node_bitmap)) {
		uint32_t *tmp = xmalloc(sizeof(uint32_t));
		*tmp = job_ptr->job_id;
		list_append(resume_job_list, tmp);
	}

	return SLURM_SUCCESS;
}

static void *_power_save_thread(void *arg)
{
	struct timespec ts = {0, 0};
	/* Locks: Write jobs and nodes */
	slurmctld_lock_t node_write_lock = {
		NO_LOCK, WRITE_LOCK, WRITE_LOCK, NO_LOCK, NO_LOCK };
	time_t now, last_power_scan = 0;
	bool auto_power_save_enabled = false;

	/*
	 * Build up resume_job_list list in case shut down before resuming
	 * jobs/nodes without having to state save the list.
	 */
	if (!resume_job_list) {
		resume_job_list = list_create(xfree_ptr);

		lock_slurmctld(node_write_lock);
		list_for_each(job_list, _build_resume_job_list, NULL);
		unlock_slurmctld(node_write_lock);
	}

	while (!slurmctld_config.shutdown_time) {
		slurm_mutex_lock(&power_mutex);
		clock_gettime(CLOCK_REALTIME, &ts);
		ts.tv_sec += 1;
		slurm_cond_timedwait(&power_cond, &power_mutex, &ts);
		auto_power_save_enabled = power_save_enabled;
		slurm_mutex_unlock(&power_mutex);

		if (slurmctld_config.shutdown_time)
			break;

		now = time(NULL);
		if ((now > (last_power_scan + power_save_min_interval)) &&
		    ((last_node_update > last_power_scan) ||
		     (now > (last_power_scan + power_save_interval)))) {
			lock_slurmctld(node_write_lock);
			_do_power_work(now, auto_power_save_enabled);
			unlock_slurmctld(node_write_lock);
			last_power_scan = now;
		}
	}

	slurm_mutex_lock(&power_mutex);
	power_save_started = false;
	slurm_cond_signal(&power_cond);
	slurm_mutex_unlock(&power_mutex);
	return NULL;
}

extern void power_save_set_timeouts(bool *suspend_time_set)
{
	node_record_t *node_ptr;

	xassert(verify_lock(CONF_LOCK, READ_LOCK));
	xassert(verify_lock(NODE_LOCK, WRITE_LOCK));
	xassert(verify_lock(PART_LOCK, READ_LOCK));

	/* Reset timeouts so new values can be calculated. */
	for (int i = 0; (node_ptr = next_node(&i)); i++) {
		node_ptr->suspend_time = node_ptr->config_ptr->suspend_time;
		node_ptr->suspend_timeout = NO_VAL16;
		node_ptr->resume_timeout = NO_VAL16;
	}

	/* Figure out per-partition options and push to node level. */
	list_for_each(part_list, _set_partition_options, NULL);

	/* Apply global options to node level if not set at partition or node level. */
	for (int i = 0; (node_ptr = next_node(&i)); i++) {
		node_ptr->suspend_time =
			((node_ptr->suspend_time == NO_VAL) ?
				slurm_conf.suspend_time :
				node_ptr->suspend_time);
		if (suspend_time_set &&
		    (node_ptr->suspend_time != INFINITE) &&
		    (node_ptr->suspend_time != NO_VAL))
			*suspend_time_set = true;
		node_ptr->suspend_timeout =
			((node_ptr->suspend_timeout == NO_VAL16) ?
				slurm_conf.suspend_timeout :
				node_ptr->suspend_timeout);
		node_ptr->resume_timeout =
			((node_ptr->resume_timeout == NO_VAL16) ?
				slurm_conf.resume_timeout :
				node_ptr->resume_timeout);
		debug5("Power save settings for '%s': SuspendTime=%u,SuspendTimeout=%u,ResumeTimeout=%u",
		       node_ptr->name, node_ptr->suspend_time,
		       node_ptr->suspend_timeout, node_ptr->resume_timeout);
	}
}

static uint64_t _timespec_to_msec(struct timespec *tv)
{
	xassert(tv);
	return (tv->tv_sec * 1000) + (tv->tv_nsec / 1000000);
}

/* Initializes and starts the rate limit operation */
static void _rl_init(rl_config_t *config,
		     uint32_t refill_count,
		     uint32_t max_tokens,
		     uint32_t refill_period_msec,
		     uint32_t start_tokens)
{
	xassert(config);
	struct timespec now = { 0 };
	xassert(!clock_gettime(CLOCK_MONOTONIC, &now));
	config->inited = true;
	config->last_update = _timespec_to_msec(&now);
	config->max_tokens = max_tokens;
	config->refill_count = refill_count;
	config->refill_period_msec = refill_period_msec;
	config->tokens = start_tokens;
}

/* Updates the token count and returns the new count of available tokens */
static uint32_t _rl_get_tokens(rl_config_t *config)
{
	struct timespec now = { 0 };

	xassert(config);
	xassert(config->inited);

	clock_gettime(CLOCK_MONOTONIC, &now);

	uint64_t now_msec = _timespec_to_msec(&now);
	uint64_t now_periods = now_msec / config->refill_period_msec;
	uint64_t delta = now_periods - config->last_update;
	config->last_update = now_periods;

	if (delta) {
		config->tokens += (delta * config->refill_count);
		config->tokens = MIN(config->tokens, config->max_tokens);
	}

	return config->tokens;
}

/*
 * Should not be called when there are no tokens to spend. Call
 * _rl_get_tokens to check first.
 */
static void _rl_spend_token(rl_config_t *config)
{
	if (!config->inited)
		return;

	if (config->tokens)
		config->tokens--;
	else
		error("Token spent when unavailable. Power save unlikely to respect resume/suspend rate.");
}
