Skip to content

Main content

Reinforcement-Learning Policy Evaluator Source Code

Browse the implementation files for Reinforcement-Learning Policy Evaluator.
tool.ts typescript
import { parseFiniteNumberList } from '../../parse/numeric-dataset/tool';
import { parseFiniteNumber } from '../../parse/number/tool';
import { parseInteger } from '../../parse/integer/tool';

export const evaluatePolicy = (
	rewardsInput: unknown,
	transitionInput: unknown,
	discountInput: unknown,
	iterationsInput: unknown
): string => {
	const rewards = parseFiniteNumberList(rewardsInput);
	const discount = parseFiniteNumber(discountInput);
	const iterations = parseInteger(iterationsInput);
	if (
		!rewards ||
		discount === null ||
		discount < 0 ||
		discount >= 1 ||
		iterations === null ||
		iterations < 1 ||
		iterations > 100_000 ||
		typeof transitionInput !== 'string'
	)
		return '';
	let transitions: unknown;
	try {
		transitions = JSON.parse(transitionInput);
	} catch {
		return '';
	}
	if (
		!Array.isArray(transitions) ||
		transitions.length !== rewards.length ||
		transitions.some(
			(row) =>
				!Array.isArray(row) ||
				row.length !== rewards.length ||
				row.some(
					(value) =>
						typeof value !== 'number' ||
						!Number.isFinite(value) ||
						value < 0
				) ||
				Math.abs(
					row.reduce((sum, value) => sum + value, 0) - 1
				) > 1e-9
		)
	)
		return '';
	const typedTransitions = transitions as number[][];
	let values = Array(rewards.length).fill(0) as number[];
	for (
		let iteration = 0;
		iteration < iterations;
		iteration += 1
	)
		values = rewards.map(
			(reward, state) =>
				reward +
				discount *
					typedTransitions[state].reduce(
						(sum, probability, nextState) =>
							sum + probability * values[nextState],
						0
					)
		);
	if (values.some((value) => !Number.isFinite(value)))
		return '';
	return JSON.stringify(
		{
			discount,
			iterations,
			values: values.map((value) =>
				Number(value.toPrecision(10))
			)
		},
		null,
		2
	);
};

export const Tool: TypeTool = {
	name: 'Reinforcement-Learning Policy Evaluator',
	slug: 'reinforcement-learning/policy-evaluation',
	intro:
		'Evaluate a fixed finite-state policy by iterating its reward vector and policy-induced transition matrix.',
	tool: evaluatePolicy,
	i: [
		{
			example: '1, 0',
			label: 'State Rewards',
			placeholder: 'One reward per state',
			render: 'textarea',
			type: 'text'
		},
		{
			example: '[[0.5,0.5],[0.2,0.8]]',
			label: 'Transition Matrix',
			placeholder: 'JSON row-stochastic matrix',
			render: 'textarea',
			type: 'text'
		},
		{
			example: '0.9',
			label: 'Discount Factor',
			placeholder: 'Value from 0 to less than 1',
			type: 'number'
		},
		{
			example: '100',
			label: 'Iterations',
			placeholder: 'Evaluation iterations',
			type: 'number'
		}
	],
	o: [
		{
			example: '{ "values": [4.19, 2.57] }',
			label: 'State Values',
			placeholder: 'Policy values will appear here...',
			render: 'textarea',
			type: 'text'
		}
	],
	cite: [
		{
			title: 'Markov decision process - Wikipedia',
			url: 'https://en.wikipedia.org/wiki/Markov_decision_process'
		}
	]
};
export default Tool;

Description

Browse the implementation files for Reinforcement-Learning Policy Evaluator.

References

  1. Tool.io on GitHub
  2. Tool.io TypeScript tools library

Similar or alternative tools