Skip to content

Main content

Policy-Gradient Training Step Source Code

Browse the implementation files for Policy-Gradient Training Step.
tool.ts typescript
import { parseFiniteNumberList } from '../../parse/numeric-dataset/tool';
import { parseFiniteNumber } from '../../parse/number/tool';
import { parseInteger } from '../../parse/integer/tool';

const softmax = (logits: number[]): number[] => {
	const maximum = logits.reduce(
		(result, value) => Math.max(result, value),
		-Infinity
	);
	const exponentials = logits.map((value) =>
		Math.exp(value - maximum)
	);
	const total = exponentials.reduce(
		(sum, value) => sum + value,
		0
	);
	return exponentials.map((value) => value / total);
};

export const calculatePolicyGradientStep = (
	logitsInput: unknown,
	actionInput: unknown,
	returnInput: unknown,
	baselineInput: unknown,
	learningRateInput: unknown
): string => {
	const logits = parseFiniteNumberList(logitsInput);
	const action = parseInteger(actionInput);
	const episodeReturn = parseFiniteNumber(returnInput);
	const baseline = parseFiniteNumber(baselineInput);
	const learningRate = parseFiniteNumber(learningRateInput);
	if (
		!logits ||
		logits.length < 2 ||
		logits.length > 100_000 ||
		action === null ||
		action < 0 ||
		action >= logits.length ||
		episodeReturn === null ||
		baseline === null ||
		learningRate === null ||
		learningRate <= 0
	)
		return '';
	const probabilities = softmax(logits);
	const advantage = episodeReturn - baseline;
	if (!Number.isFinite(advantage)) return '';
	const gradient = probabilities.map(
		(probability, index) =>
			((index === action ? 1 : 0) - probability) * advantage
	);
	const nextLogits = logits.map(
		(value, index) => value + learningRate * gradient[index]
	);
	if (
		gradient.some((value) => !Number.isFinite(value)) ||
		nextLogits.some((value) => !Number.isFinite(value))
	)
		return '';
	return JSON.stringify(
		{
			action,
			advantage: Number(advantage.toPrecision(10)),
			gradient: gradient.map((value) =>
				Number(value.toPrecision(10))
			),
			logits: nextLogits.map((value) =>
				Number(value.toPrecision(10))
			),
			probabilities: softmax(nextLogits).map((value) =>
				Number(value.toPrecision(10))
			)
		},
		null,
		2
	);
};

export const Tool: TypeTool = {
	name: 'Policy-Gradient Training Step',
	slug: 'reinforcement-learning/policy-gradient-step',
	intro:
		'Apply one categorical REINFORCE policy-gradient update from an action, return, and baseline.',
	tool: calculatePolicyGradientStep,
	i: [
		{
			example: '0, 0, 0',
			label: 'Policy Logits',
			placeholder: 'One logit per action',
			render: 'textarea',
			type: 'text'
		},
		{
			example: '1',
			label: 'Selected Action',
			placeholder: 'Zero-based action index',
			type: 'number'
		},
		{
			example: '2',
			label: 'Episode Return',
			placeholder: 'Observed return',
			type: 'number'
		},
		{
			example: '0.5',
			label: 'Baseline',
			placeholder: 'Value baseline',
			type: 'number'
		},
		{
			example: '0.1',
			label: 'Learning Rate',
			placeholder: 'Positive step size',
			type: 'number'
		}
	],
	o: [
		{
			example: '{ "probabilities": [0.31, 0.38, 0.31] }',
			label: 'Policy Update',
			placeholder: 'Updated policy will appear here...',
			render: 'textarea',
			type: 'text'
		}
	],
	cite: [
		{
			title: 'Reinforcement learning - Wikipedia',
			url: 'https://en.wikipedia.org/wiki/Reinforcement_learning'
		}
	]
};
export default Tool;

Description

Browse the implementation files for Policy-Gradient Training Step.

References

  1. Tool.io on GitHub
  2. Tool.io TypeScript tools library

Similar or alternative tools