Policy-Gradient Training Step Source Code
Browse the implementation files for Policy-Gradient Training Step.tool.ts typescript
import { parseFiniteNumberList } from '../../parse/numeric-dataset/tool';
import { parseFiniteNumber } from '../../parse/number/tool';
import { parseInteger } from '../../parse/integer/tool';
const softmax = (logits: number[]): number[] => {
const maximum = logits.reduce(
(result, value) => Math.max(result, value),
-Infinity
);
const exponentials = logits.map((value) =>
Math.exp(value - maximum)
);
const total = exponentials.reduce(
(sum, value) => sum + value,
0
);
return exponentials.map((value) => value / total);
};
export const calculatePolicyGradientStep = (
logitsInput: unknown,
actionInput: unknown,
returnInput: unknown,
baselineInput: unknown,
learningRateInput: unknown
): string => {
const logits = parseFiniteNumberList(logitsInput);
const action = parseInteger(actionInput);
const episodeReturn = parseFiniteNumber(returnInput);
const baseline = parseFiniteNumber(baselineInput);
const learningRate = parseFiniteNumber(learningRateInput);
if (
!logits ||
logits.length < 2 ||
logits.length > 100_000 ||
action === null ||
action < 0 ||
action >= logits.length ||
episodeReturn === null ||
baseline === null ||
learningRate === null ||
learningRate <= 0
)
return '';
const probabilities = softmax(logits);
const advantage = episodeReturn - baseline;
if (!Number.isFinite(advantage)) return '';
const gradient = probabilities.map(
(probability, index) =>
((index === action ? 1 : 0) - probability) * advantage
);
const nextLogits = logits.map(
(value, index) => value + learningRate * gradient[index]
);
if (
gradient.some((value) => !Number.isFinite(value)) ||
nextLogits.some((value) => !Number.isFinite(value))
)
return '';
return JSON.stringify(
{
action,
advantage: Number(advantage.toPrecision(10)),
gradient: gradient.map((value) =>
Number(value.toPrecision(10))
),
logits: nextLogits.map((value) =>
Number(value.toPrecision(10))
),
probabilities: softmax(nextLogits).map((value) =>
Number(value.toPrecision(10))
)
},
null,
2
);
};
export const Tool: TypeTool = {
name: 'Policy-Gradient Training Step',
slug: 'reinforcement-learning/policy-gradient-step',
intro:
'Apply one categorical REINFORCE policy-gradient update from an action, return, and baseline.',
tool: calculatePolicyGradientStep,
i: [
{
example: '0, 0, 0',
label: 'Policy Logits',
placeholder: 'One logit per action',
render: 'textarea',
type: 'text'
},
{
example: '1',
label: 'Selected Action',
placeholder: 'Zero-based action index',
type: 'number'
},
{
example: '2',
label: 'Episode Return',
placeholder: 'Observed return',
type: 'number'
},
{
example: '0.5',
label: 'Baseline',
placeholder: 'Value baseline',
type: 'number'
},
{
example: '0.1',
label: 'Learning Rate',
placeholder: 'Positive step size',
type: 'number'
}
],
o: [
{
example: '{ "probabilities": [0.31, 0.38, 0.31] }',
label: 'Policy Update',
placeholder: 'Updated policy will appear here...',
render: 'textarea',
type: 'text'
}
],
cite: [
{
title: 'Reinforcement learning - Wikipedia',
url: 'https://en.wikipedia.org/wiki/Reinforcement_learning'
}
]
};
export default Tool;
Description
Browse the implementation files for Policy-Gradient Training Step.