Bookmark this: tool.io/optimization/adam-step/source-code
Adam Optimizer Step Calculator Source Code
Browse the implementation files for Adam Optimizer Step Calculator.tool.ts typescript
// Tool: Adam Optimizer Step Calculator
const parseFiniteNumber = (input: unknown): number | null => {
if (
(typeof input !== 'string' && typeof input !== 'number') ||
(typeof input === 'string' && input.trim() === '')
) {
return null;
}
const value = Number(input);
return Number.isFinite(value) ? value : null;
};
const parsePositiveInteger = (input: unknown): number | null => {
const value = parseFiniteNumber(input);
return value !== null && Number.isSafeInteger(value) && value >= 1
? value
: null;
};
const round = (value: number): number =>
Number(value.toPrecision(12));
export const calculateAdamStep = (
parameterInput: unknown,
gradientInput: unknown,
firstMomentInput: unknown,
secondMomentInput: unknown,
stepInput: unknown,
learningRateInput: unknown,
betaOneInput: unknown,
betaTwoInput: unknown,
epsilonInput: unknown
): string => {
const parameter = parseFiniteNumber(parameterInput);
const gradient = parseFiniteNumber(gradientInput);
const firstMoment = parseFiniteNumber(firstMomentInput);
const secondMoment = parseFiniteNumber(secondMomentInput);
const step = parsePositiveInteger(stepInput);
const learningRate = parseFiniteNumber(learningRateInput);
const betaOne = parseFiniteNumber(betaOneInput);
const betaTwo = parseFiniteNumber(betaTwoInput);
const epsilon = parseFiniteNumber(epsilonInput);
if (
parameter === null ||
gradient === null ||
firstMoment === null ||
secondMoment === null ||
step === null ||
learningRate === null ||
betaOne === null ||
betaTwo === null ||
epsilon === null ||
secondMoment < 0 ||
learningRate <= 0 ||
betaOne < 0 ||
betaOne >= 1 ||
betaTwo < 0 ||
betaTwo >= 1 ||
epsilon <= 0
) {
return '';
}
const nextFirstMoment =
betaOne * firstMoment + (1 - betaOne) * gradient;
const nextSecondMoment =
betaTwo * secondMoment +
(1 - betaTwo) * gradient ** 2;
const correctedFirstMoment =
nextFirstMoment / (1 - betaOne ** step);
const correctedSecondMoment =
nextSecondMoment / (1 - betaTwo ** step);
const update =
(learningRate * correctedFirstMoment) /
(Math.sqrt(correctedSecondMoment) + epsilon);
const nextParameter = parameter - update;
if (
![
correctedFirstMoment,
correctedSecondMoment,
nextFirstMoment,
nextParameter,
nextSecondMoment,
update
].every(Number.isFinite)
) {
return '';
}
return JSON.stringify(
{
correctedFirstMoment: round(correctedFirstMoment),
correctedSecondMoment: round(correctedSecondMoment),
firstMoment: round(nextFirstMoment),
parameter: round(nextParameter),
secondMoment: round(nextSecondMoment),
step,
update: round(update)
},
null,
2
);
};
export const Tool: TypeTool = {
name: 'Adam Optimizer Step Calculator',
slug: 'optimization/adam-step',
intro:
'Calculate one scalar Adam parameter update with bias-corrected moments.',
tool: calculateAdamStep,
i: [
{
example: '1',
label: 'Parameter',
placeholder: 'Current parameter value',
type: 'number',
value: 1
},
{
example: '0.1',
label: 'Gradient',
placeholder: 'Current gradient',
type: 'number',
value: 0.1
},
{
example: '0',
label: 'First Moment',
placeholder: 'Previous first moment',
type: 'number',
value: 0
},
{
example: '0',
label: 'Second Moment',
placeholder: 'Previous non-negative second moment',
type: 'number',
value: 0
},
{
example: '1',
label: 'Step',
placeholder: 'Positive optimization step',
type: 'number',
value: 1
},
{
example: '0.001',
label: 'Learning Rate',
placeholder: 'Positive learning rate',
type: 'number',
value: 0.001
},
{
example: '0.9',
label: 'Beta 1',
placeholder: 'First-moment decay from 0 up to 1',
type: 'number',
value: 0.9
},
{
example: '0.999',
label: 'Beta 2',
placeholder: 'Second-moment decay from 0 up to 1',
type: 'number',
value: 0.999
},
{
example: '0.00000001',
label: 'Epsilon',
placeholder: 'Positive stability constant',
type: 'number',
value: 0.00000001
}
],
o: [
{
example: '{ "parameter": 0.999, "update": 0.001 }',
label: 'Adam Step',
placeholder: 'Updated optimizer state will appear here...',
render: 'textarea',
type: 'text'
}
],
cite: [
{
title: 'Adam: A Method for Stochastic Optimization',
url: 'https://arxiv.org/abs/1412.6980'
}
]
};
export default Tool;
Description
Browse the implementation files for Adam Optimizer Step Calculator.