Bookmark this: tool.io/optimization/adadelta-step/source-code
AdaDelta Optimizer Step Calculator Source Code
Browse the implementation files for AdaDelta Optimizer Step Calculator.tool.ts typescript
// Tool: AdaDelta Optimizer Step Calculator
const parseFiniteNumber = (input: unknown): number | null => {
if (
(typeof input !== 'string' && typeof input !== 'number') ||
(typeof input === 'string' && input.trim() === '')
) {
return null;
}
const value = Number(input);
return Number.isFinite(value) ? value : null;
};
const round = (value: number): number =>
Number(value.toPrecision(12));
export const calculateAdaDeltaStep = (
parameterInput: unknown,
gradientInput: unknown,
averageSquaredGradientInput: unknown,
averageSquaredUpdateInput: unknown,
decayRateInput: unknown,
epsilonInput: unknown
): string => {
const parameter = parseFiniteNumber(parameterInput);
const gradient = parseFiniteNumber(gradientInput);
const averageSquaredGradient = parseFiniteNumber(
averageSquaredGradientInput
);
const averageSquaredUpdate = parseFiniteNumber(
averageSquaredUpdateInput
);
const decayRate = parseFiniteNumber(decayRateInput);
const epsilon = parseFiniteNumber(epsilonInput);
if (
parameter === null ||
gradient === null ||
averageSquaredGradient === null ||
averageSquaredUpdate === null ||
decayRate === null ||
epsilon === null ||
averageSquaredGradient < 0 ||
averageSquaredUpdate < 0 ||
decayRate < 0 ||
decayRate >= 1 ||
epsilon <= 0
) {
return '';
}
const nextAverageSquaredGradient =
decayRate * averageSquaredGradient +
(1 - decayRate) * gradient ** 2;
const update =
-Math.sqrt(
(averageSquaredUpdate + epsilon) /
(nextAverageSquaredGradient + epsilon)
) * gradient;
const nextAverageSquaredUpdate =
decayRate * averageSquaredUpdate +
(1 - decayRate) * update ** 2;
const nextParameter = parameter + update;
if (
![
nextAverageSquaredGradient,
nextAverageSquaredUpdate,
nextParameter,
update
].every(Number.isFinite)
) {
return '';
}
return JSON.stringify(
{
averageSquaredGradient: round(
nextAverageSquaredGradient
),
averageSquaredUpdate: round(nextAverageSquaredUpdate),
parameter: round(nextParameter),
update: round(update)
},
null,
2
);
};
export const Tool: TypeTool = {
name: 'AdaDelta Optimizer Step Calculator',
slug: 'optimization/adadelta-step',
intro:
'Calculate one scalar AdaDelta parameter update and its accumulated state.',
tool: calculateAdaDeltaStep,
i: [
{
example: '1',
label: 'Parameter',
placeholder: 'Current parameter value',
type: 'number',
value: 1
},
{
example: '0.1',
label: 'Gradient',
placeholder: 'Current gradient',
type: 'number',
value: 0.1
},
{
example: '0',
label: 'Average Squared Gradient',
placeholder: 'Previous accumulated gradient state',
type: 'number',
value: 0
},
{
example: '0',
label: 'Average Squared Update',
placeholder: 'Previous accumulated update state',
type: 'number',
value: 0
},
{
example: '0.95',
label: 'Decay Rate',
placeholder: 'Rho from 0 up to 1',
type: 'number',
value: 0.95
},
{
example: '0.000001',
label: 'Epsilon',
placeholder: 'Positive stability constant',
type: 'number',
value: 0.000001
}
],
o: [
{
example: '{ "parameter": 0.9955, "update": -0.0045 }',
label: 'AdaDelta Step',
placeholder: 'Updated optimizer state will appear here...',
render: 'textarea',
type: 'text'
}
],
cite: [
{
title: 'ADADELTA: An Adaptive Learning Rate Method',
url: 'https://arxiv.org/abs/1212.5701'
}
]
};
export default Tool;
Description
Browse the implementation files for AdaDelta Optimizer Step Calculator.