1 | "use strict";
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 |
|
13 |
|
14 |
|
15 |
|
16 |
|
17 |
|
18 | var _a;
|
19 | Object.defineProperty(exports, "__esModule", { value: true });
|
20 | exports.setup = exports.OutlierDetectionLoadBalancer = exports.OutlierDetectionLoadBalancingConfig = void 0;
|
21 | const connectivity_state_1 = require("./connectivity-state");
|
22 | const constants_1 = require("./constants");
|
23 | const duration_1 = require("./duration");
|
24 | const experimental_1 = require("./experimental");
|
25 | const load_balancer_1 = require("./load-balancer");
|
26 | const load_balancer_child_handler_1 = require("./load-balancer-child-handler");
|
27 | const picker_1 = require("./picker");
|
28 | const subchannel_address_1 = require("./subchannel-address");
|
29 | const subchannel_interface_1 = require("./subchannel-interface");
|
30 | const logging = require("./logging");
|
31 | const TRACER_NAME = 'outlier_detection';
|
32 | function trace(text) {
|
33 | logging.trace(constants_1.LogVerbosity.DEBUG, TRACER_NAME, text);
|
34 | }
|
35 | const TYPE_NAME = 'outlier_detection';
|
36 | const OUTLIER_DETECTION_ENABLED = ((_a = process.env.GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION) !== null && _a !== void 0 ? _a : 'true') === 'true';
|
37 | const defaultSuccessRateEjectionConfig = {
|
38 | stdev_factor: 1900,
|
39 | enforcement_percentage: 100,
|
40 | minimum_hosts: 5,
|
41 | request_volume: 100,
|
42 | };
|
43 | const defaultFailurePercentageEjectionConfig = {
|
44 | threshold: 85,
|
45 | enforcement_percentage: 100,
|
46 | minimum_hosts: 5,
|
47 | request_volume: 50,
|
48 | };
|
49 | function validateFieldType(obj, fieldName, expectedType, objectName) {
|
50 | if (fieldName in obj && typeof obj[fieldName] !== expectedType) {
|
51 | const fullFieldName = objectName ? `${objectName}.${fieldName}` : fieldName;
|
52 | throw new Error(`outlier detection config ${fullFieldName} parse error: expected ${expectedType}, got ${typeof obj[fieldName]}`);
|
53 | }
|
54 | }
|
55 | function validatePositiveDuration(obj, fieldName, objectName) {
|
56 | const fullFieldName = objectName ? `${objectName}.${fieldName}` : fieldName;
|
57 | if (fieldName in obj) {
|
58 | if (!(0, duration_1.isDuration)(obj[fieldName])) {
|
59 | throw new Error(`outlier detection config ${fullFieldName} parse error: expected Duration, got ${typeof obj[fieldName]}`);
|
60 | }
|
61 | if (!(obj[fieldName].seconds >= 0 &&
|
62 | obj[fieldName].seconds <= 315576000000 &&
|
63 | obj[fieldName].nanos >= 0 &&
|
64 | obj[fieldName].nanos <= 999999999)) {
|
65 | throw new Error(`outlier detection config ${fullFieldName} parse error: values out of range for non-negative Duaration`);
|
66 | }
|
67 | }
|
68 | }
|
69 | function validatePercentage(obj, fieldName, objectName) {
|
70 | const fullFieldName = objectName ? `${objectName}.${fieldName}` : fieldName;
|
71 | validateFieldType(obj, fieldName, 'number', objectName);
|
72 | if (fieldName in obj && !(obj[fieldName] >= 0 && obj[fieldName] <= 100)) {
|
73 | throw new Error(`outlier detection config ${fullFieldName} parse error: value out of range for percentage (0-100)`);
|
74 | }
|
75 | }
|
76 | class OutlierDetectionLoadBalancingConfig {
|
77 | constructor(intervalMs, baseEjectionTimeMs, maxEjectionTimeMs, maxEjectionPercent, successRateEjection, failurePercentageEjection, childPolicy) {
|
78 | this.childPolicy = childPolicy;
|
79 | if (childPolicy.length > 0 &&
|
80 | childPolicy[0].getLoadBalancerName() === 'pick_first') {
|
81 | throw new Error('outlier_detection LB policy cannot have a pick_first child policy');
|
82 | }
|
83 | this.intervalMs = intervalMs !== null && intervalMs !== void 0 ? intervalMs : 10000;
|
84 | this.baseEjectionTimeMs = baseEjectionTimeMs !== null && baseEjectionTimeMs !== void 0 ? baseEjectionTimeMs : 30000;
|
85 | this.maxEjectionTimeMs = maxEjectionTimeMs !== null && maxEjectionTimeMs !== void 0 ? maxEjectionTimeMs : 300000;
|
86 | this.maxEjectionPercent = maxEjectionPercent !== null && maxEjectionPercent !== void 0 ? maxEjectionPercent : 10;
|
87 | this.successRateEjection = successRateEjection
|
88 | ? Object.assign(Object.assign({}, defaultSuccessRateEjectionConfig), successRateEjection) : null;
|
89 | this.failurePercentageEjection = failurePercentageEjection
|
90 | ? Object.assign(Object.assign({}, defaultFailurePercentageEjectionConfig), failurePercentageEjection) : null;
|
91 | }
|
92 | getLoadBalancerName() {
|
93 | return TYPE_NAME;
|
94 | }
|
95 | toJsonObject() {
|
96 | return {
|
97 | interval: (0, duration_1.msToDuration)(this.intervalMs),
|
98 | base_ejection_time: (0, duration_1.msToDuration)(this.baseEjectionTimeMs),
|
99 | max_ejection_time: (0, duration_1.msToDuration)(this.maxEjectionTimeMs),
|
100 | max_ejection_percent: this.maxEjectionPercent,
|
101 | success_rate_ejection: this.successRateEjection,
|
102 | failure_percentage_ejection: this.failurePercentageEjection,
|
103 | child_policy: this.childPolicy.map(policy => policy.toJsonObject()),
|
104 | };
|
105 | }
|
106 | getIntervalMs() {
|
107 | return this.intervalMs;
|
108 | }
|
109 | getBaseEjectionTimeMs() {
|
110 | return this.baseEjectionTimeMs;
|
111 | }
|
112 | getMaxEjectionTimeMs() {
|
113 | return this.maxEjectionTimeMs;
|
114 | }
|
115 | getMaxEjectionPercent() {
|
116 | return this.maxEjectionPercent;
|
117 | }
|
118 | getSuccessRateEjectionConfig() {
|
119 | return this.successRateEjection;
|
120 | }
|
121 | getFailurePercentageEjectionConfig() {
|
122 | return this.failurePercentageEjection;
|
123 | }
|
124 | getChildPolicy() {
|
125 | return this.childPolicy;
|
126 | }
|
127 | copyWithChildPolicy(childPolicy) {
|
128 | return new OutlierDetectionLoadBalancingConfig(this.intervalMs, this.baseEjectionTimeMs, this.maxEjectionTimeMs, this.maxEjectionPercent, this.successRateEjection, this.failurePercentageEjection, childPolicy);
|
129 | }
|
130 | static createFromJson(obj) {
|
131 | var _a;
|
132 | validatePositiveDuration(obj, 'interval');
|
133 | validatePositiveDuration(obj, 'base_ejection_time');
|
134 | validatePositiveDuration(obj, 'max_ejection_time');
|
135 | validatePercentage(obj, 'max_ejection_percent');
|
136 | if ('success_rate_ejection' in obj) {
|
137 | if (typeof obj.success_rate_ejection !== 'object') {
|
138 | throw new Error('outlier detection config success_rate_ejection must be an object');
|
139 | }
|
140 | validateFieldType(obj.success_rate_ejection, 'stdev_factor', 'number', 'success_rate_ejection');
|
141 | validatePercentage(obj.success_rate_ejection, 'enforcement_percentage', 'success_rate_ejection');
|
142 | validateFieldType(obj.success_rate_ejection, 'minimum_hosts', 'number', 'success_rate_ejection');
|
143 | validateFieldType(obj.success_rate_ejection, 'request_volume', 'number', 'success_rate_ejection');
|
144 | }
|
145 | if ('failure_percentage_ejection' in obj) {
|
146 | if (typeof obj.failure_percentage_ejection !== 'object') {
|
147 | throw new Error('outlier detection config failure_percentage_ejection must be an object');
|
148 | }
|
149 | validatePercentage(obj.failure_percentage_ejection, 'threshold', 'failure_percentage_ejection');
|
150 | validatePercentage(obj.failure_percentage_ejection, 'enforcement_percentage', 'failure_percentage_ejection');
|
151 | validateFieldType(obj.failure_percentage_ejection, 'minimum_hosts', 'number', 'failure_percentage_ejection');
|
152 | validateFieldType(obj.failure_percentage_ejection, 'request_volume', 'number', 'failure_percentage_ejection');
|
153 | }
|
154 | return new OutlierDetectionLoadBalancingConfig(obj.interval ? (0, duration_1.durationToMs)(obj.interval) : null, obj.base_ejection_time ? (0, duration_1.durationToMs)(obj.base_ejection_time) : null, obj.max_ejection_time ? (0, duration_1.durationToMs)(obj.max_ejection_time) : null, (_a = obj.max_ejection_percent) !== null && _a !== void 0 ? _a : null, obj.success_rate_ejection, obj.failure_percentage_ejection, obj.child_policy.map(load_balancer_1.validateLoadBalancingConfig));
|
155 | }
|
156 | }
|
157 | exports.OutlierDetectionLoadBalancingConfig = OutlierDetectionLoadBalancingConfig;
|
158 | class OutlierDetectionSubchannelWrapper extends subchannel_interface_1.BaseSubchannelWrapper {
|
159 | constructor(childSubchannel, mapEntry) {
|
160 | super(childSubchannel);
|
161 | this.mapEntry = mapEntry;
|
162 | this.stateListeners = [];
|
163 | this.ejected = false;
|
164 | this.refCount = 0;
|
165 | this.childSubchannelState = childSubchannel.getConnectivityState();
|
166 | childSubchannel.addConnectivityStateListener((subchannel, previousState, newState, keepaliveTime) => {
|
167 | this.childSubchannelState = newState;
|
168 | if (!this.ejected) {
|
169 | for (const listener of this.stateListeners) {
|
170 | listener(this, previousState, newState, keepaliveTime);
|
171 | }
|
172 | }
|
173 | });
|
174 | }
|
175 | getConnectivityState() {
|
176 | if (this.ejected) {
|
177 | return connectivity_state_1.ConnectivityState.TRANSIENT_FAILURE;
|
178 | }
|
179 | else {
|
180 | return this.childSubchannelState;
|
181 | }
|
182 | }
|
183 | |
184 |
|
185 |
|
186 |
|
187 |
|
188 | addConnectivityStateListener(listener) {
|
189 | this.stateListeners.push(listener);
|
190 | }
|
191 | |
192 |
|
193 |
|
194 |
|
195 |
|
196 | removeConnectivityStateListener(listener) {
|
197 | const listenerIndex = this.stateListeners.indexOf(listener);
|
198 | if (listenerIndex > -1) {
|
199 | this.stateListeners.splice(listenerIndex, 1);
|
200 | }
|
201 | }
|
202 | ref() {
|
203 | this.child.ref();
|
204 | this.refCount += 1;
|
205 | }
|
206 | unref() {
|
207 | this.child.unref();
|
208 | this.refCount -= 1;
|
209 | if (this.refCount <= 0) {
|
210 | if (this.mapEntry) {
|
211 | const index = this.mapEntry.subchannelWrappers.indexOf(this);
|
212 | if (index >= 0) {
|
213 | this.mapEntry.subchannelWrappers.splice(index, 1);
|
214 | }
|
215 | }
|
216 | }
|
217 | }
|
218 | eject() {
|
219 | this.ejected = true;
|
220 | for (const listener of this.stateListeners) {
|
221 | listener(this, this.childSubchannelState, connectivity_state_1.ConnectivityState.TRANSIENT_FAILURE, -1);
|
222 | }
|
223 | }
|
224 | uneject() {
|
225 | this.ejected = false;
|
226 | for (const listener of this.stateListeners) {
|
227 | listener(this, connectivity_state_1.ConnectivityState.TRANSIENT_FAILURE, this.childSubchannelState, -1);
|
228 | }
|
229 | }
|
230 | getMapEntry() {
|
231 | return this.mapEntry;
|
232 | }
|
233 | getWrappedSubchannel() {
|
234 | return this.child;
|
235 | }
|
236 | }
|
237 | function createEmptyBucket() {
|
238 | return {
|
239 | success: 0,
|
240 | failure: 0,
|
241 | };
|
242 | }
|
243 | class CallCounter {
|
244 | constructor() {
|
245 | this.activeBucket = createEmptyBucket();
|
246 | this.inactiveBucket = createEmptyBucket();
|
247 | }
|
248 | addSuccess() {
|
249 | this.activeBucket.success += 1;
|
250 | }
|
251 | addFailure() {
|
252 | this.activeBucket.failure += 1;
|
253 | }
|
254 | switchBuckets() {
|
255 | this.inactiveBucket = this.activeBucket;
|
256 | this.activeBucket = createEmptyBucket();
|
257 | }
|
258 | getLastSuccesses() {
|
259 | return this.inactiveBucket.success;
|
260 | }
|
261 | getLastFailures() {
|
262 | return this.inactiveBucket.failure;
|
263 | }
|
264 | }
|
265 | class OutlierDetectionPicker {
|
266 | constructor(wrappedPicker, countCalls) {
|
267 | this.wrappedPicker = wrappedPicker;
|
268 | this.countCalls = countCalls;
|
269 | }
|
270 | pick(pickArgs) {
|
271 | const wrappedPick = this.wrappedPicker.pick(pickArgs);
|
272 | if (wrappedPick.pickResultType === picker_1.PickResultType.COMPLETE) {
|
273 | const subchannelWrapper = wrappedPick.subchannel;
|
274 | const mapEntry = subchannelWrapper.getMapEntry();
|
275 | if (mapEntry) {
|
276 | let onCallEnded = wrappedPick.onCallEnded;
|
277 | if (this.countCalls) {
|
278 | onCallEnded = statusCode => {
|
279 | var _a;
|
280 | if (statusCode === constants_1.Status.OK) {
|
281 | mapEntry.counter.addSuccess();
|
282 | }
|
283 | else {
|
284 | mapEntry.counter.addFailure();
|
285 | }
|
286 | (_a = wrappedPick.onCallEnded) === null || _a === void 0 ? void 0 : _a.call(wrappedPick, statusCode);
|
287 | };
|
288 | }
|
289 | return Object.assign(Object.assign({}, wrappedPick), { subchannel: subchannelWrapper.getWrappedSubchannel(), onCallEnded: onCallEnded });
|
290 | }
|
291 | else {
|
292 | return Object.assign(Object.assign({}, wrappedPick), { subchannel: subchannelWrapper.getWrappedSubchannel() });
|
293 | }
|
294 | }
|
295 | else {
|
296 | return wrappedPick;
|
297 | }
|
298 | }
|
299 | }
|
300 | class OutlierDetectionLoadBalancer {
|
301 | constructor(channelControlHelper) {
|
302 | this.addressMap = new Map();
|
303 | this.latestConfig = null;
|
304 | this.timerStartTime = null;
|
305 | this.childBalancer = new load_balancer_child_handler_1.ChildLoadBalancerHandler((0, experimental_1.createChildChannelControlHelper)(channelControlHelper, {
|
306 | createSubchannel: (subchannelAddress, subchannelArgs) => {
|
307 | const originalSubchannel = channelControlHelper.createSubchannel(subchannelAddress, subchannelArgs);
|
308 | const mapEntry = this.addressMap.get((0, subchannel_address_1.subchannelAddressToString)(subchannelAddress));
|
309 | const subchannelWrapper = new OutlierDetectionSubchannelWrapper(originalSubchannel, mapEntry);
|
310 | if ((mapEntry === null || mapEntry === void 0 ? void 0 : mapEntry.currentEjectionTimestamp) !== null) {
|
311 |
|
312 | subchannelWrapper.eject();
|
313 | }
|
314 | mapEntry === null || mapEntry === void 0 ? void 0 : mapEntry.subchannelWrappers.push(subchannelWrapper);
|
315 | return subchannelWrapper;
|
316 | },
|
317 | updateState: (connectivityState, picker) => {
|
318 | if (connectivityState === connectivity_state_1.ConnectivityState.READY) {
|
319 | channelControlHelper.updateState(connectivityState, new OutlierDetectionPicker(picker, this.isCountingEnabled()));
|
320 | }
|
321 | else {
|
322 | channelControlHelper.updateState(connectivityState, picker);
|
323 | }
|
324 | },
|
325 | }));
|
326 | this.ejectionTimer = setInterval(() => { }, 0);
|
327 | clearInterval(this.ejectionTimer);
|
328 | }
|
329 | isCountingEnabled() {
|
330 | return (this.latestConfig !== null &&
|
331 | (this.latestConfig.getSuccessRateEjectionConfig() !== null ||
|
332 | this.latestConfig.getFailurePercentageEjectionConfig() !== null));
|
333 | }
|
334 | getCurrentEjectionPercent() {
|
335 | let ejectionCount = 0;
|
336 | for (const mapEntry of this.addressMap.values()) {
|
337 | if (mapEntry.currentEjectionTimestamp !== null) {
|
338 | ejectionCount += 1;
|
339 | }
|
340 | }
|
341 | return (ejectionCount * 100) / this.addressMap.size;
|
342 | }
|
343 | runSuccessRateCheck(ejectionTimestamp) {
|
344 | if (!this.latestConfig) {
|
345 | return;
|
346 | }
|
347 | const successRateConfig = this.latestConfig.getSuccessRateEjectionConfig();
|
348 | if (!successRateConfig) {
|
349 | return;
|
350 | }
|
351 | trace('Running success rate check');
|
352 |
|
353 | const targetRequestVolume = successRateConfig.request_volume;
|
354 | let addresesWithTargetVolume = 0;
|
355 | const successRates = [];
|
356 | for (const [address, mapEntry] of this.addressMap) {
|
357 | const successes = mapEntry.counter.getLastSuccesses();
|
358 | const failures = mapEntry.counter.getLastFailures();
|
359 | trace('Stats for ' +
|
360 | address +
|
361 | ': successes=' +
|
362 | successes +
|
363 | ' failures=' +
|
364 | failures +
|
365 | ' targetRequestVolume=' +
|
366 | targetRequestVolume);
|
367 | if (successes + failures >= targetRequestVolume) {
|
368 | addresesWithTargetVolume += 1;
|
369 | successRates.push(successes / (successes + failures));
|
370 | }
|
371 | }
|
372 | trace('Found ' +
|
373 | addresesWithTargetVolume +
|
374 | ' success rate candidates; currentEjectionPercent=' +
|
375 | this.getCurrentEjectionPercent() +
|
376 | ' successRates=[' +
|
377 | successRates +
|
378 | ']');
|
379 | if (addresesWithTargetVolume < successRateConfig.minimum_hosts) {
|
380 | return;
|
381 | }
|
382 |
|
383 | const successRateMean = successRates.reduce((a, b) => a + b) / successRates.length;
|
384 | let successRateDeviationSum = 0;
|
385 | for (const rate of successRates) {
|
386 | const deviation = rate - successRateMean;
|
387 | successRateDeviationSum += deviation * deviation;
|
388 | }
|
389 | const successRateVariance = successRateDeviationSum / successRates.length;
|
390 | const successRateStdev = Math.sqrt(successRateVariance);
|
391 | const ejectionThreshold = successRateMean -
|
392 | successRateStdev * (successRateConfig.stdev_factor / 1000);
|
393 | trace('stdev=' + successRateStdev + ' ejectionThreshold=' + ejectionThreshold);
|
394 |
|
395 | for (const [address, mapEntry] of this.addressMap.entries()) {
|
396 |
|
397 | if (this.getCurrentEjectionPercent() >=
|
398 | this.latestConfig.getMaxEjectionPercent()) {
|
399 | break;
|
400 | }
|
401 |
|
402 | const successes = mapEntry.counter.getLastSuccesses();
|
403 | const failures = mapEntry.counter.getLastFailures();
|
404 | if (successes + failures < targetRequestVolume) {
|
405 | continue;
|
406 | }
|
407 |
|
408 | const successRate = successes / (successes + failures);
|
409 | trace('Checking candidate ' + address + ' successRate=' + successRate);
|
410 | if (successRate < ejectionThreshold) {
|
411 | const randomNumber = Math.random() * 100;
|
412 | trace('Candidate ' +
|
413 | address +
|
414 | ' randomNumber=' +
|
415 | randomNumber +
|
416 | ' enforcement_percentage=' +
|
417 | successRateConfig.enforcement_percentage);
|
418 | if (randomNumber < successRateConfig.enforcement_percentage) {
|
419 | trace('Ejecting candidate ' + address);
|
420 | this.eject(mapEntry, ejectionTimestamp);
|
421 | }
|
422 | }
|
423 | }
|
424 | }
|
425 | runFailurePercentageCheck(ejectionTimestamp) {
|
426 | if (!this.latestConfig) {
|
427 | return;
|
428 | }
|
429 | const failurePercentageConfig = this.latestConfig.getFailurePercentageEjectionConfig();
|
430 | if (!failurePercentageConfig) {
|
431 | return;
|
432 | }
|
433 | trace('Running failure percentage check. threshold=' +
|
434 | failurePercentageConfig.threshold +
|
435 | ' request volume threshold=' +
|
436 | failurePercentageConfig.request_volume);
|
437 |
|
438 | let addressesWithTargetVolume = 0;
|
439 | for (const mapEntry of this.addressMap.values()) {
|
440 | const successes = mapEntry.counter.getLastSuccesses();
|
441 | const failures = mapEntry.counter.getLastFailures();
|
442 | if (successes + failures >= failurePercentageConfig.request_volume) {
|
443 | addressesWithTargetVolume += 1;
|
444 | }
|
445 | }
|
446 | if (addressesWithTargetVolume < failurePercentageConfig.minimum_hosts) {
|
447 | return;
|
448 | }
|
449 |
|
450 | for (const [address, mapEntry] of this.addressMap.entries()) {
|
451 |
|
452 | if (this.getCurrentEjectionPercent() >=
|
453 | this.latestConfig.getMaxEjectionPercent()) {
|
454 | break;
|
455 | }
|
456 |
|
457 | const successes = mapEntry.counter.getLastSuccesses();
|
458 | const failures = mapEntry.counter.getLastFailures();
|
459 | trace('Candidate successes=' + successes + ' failures=' + failures);
|
460 | if (successes + failures < failurePercentageConfig.request_volume) {
|
461 | continue;
|
462 | }
|
463 |
|
464 | const failurePercentage = (failures * 100) / (failures + successes);
|
465 | if (failurePercentage > failurePercentageConfig.threshold) {
|
466 | const randomNumber = Math.random() * 100;
|
467 | trace('Candidate ' +
|
468 | address +
|
469 | ' randomNumber=' +
|
470 | randomNumber +
|
471 | ' enforcement_percentage=' +
|
472 | failurePercentageConfig.enforcement_percentage);
|
473 | if (randomNumber < failurePercentageConfig.enforcement_percentage) {
|
474 | trace('Ejecting candidate ' + address);
|
475 | this.eject(mapEntry, ejectionTimestamp);
|
476 | }
|
477 | }
|
478 | }
|
479 | }
|
480 | eject(mapEntry, ejectionTimestamp) {
|
481 | mapEntry.currentEjectionTimestamp = new Date();
|
482 | mapEntry.ejectionTimeMultiplier += 1;
|
483 | for (const subchannelWrapper of mapEntry.subchannelWrappers) {
|
484 | subchannelWrapper.eject();
|
485 | }
|
486 | }
|
487 | uneject(mapEntry) {
|
488 | mapEntry.currentEjectionTimestamp = null;
|
489 | for (const subchannelWrapper of mapEntry.subchannelWrappers) {
|
490 | subchannelWrapper.uneject();
|
491 | }
|
492 | }
|
493 | switchAllBuckets() {
|
494 | for (const mapEntry of this.addressMap.values()) {
|
495 | mapEntry.counter.switchBuckets();
|
496 | }
|
497 | }
|
498 | startTimer(delayMs) {
|
499 | var _a, _b;
|
500 | this.ejectionTimer = setTimeout(() => this.runChecks(), delayMs);
|
501 | (_b = (_a = this.ejectionTimer).unref) === null || _b === void 0 ? void 0 : _b.call(_a);
|
502 | }
|
503 | runChecks() {
|
504 | const ejectionTimestamp = new Date();
|
505 | trace('Ejection timer running');
|
506 | this.switchAllBuckets();
|
507 | if (!this.latestConfig) {
|
508 | return;
|
509 | }
|
510 | this.timerStartTime = ejectionTimestamp;
|
511 | this.startTimer(this.latestConfig.getIntervalMs());
|
512 | this.runSuccessRateCheck(ejectionTimestamp);
|
513 | this.runFailurePercentageCheck(ejectionTimestamp);
|
514 | for (const [address, mapEntry] of this.addressMap.entries()) {
|
515 | if (mapEntry.currentEjectionTimestamp === null) {
|
516 | if (mapEntry.ejectionTimeMultiplier > 0) {
|
517 | mapEntry.ejectionTimeMultiplier -= 1;
|
518 | }
|
519 | }
|
520 | else {
|
521 | const baseEjectionTimeMs = this.latestConfig.getBaseEjectionTimeMs();
|
522 | const maxEjectionTimeMs = this.latestConfig.getMaxEjectionTimeMs();
|
523 | const returnTime = new Date(mapEntry.currentEjectionTimestamp.getTime());
|
524 | returnTime.setMilliseconds(returnTime.getMilliseconds() +
|
525 | Math.min(baseEjectionTimeMs * mapEntry.ejectionTimeMultiplier, Math.max(baseEjectionTimeMs, maxEjectionTimeMs)));
|
526 | if (returnTime < new Date()) {
|
527 | trace('Unejecting ' + address);
|
528 | this.uneject(mapEntry);
|
529 | }
|
530 | }
|
531 | }
|
532 | }
|
533 | updateAddressList(addressList, lbConfig, attributes) {
|
534 | if (!(lbConfig instanceof OutlierDetectionLoadBalancingConfig)) {
|
535 | return;
|
536 | }
|
537 | const subchannelAddresses = new Set();
|
538 | for (const address of addressList) {
|
539 | subchannelAddresses.add((0, subchannel_address_1.subchannelAddressToString)(address));
|
540 | }
|
541 | for (const address of subchannelAddresses) {
|
542 | if (!this.addressMap.has(address)) {
|
543 | trace('Adding map entry for ' + address);
|
544 | this.addressMap.set(address, {
|
545 | counter: new CallCounter(),
|
546 | currentEjectionTimestamp: null,
|
547 | ejectionTimeMultiplier: 0,
|
548 | subchannelWrappers: [],
|
549 | });
|
550 | }
|
551 | }
|
552 | for (const key of this.addressMap.keys()) {
|
553 | if (!subchannelAddresses.has(key)) {
|
554 | trace('Removing map entry for ' + key);
|
555 | this.addressMap.delete(key);
|
556 | }
|
557 | }
|
558 | const childPolicy = (0, load_balancer_1.getFirstUsableConfig)(lbConfig.getChildPolicy(), true);
|
559 | this.childBalancer.updateAddressList(addressList, childPolicy, attributes);
|
560 | if (lbConfig.getSuccessRateEjectionConfig() ||
|
561 | lbConfig.getFailurePercentageEjectionConfig()) {
|
562 | if (this.timerStartTime) {
|
563 | trace('Previous timer existed. Replacing timer');
|
564 | clearTimeout(this.ejectionTimer);
|
565 | const remainingDelay = lbConfig.getIntervalMs() -
|
566 | (new Date().getTime() - this.timerStartTime.getTime());
|
567 | this.startTimer(remainingDelay);
|
568 | }
|
569 | else {
|
570 | trace('Starting new timer');
|
571 | this.timerStartTime = new Date();
|
572 | this.startTimer(lbConfig.getIntervalMs());
|
573 | this.switchAllBuckets();
|
574 | }
|
575 | }
|
576 | else {
|
577 | trace('Counting disabled. Cancelling timer.');
|
578 | this.timerStartTime = null;
|
579 | clearTimeout(this.ejectionTimer);
|
580 | for (const mapEntry of this.addressMap.values()) {
|
581 | this.uneject(mapEntry);
|
582 | mapEntry.ejectionTimeMultiplier = 0;
|
583 | }
|
584 | }
|
585 | this.latestConfig = lbConfig;
|
586 | }
|
587 | exitIdle() {
|
588 | this.childBalancer.exitIdle();
|
589 | }
|
590 | resetBackoff() {
|
591 | this.childBalancer.resetBackoff();
|
592 | }
|
593 | destroy() {
|
594 | clearTimeout(this.ejectionTimer);
|
595 | this.childBalancer.destroy();
|
596 | }
|
597 | getTypeName() {
|
598 | return TYPE_NAME;
|
599 | }
|
600 | }
|
601 | exports.OutlierDetectionLoadBalancer = OutlierDetectionLoadBalancer;
|
602 | function setup() {
|
603 | if (OUTLIER_DETECTION_ENABLED) {
|
604 | (0, experimental_1.registerLoadBalancerType)(TYPE_NAME, OutlierDetectionLoadBalancer, OutlierDetectionLoadBalancingConfig);
|
605 | }
|
606 | }
|
607 | exports.setup = setup;
|
608 |
|
\ | No newline at end of file |