// SPDX-License-Identifier: Apache-2.0 import {Listr} from 'listr2'; import {ListrInquirerPromptAdapter} from '@listr2/prompt-adapter-inquirer'; import {confirm as confirmPrompt} from '@inquirer/prompts'; import chalk from 'chalk'; import {SoloError} from '../core/errors/solo-error.js'; import {UserBreak} from '../core/errors/user-break.js'; import {BaseCommand} from './base.js'; import {Flags as flags} from './flags.js'; import * as constants from '../core/constants.js'; import {getEnvironmentVariable} from '../core/constants.js'; import {Templates} from '../core/templates.js'; import { addRootImageValues, createAndCopyBlockNodeJsonFileForConsensusNode, parseNodeAliases, prepareValuesFilesMapMultipleCluster, resolveValidJsonFilePath, showVersionBanner, sleep, } from '../core/helpers.js'; import {helmValuesHelper} from '../core/helm-values-helper.js'; import {type PerNodeIdentity} from '../types/helm-values.js'; import {resolveNamespaceFromDeployment} from '../core/resolvers.js'; import fs from 'node:fs'; import path from 'node:path'; import {type KeyManager} from '../core/key-manager.js'; import {type PlatformInstaller} from '../core/platform-installer.js'; import {type ProfileManager} from '../core/profile-manager.js'; import {type CertificateManager} from '../core/certificate-manager.js'; import {type AnyListrContext, type ArgvStruct, type IP, type NodeAlias, type NodeAliases} from '../types/aliases.js'; import {ListrLock} from '../core/lock/listr-lock.js'; import {v4 as uuidv4} from 'uuid'; import { type ClusterReferenceName, type ClusterReferences, type ComponentId, type Context, type DeploymentName, type PrivateKeyAndCertificateObject, type Realm, type Shard, type SoloListr, type SoloListrTask, type SoloListrTaskWrapper, } from '../types/index.js'; import {Base64} from 'js-base64'; import {SecretType} from '../integration/kube/resources/secret/secret-type.js'; import {Duration} from '../core/time/duration.js'; import {type Pod} from '../integration/kube/resources/pod/pod.js'; import {PathEx} from '../business/utils/path-ex.js'; import {inject, injectable} from 'tsyringe-neo'; import {InjectTokens} from '../core/dependency-injection/inject-tokens.js'; import {patchInject} from '../core/dependency-injection/container-helper.js'; import {type CommandFlag, type CommandFlags} from '../types/flag-types.js'; import {type K8} from '../integration/kube/k8.js'; import {type Lock} from '../core/lock/lock.js'; import {type LoadBalancerIngress} from '../integration/kube/resources/load-balancer-ingress.js'; import {type Service} from '../integration/kube/resources/service/service.js'; import {type Container} from '../integration/kube/resources/container/container.js'; import {DeploymentPhase} from '../data/schema/model/remote/deployment-phase.js'; import {ComponentTypes} from '../core/config/remote/enumerations/component-types.js'; import {PvcName} from '../integration/kube/resources/pvc/pvc-name.js'; import {PvcReference} from '../integration/kube/resources/pvc/pvc-reference.js'; import {NamespaceName} from '../types/namespace/namespace-name.js'; import {ConsensusNode} from '../core/model/consensus-node.js'; import {BlockNodeStateSchema} from '../data/schema/model/remote/state/block-node-state-schema.js'; import {SemanticVersion} from '../business/utils/semantic-version.js'; import {Secret} from '../integration/kube/resources/secret/secret.js'; import * as versions from '../../version.js'; import {K8Helper} from '../business/utils/k8-helper.js'; import {PackageDownloader} from '../core/package-downloader.js'; import {Zippy} from '../core/zippy.js'; import {type SoloEventBus} from '../core/events/solo-event-bus.js'; import {NetworkDeployedEvent} from '../core/events/event-types/network-deployed-event.js'; import {type Wraps} from '../business/runtime-state/config/solo/wraps.js'; export interface NetworkDeployConfigClass { isUpgrade: boolean; applicationEnv: string; chainId: string; cacheDir: string; chartDirectory: string; loadBalancerEnabled: boolean; soloChartVersion: string; namespace: NamespaceName; deployment: string; nodeAliasesUnparsed: string; persistentVolumeClaims: string; releaseTag: string; keysDir: string; nodeAliases: NodeAliases; stagingDir: string; stagingKeysDir: string; valuesFile: string; valuesArgMap: Record; grpcTlsCertificatePath: string; grpcWebTlsCertificatePath: string; grpcTlsKeyPath: string; grpcWebTlsKeyPath: string; genesisThrottlesFile: string; resolvedThrottlesFile: string; haproxyIps: string; envoyIps: string; haproxyIpsParsed?: Record; envoyIpsParsed?: Record; storageType: constants.StorageType; gcsWriteAccessKey: string; gcsWriteSecrets: string; gcsEndpoint: string; gcsBucket: string; gcsBucketPrefix: string; awsWriteAccessKey: string; awsWriteSecrets: string; awsEndpoint: string; awsBucket: string; awsBucketPrefix: string; awsBucketRegion: string; backupBucket: string; backupWriteSecrets: string; backupWriteAccessKey: string; backupEndpoint: string; backupRegion: string; backupProvider: string; consensusNodes: ConsensusNode[]; contexts: string[]; clusterRefs: ClusterReferences; domainNames?: string; domainNamesMapping?: Record; blockNodeComponents: BlockNodeStateSchema[]; debugNodeAlias: NodeAlias; app: string; serviceMonitor: string; podLog: string; singleUseServiceMonitor: string; singleUsePodLog: string; enableMonitoringSupport: boolean; javaFlightRecorderConfiguration: string; wrapsEnabled: boolean; wrapsKeyPath: string; tssEnabled: boolean; } interface NetworkDeployContext { config: NetworkDeployConfigClass; } export interface NetworkDestroyContext { config: { deletePvcs: boolean; deleteSecrets: boolean; namespace: NamespaceName; enableTimeout: boolean; force: boolean; contexts: string[]; deployment: string; }; checkTimeout: boolean; } @injectable() export class NetworkCommand extends BaseCommand { private profileValuesFile?: Record; public constructor( @inject(InjectTokens.CertificateManager) private readonly certificateManager: CertificateManager, @inject(InjectTokens.KeyManager) private readonly keyManager: KeyManager, @inject(InjectTokens.PlatformInstaller) private readonly platformInstaller: PlatformInstaller, @inject(InjectTokens.ProfileManager) private readonly profileManager: ProfileManager, @inject(InjectTokens.Zippy) private readonly zippy: Zippy, @inject(InjectTokens.PackageDownloader) private readonly downloader: PackageDownloader, @inject(InjectTokens.SoloEventBus) private readonly eventBus: SoloEventBus, ) { super(); this.certificateManager = patchInject(certificateManager, InjectTokens.CertificateManager, this.constructor.name); this.keyManager = patchInject(keyManager, InjectTokens.KeyManager, this.constructor.name); this.platformInstaller = patchInject(platformInstaller, InjectTokens.PlatformInstaller, this.constructor.name); this.profileManager = patchInject(profileManager, InjectTokens.ProfileManager, this.constructor.name); this.zippy = patchInject(zippy, InjectTokens.Zippy, this.constructor.name); this.downloader = patchInject(downloader, InjectTokens.PackageDownloader, this.constructor.name); } private static readonly DEPLOY_CONFIGS_NAME: string = 'deployConfigs'; public static readonly DESTROY_FLAGS_LIST: CommandFlags = { required: [flags.deployment], optional: [flags.deletePvcs, flags.deleteSecrets, flags.enableTimeout, flags.force, flags.quiet], }; public static readonly DEPLOY_FLAGS_LIST: CommandFlags = { required: [flags.deployment], optional: [ flags.apiPermissionProperties, flags.app, flags.applicationEnv, flags.applicationProperties, flags.bootstrapProperties, flags.genesisThrottlesFile, flags.cacheDir, flags.chainId, flags.chartDirectory, flags.soloChartVersion, flags.debugNodeAlias, flags.loadBalancerEnabled, flags.log4j2Xml, flags.persistentVolumeClaims, flags.quiet, flags.releaseTag, flags.settingTxt, flags.networkDeploymentValuesFile, flags.nodeAliasesUnparsed, flags.grpcTlsCertificatePath, flags.grpcWebTlsCertificatePath, flags.grpcTlsKeyPath, flags.grpcWebTlsKeyPath, flags.haproxyIps, flags.envoyIps, flags.storageType, flags.gcsWriteAccessKey, flags.gcsWriteSecrets, flags.gcsEndpoint, flags.gcsBucket, flags.gcsBucketPrefix, flags.awsWriteAccessKey, flags.awsWriteSecrets, flags.awsEndpoint, flags.awsBucket, flags.awsBucketRegion, flags.awsBucketPrefix, flags.backupBucket, flags.backupWriteAccessKey, flags.backupWriteSecrets, flags.backupEndpoint, flags.backupRegion, flags.backupProvider, flags.domainNames, flags.serviceMonitor, flags.podLog, flags.enableMonitoringSupport, flags.javaFlightRecorderConfiguration, flags.wrapsEnabled, flags.wrapsKeyPath, flags.tssEnabled, ], }; private waitForNetworkPods(): SoloListrTask { return { title: 'Check node pods are running', task: (context_, task): SoloListr => { const subTasks: SoloListrTask[] = []; const config: NetworkDeployConfigClass = context_.config; for (const consensusNode of config.consensusNodes) { subTasks.push({ title: `Check Node: ${chalk.yellow(consensusNode.name)}, Cluster: ${chalk.yellow(consensusNode.cluster)}`, task: async (): Promise => { await this.k8Factory .getK8(consensusNode.context) .pods() .waitForRunningPhase( config.namespace, [`solo.hedera.com/node-name=${consensusNode.name}`, 'solo.hedera.com/type=network-node'], constants.PODS_RUNNING_MAX_ATTEMPTS, constants.PODS_RUNNING_DELAY, ); }, }); } // set up the sub-tasks return task.newListr(subTasks, { concurrent: true, rendererOptions: { collapseSubtasks: false, }, }); }, }; } private async prepareMinioSecrets( config: NetworkDeployConfigClass, minioAccessKey: string, minioSecretKey: string, ): Promise { // Generating new minio credentials const minioData: Record = {}; const namespace: NamespaceName = config.namespace; const environmentString: string = `MINIO_ROOT_USER=${minioAccessKey}\nMINIO_ROOT_PASSWORD=${minioSecretKey}`; minioData['config.env'] = Base64.encode(environmentString); // create minio secret in each cluster for (const context of config.contexts) { this.logger.debug(`creating minio secret using context: ${context}`); const isMinioSecretCreated: boolean = await this.k8Factory .getK8(context) .secrets() .createOrReplace(namespace, constants.MINIO_SECRET_NAME, SecretType.OPAQUE, minioData); if (!isMinioSecretCreated) { throw new SoloError(`failed to create new minio secret using context: ${context}`); } this.logger.debug(`created minio secret using context: ${context}`); } } private async prepareStreamUploaderSecrets(config: NetworkDeployConfigClass): Promise { const namespace: NamespaceName = config.namespace; // Generating cloud storage secrets const {gcsWriteAccessKey, gcsWriteSecrets, gcsEndpoint, awsWriteAccessKey, awsWriteSecrets, awsEndpoint} = config; const cloudData: Record = {}; if ( config.storageType === constants.StorageType.AWS_ONLY || config.storageType === constants.StorageType.AWS_AND_GCS ) { cloudData['S3_ACCESS_KEY'] = Base64.encode(awsWriteAccessKey); cloudData['S3_SECRET_KEY'] = Base64.encode(awsWriteSecrets); cloudData['S3_ENDPOINT'] = Base64.encode(awsEndpoint); } if ( config.storageType === constants.StorageType.GCS_ONLY || config.storageType === constants.StorageType.AWS_AND_GCS ) { cloudData['GCS_ACCESS_KEY'] = Base64.encode(gcsWriteAccessKey); cloudData['GCS_SECRET_KEY'] = Base64.encode(gcsWriteSecrets); cloudData['GCS_ENDPOINT'] = Base64.encode(gcsEndpoint); } // create secret in each cluster for (const context of config.contexts) { this.logger.debug( `creating secret for storage credential of type '${config.storageType}' using context: ${context}`, ); const isCloudSecretCreated: boolean = await this.k8Factory .getK8(context) .secrets() .createOrReplace(namespace, constants.UPLOADER_SECRET_NAME, SecretType.OPAQUE, cloudData); if (!isCloudSecretCreated) { throw new SoloError( `failed to create secret for storage credentials of type '${config.storageType}' using context: ${context}`, ); } this.logger.debug( `created secret for storage credential of type '${config.storageType}' using context: ${context}`, ); } } private async prepareBackupUploaderSecrets(config: NetworkDeployConfigClass): Promise { const {backupWriteAccessKey, backupWriteSecrets, backupEndpoint, backupRegion, backupProvider} = config; const backupData: Record = {}; const namespace: NamespaceName = config.namespace; backupData['AWS_ACCESS_KEY_ID'] = Base64.encode(backupWriteAccessKey); backupData['AWS_SECRET_ACCESS_KEY'] = Base64.encode(backupWriteSecrets); backupData['RCLONE_CONFIG_BACKUPS_ENDPOINT'] = Base64.encode(backupEndpoint); backupData['RCLONE_CONFIG_BACKUPS_REGION'] = Base64.encode(backupRegion); backupData['RCLONE_CONFIG_BACKUPS_TYPE'] = Base64.encode('s3'); backupData['RCLONE_CONFIG_BACKUPS_PROVIDER'] = Base64.encode(backupProvider); // create secret in each cluster for (const context of config.contexts) { this.logger.debug(`creating secret for backup uploader using context: ${context}`); const k8client: K8 = this.k8Factory.getK8(context); const isBackupSecretCreated: boolean = await k8client .secrets() .createOrReplace(namespace, constants.BACKUP_SECRET_NAME, SecretType.OPAQUE, backupData); if (!isBackupSecretCreated) { throw new SoloError(`failed to create secret for backup uploader using context: ${context}`); } this.logger.debug(`created secret for backup uploader using context: ${context}`); } } private async prepareStorageSecrets(config: NetworkDeployConfigClass): Promise { try { if (config.storageType !== constants.StorageType.MINIO_ONLY) { const minioAccessKey: string = uuidv4(); const minioSecretKey: string = uuidv4(); await this.prepareMinioSecrets(config, minioAccessKey, minioSecretKey); await this.prepareStreamUploaderSecrets(config); } if (config.backupBucket) { await this.prepareBackupUploaderSecrets(config); } } catch (error) { throw new SoloError('Failed to create Kubernetes storage secret', error); } } /** * Prepare values args string for each cluster-ref * @param config */ private async prepareValuesArgMap(config: NetworkDeployConfigClass): Promise> { const valuesArguments: Record = this.prepareValuesArg(config); // prepare values files for each cluster const valuesArgumentMap: Record = {}; const deploymentName: DeploymentName = this.configManager.getFlag(flags.deployment); const applicationPropertiesPath: string = PathEx.joinWithRealPath( config.cacheDir, 'templates', constants.APPLICATION_PROPERTIES, ); const jfrFilePath: string = config.javaFlightRecorderConfiguration; const jfrFile: string = jfrFilePath === '' ? '' : jfrFilePath.slice(Math.max(0, jfrFilePath.lastIndexOf(path.sep) + 1)); this.profileValuesFile = await this.profileManager.prepareValuesForSoloChart( config.consensusNodes, deploymentName, applicationPropertiesPath, jfrFile, { // Pass command-scoped values explicitly so profile/staging generation is isolated // from mutable global flags when one-shot runs parallel subcommands. cacheDir: config.cacheDir, releaseTag: config.releaseTag, appName: config.app, chainId: config.chainId, }, ); const valuesFiles: Record = prepareValuesFilesMapMultipleCluster( config.clusterRefs, config.chartDirectory, this.profileValuesFile, config.valuesFile, [constants.SOLO_DEPLOYMENT_VALUES_FILE], ); // Generate per-cluster extraEnv values files to avoid passing the global node list to every // cluster's Helm upgrade (in multi-cluster deployments each cluster has its own node subset). // Each file carries only the nodes that belong to the target cluster, preventing Helm's // array-replacement semantics from inserting nodes from other clusters. const perClusterExtraEnvironmentValuesFiles: Record = {}; const needsExtraEnvironment: boolean = config.wrapsEnabled || !!config.debugNodeAlias || config.app !== constants.HEDERA_APP_NAME; // JAVA_MAIN_CLASS for tools/local builds if (needsExtraEnvironment) { const realm: Realm = this.localConfig.configuration.realmForDeployment(config.deployment); const shard: Shard = this.localConfig.configuration.shardForDeployment(config.deployment); for (const clusterReference of Object.keys(valuesFiles)) { // Only include nodes belonging to this cluster so the generated hedera.nodes array // matches the cluster-specific node set and does not overwrite nodes in other clusters. // Sort deterministically by nodeId so per-node Helm values align with the chart's // expected node ordering regardless of upstream object iteration order. const clusterConsensusNodes: ConsensusNode[] = config.consensusNodes .filter((node): boolean => node.cluster === clusterReference) // eslint-disable-next-line unicorn/no-array-sort .sort((left, right): number => left.nodeId - right.nodeId); if (clusterConsensusNodes.length === 0) { continue; } const additionalNodeValues: Record< NodeAlias, {name: NodeAlias; nodeId: number; accountId: string; blockNodesJson?: string} > = {}; // Preserve blockNodesJson from the per-cluster profile values file so that it is not // silently dropped when the extraEnv values file replaces the hedera.nodes array. const clusterProfileValuesFile: string | undefined = this.profileValuesFile?.[clusterReference]; const nodeIdentityMap: Record = clusterProfileValuesFile ? helmValuesHelper.extractPerNodeIdentityFromValuesFile(clusterProfileValuesFile, clusterConsensusNodes) : {}; const blockNodesJsonMap: Record = clusterProfileValuesFile ? helmValuesHelper.extractPerNodeBlockNodesJsonFromValuesFile(clusterProfileValuesFile, clusterConsensusNodes) : {}; for (const consensusNode of clusterConsensusNodes) { const identity: PerNodeIdentity = nodeIdentityMap[consensusNode.name] ?? {}; additionalNodeValues[consensusNode.name] = { name: identity.name ?? consensusNode.name, nodeId: identity.nodeId ?? consensusNode.nodeId, // Prefer the accountId recorded in the profile values file (set by the account // manager using the deployment's configured start account ID) over the computed // default, so custom account IDs assigned via node transactions are preserved. accountId: identity.accountId ?? `${shard}.${realm}.${constants.DEFAULT_START_ID_NUMBER + consensusNode.nodeId}`, }; if (blockNodesJsonMap[consensusNode.name]) { additionalNodeValues[consensusNode.name].blockNodesJson = blockNodesJsonMap[consensusNode.name]; } } // Collect extraEnv entries already present in this cluster's values files so that the // generated file can include them and avoid Helm array replacement silently dropping // env vars set by user-provided values files. const existingValuesFilePaths: string[] = helmValuesHelper.parseValuesFilePaths(valuesFiles[clusterReference]); const clusterExtraEnvironmentValuesFile: string = helmValuesHelper.generateExtraEnvironmentValuesFile( clusterConsensusNodes, { wrapsEnabled: config.wrapsEnabled, tss: this.soloConfig.tss, debugNodeAlias: config.debugNodeAlias, useJavaMainClass: config.app !== constants.HEDERA_APP_NAME, additionalNodeValues, baseExtraEnvironmentVariables: helmValuesHelper.extractExtraEnvironmentFromValuesFiles( existingValuesFilePaths, clusterConsensusNodes, ), }, config.cacheDir, ); perClusterExtraEnvironmentValuesFiles[clusterReference] = clusterExtraEnvironmentValuesFile; this.logger.debug( `Created per-cluster extraEnv values file for ${clusterReference}: ${clusterExtraEnvironmentValuesFile}`, ); } } for (const clusterReference of Object.keys(valuesFiles)) { // Keep --set flags last so they override values files. This is critical when we also // provide per-node extraEnv via a values file (e.g. --debug-node-alias), because a later // values file can replace array elements and drop fields like node labels/account IDs. let valuesArgument: string = valuesFiles[clusterReference]; // Add per-cluster extraEnv values file if any extraEnv customizations are needed if (perClusterExtraEnvironmentValuesFiles[clusterReference]) { valuesArgument += ` --values "${perClusterExtraEnvironmentValuesFiles[clusterReference]}"`; } valuesArgument += valuesArguments[clusterReference]; valuesArgumentMap[clusterReference] = valuesArgument; this.logger.debug(`Prepared helm chart values for cluster-ref: ${clusterReference}`, { valuesArgument: valuesArgumentMap[clusterReference], }); } return valuesArgumentMap; } /** * Prepare the values argument for the helm chart for a given config * @param config */ private prepareValuesArg(config: NetworkDeployConfigClass): Record { const valuesArguments: Record = {}; const clusterReferences: ClusterReferenceName[] = []; // initialize the valueArgs for (const consensusNode of config.consensusNodes) { // add the cluster to the list of clusters if (!clusterReferences.includes(consensusNode.cluster)) { clusterReferences.push(consensusNode.cluster); } // Initialize empty valuesArg for each cluster // All extraEnv logic (JAVA_MAIN_CLASS, TSS wraps, debug) is now handled via values files if (!valuesArguments[consensusNode.cluster]) { valuesArguments[consensusNode.cluster] = ''; } } // All extraEnv customizations (wraps, debug, JAVA_MAIN_CLASS) are handled // via generateExtraEnvironmentValuesFile() in prepareValuesArgMap() to avoid Helm --set replacement issues if ( config.storageType === constants.StorageType.AWS_AND_GCS || config.storageType === constants.StorageType.GCS_ONLY ) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ' --set cloud.gcs.enabled=true'; } } if ( config.storageType === constants.StorageType.AWS_AND_GCS || config.storageType === constants.StorageType.AWS_ONLY ) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ' --set cloud.s3.enabled=true'; } } if ( config.storageType === constants.StorageType.GCS_ONLY || config.storageType === constants.StorageType.AWS_ONLY || config.storageType === constants.StorageType.AWS_AND_GCS ) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ' --set cloud.minio.enabled=false'; } } if (config.storageType !== constants.StorageType.MINIO_ONLY) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ' --set cloud.generateNewSecrets=false'; } } if (config.gcsBucket) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ` --set cloud.buckets.streamBucket=${config.gcsBucket}` + ` --set minio-server.tenant.buckets[0].name=${config.gcsBucket}`; } } if (config.gcsBucketPrefix) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ` --set cloud.buckets.streamBucketPrefix=${config.gcsBucketPrefix}`; } } if (config.awsBucket) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ` --set cloud.buckets.streamBucket=${config.awsBucket}` + ` --set minio-server.tenant.buckets[0].name=${config.awsBucket}`; } } if (config.awsBucketPrefix) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ` --set cloud.buckets.streamBucketPrefix=${config.awsBucketPrefix}`; } } if (config.awsBucketRegion) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ` --set cloud.buckets.streamBucketRegion=${config.awsBucketRegion}`; } } if (config.backupBucket) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ' --set defaults.sidecars.backupUploader.enabled=true' + ` --set defaults.sidecars.backupUploader.config.backupBucket=${config.backupBucket}`; } } const nodeIndexByClusterAndName: Map = new Map(); const nextNodeIndexByCluster: Map = new Map(); for (const consensusNode of config.consensusNodes) { const nodeIndex: number = nextNodeIndexByCluster.get(consensusNode.cluster) ?? 0; nextNodeIndexByCluster.set(consensusNode.cluster, nodeIndex + 1); nodeIndexByClusterAndName.set(`${consensusNode.cluster}:${consensusNode.name}`, nodeIndex); } for (const consensusNode of config.consensusNodes) { const nodeIndex: number | undefined = nodeIndexByClusterAndName.get( `${consensusNode.cluster}:${consensusNode.name}`, ); if (nodeIndex === undefined) { continue; } let valuesArgument: string = valuesArguments[consensusNode.cluster] ?? ''; valuesArgument += ` --set "hedera.nodes[${nodeIndex}].name=${consensusNode.name}"`; valuesArgument = addRootImageValues( valuesArgument, `hedera.nodes[${nodeIndex}]`, constants.S6_NODE_IMAGE_REGISTRY, constants.S6_NODE_IMAGE_REPOSITORY, versions.S6_NODE_IMAGE_VERSION, ); valuesArguments[consensusNode.cluster] = valuesArgument; } for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ' --install' + ' --set "telemetry.prometheus.svcMonitor.enabled=false"' + // remove after chart version is bumped ` --set "crds.serviceMonitor.enabled=${config.singleUseServiceMonitor}"` + ` --set "crds.podLog.enabled=${config.singleUsePodLog}"` + ` --set "defaults.volumeClaims.enabled=${config.persistentVolumeClaims}"`; } config.singleUseServiceMonitor = 'false'; config.singleUsePodLog = 'false'; // Iterate over each node and set static IPs for HAProxy this.addArgForEachRecord( config.haproxyIpsParsed, config.consensusNodes, valuesArguments, ' --set "hedera.nodes[${nodeId}].haproxyStaticIP=${recordValue}"', ); // Iterate over each node and set static IPs for Envoy Proxy this.addArgForEachRecord( config.envoyIpsParsed, config.consensusNodes, valuesArguments, ' --set "hedera.nodes[${nodeId}].envoyProxyStaticIP=${recordValue}"', ); if (config.resolvedThrottlesFile) { // repairing the path, this avoid helm failing when running on windows const throttlesFilePath: string = config.resolvedThrottlesFile.replaceAll('\\', '/'); for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ` --set-file "hedera.configMaps.genesisThrottlesJson=${throttlesFilePath}"`; } } if (config.loadBalancerEnabled) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ' --set "defaults.haproxy.service.type=LoadBalancer"' + ' --set "defaults.envoyProxy.service.type=LoadBalancer"' + ' --set "defaults.consensus.service.type=LoadBalancer"'; } } if (config.enableMonitoringSupport) { for (const clusterReference of clusterReferences) { valuesArguments[clusterReference] += ' --set "crs.podLog.enabled=true" --set "crs.serviceMonitor.enabled=true"'; } } return valuesArguments; } /** * Adds the template string to the argument for each record * @param records - the records to iterate over * @param consensusNodes - the consensus nodes to iterate over * @param valuesArguments - the values arguments to add to * @param templateString - the template string to add */ private addArgForEachRecord( records: Record, consensusNodes: ConsensusNode[], valuesArguments: Record, templateString: string, ): void { if (records) { for (const consensusNode of consensusNodes) { if (records[consensusNode.name]) { const newTemplateString: string = templateString.replace('{nodeId}', consensusNode.nodeId.toString()); valuesArguments[consensusNode.cluster] += newTemplateString.replace( '{recordValue}', records[consensusNode.name], ); } } } } private async prepareNamespaces(config: NetworkDeployConfigClass): Promise { const namespace: NamespaceName = config.namespace; // check and create namespace in each cluster for (const context of config.contexts) { const k8client: K8 = this.k8Factory.getK8(context); if (await k8client.namespaces().has(namespace)) { this.logger.debug(`namespace '${namespace}' found using context: ${context}`); } else { this.logger.debug(`creating namespace '${namespace}' using context: ${context}`); await k8client.namespaces().create(namespace); this.logger.debug(`created namespace '${namespace}' using context: ${context}`); } } } private async prepareConfig( task: SoloListrTaskWrapper, argv: ArgvStruct, ): Promise { const flagsWithDisabledPrompts: CommandFlag[] = [ flags.apiPermissionProperties, flags.app, flags.applicationEnv, flags.applicationProperties, flags.bootstrapProperties, flags.genesisThrottlesFile, flags.cacheDir, flags.chainId, flags.chartDirectory, flags.debugNodeAlias, flags.loadBalancerEnabled, flags.log4j2Xml, flags.persistentVolumeClaims, flags.settingTxt, flags.grpcTlsCertificatePath, flags.grpcWebTlsCertificatePath, flags.grpcTlsKeyPath, flags.grpcWebTlsKeyPath, flags.haproxyIps, flags.envoyIps, flags.storageType, flags.gcsWriteAccessKey, flags.gcsWriteSecrets, flags.gcsEndpoint, flags.gcsBucket, flags.gcsBucketPrefix, flags.nodeAliasesUnparsed, flags.domainNames, ]; // disable the prompts that we don't want to prompt the user for flags.disablePrompts(flagsWithDisabledPrompts); const allFlags: CommandFlag[] = [ ...NetworkCommand.DEPLOY_FLAGS_LIST.optional, ...NetworkCommand.DEPLOY_FLAGS_LIST.required, ]; await this.configManager.executePrompt(task, allFlags); const namespace: NamespaceName = (await resolveNamespaceFromDeployment(this.localConfig, this.configManager, task)) ?? NamespaceName.of(this.configManager.getFlag(flags.deployment)); this.configManager.setFlag(flags.namespace, namespace); // create a config object for subsequent steps const config: NetworkDeployConfigClass = this.configManager.getConfig( NetworkCommand.DEPLOY_CONFIGS_NAME, allFlags, [ 'keysDir', 'nodeAliases', 'stagingDir', 'stagingKeysDir', 'valuesArgMap', 'resolvedThrottlesFile', 'namespace', 'consensusNodes', 'contexts', 'clusterRefs', 'singleUsePodLog', 'singleUseServiceMonitor', ], ) as NetworkDeployConfigClass; const realm: Realm = this.localConfig.configuration.realmForDeployment(config.deployment); const shard: Shard = this.localConfig.configuration.shardForDeployment(config.deployment); const networkNodeVersion: SemanticVersion = new SemanticVersion(config.releaseTag); const minimumVersionForNonZeroRealms: SemanticVersion = new SemanticVersion('0.60.0'); if ( (realm !== 0 || shard !== 0) && new SemanticVersion(networkNodeVersion).lessThan(minimumVersionForNonZeroRealms) ) { throw new SoloError( `The realm and shard values must be 0 when using the ${minimumVersionForNonZeroRealms} version of the network node`, ); } if (config.haproxyIps) { config.haproxyIpsParsed = Templates.parseNodeAliasToIpMapping(config.haproxyIps); } if (config.envoyIps) { config.envoyIpsParsed = Templates.parseNodeAliasToIpMapping(config.envoyIps); } if (config.domainNames) { config.domainNamesMapping = Templates.parseNodeAliasToDomainNameMapping(config.domainNames); } // compute other config parameters config.keysDir = PathEx.join(config.cacheDir, 'keys'); config.stagingDir = Templates.renderStagingDir(config.cacheDir, config.releaseTag); config.stagingKeysDir = PathEx.join(config.stagingDir, 'keys'); config.resolvedThrottlesFile = resolveValidJsonFilePath( config.genesisThrottlesFile, flags.genesisThrottlesFile.definition.defaultValue as string, ); config.consensusNodes = this.remoteConfig.getConsensusNodes(); config.contexts = this.remoteConfig.getContexts(); config.clusterRefs = this.remoteConfig.getClusterRefs(); config.nodeAliases = parseNodeAliases(config.nodeAliasesUnparsed, config.consensusNodes, this.configManager); argv[flags.nodeAliasesUnparsed.name] = config.nodeAliases.join(','); config.blockNodeComponents = this.getBlockNodes(); config.javaFlightRecorderConfiguration = this.configManager.getFlag(flags.javaFlightRecorderConfiguration); if (config.javaFlightRecorderConfiguration === '') { config.javaFlightRecorderConfiguration = getEnvironmentVariable('JAVA_FLIGHT_RECORDER_CONFIGURATION') || ''; } config.singleUseServiceMonitor = config.serviceMonitor; config.singleUsePodLog = config.podLog; config.valuesArgMap = await this.prepareValuesArgMap(config); // need to prepare the namespaces before we can proceed config.namespace = namespace; await this.prepareNamespaces(config); // prepare staging keys directory if (!fs.existsSync(config.stagingKeysDir)) { fs.mkdirSync(config.stagingKeysDir, {recursive: true}); } // create cached keys dir if it does not exist yet if (!fs.existsSync(config.keysDir)) { fs.mkdirSync(config.keysDir); } this.logger.debug('Preparing storage secrets'); await this.prepareStorageSecrets(config); return config; } private async destroyTask( task: SoloListrTaskWrapper, namespace: NamespaceName, deletePvcs: boolean, deleteSecrets: boolean, contexts: Context[], ): Promise { task.title = `Uninstalling chart ${constants.SOLO_DEPLOYMENT_CHART}`; // Uninstall all 'solo deployment' charts for each cluster using the contexts await this.logDestroyResults( 'Uninstall solo-deployment chart', await Promise.allSettled( contexts.map(async (context): Promise => { await this.chartManager.uninstall( namespace, constants.SOLO_DEPLOYMENT_CHART, this.k8Factory.getK8(context).contexts().readCurrent(), ); }), ), ); task.title = `Deleting the RemoteConfig configmap in namespace ${namespace}`; await this.logDestroyResults( 'Delete remote config configmap', await Promise.allSettled( contexts.map(async (context): Promise => { await this.k8Factory.getK8(context).configMaps().delete(namespace, constants.SOLO_REMOTE_CONFIGMAP_NAME); }), ), ); if (deletePvcs) { task.title = `Deleting PVCs in namespace ${namespace}`; await this.logDestroyResults('Delete PVCs', await Promise.allSettled([this.deletePvcs(namespace, contexts)])); } if (deleteSecrets) { task.title = `Deleting Secrets in namespace ${namespace}`; await this.logDestroyResults( 'Delete secrets', await Promise.allSettled([this.deleteSecrets(namespace, contexts)]), ); } if (deleteSecrets && deletePvcs) { task.title = `Deleting namespace ${namespace}`; await this.logDestroyResults( 'Delete namespace', await Promise.allSettled( contexts.map(async (context): Promise => { await this.k8Factory.getK8(context).namespaces().delete(namespace); }), ), ); } else { task.title = `Deleting the RemoteConfig configmap in namespace ${namespace}`; await Promise.all( contexts.map(async (context): Promise => { await this.k8Factory.getK8(context).configMaps().delete(namespace, constants.SOLO_REMOTE_CONFIGMAP_NAME); }), ); if (deletePvcs) { task.title = `Deleting PVCs in namespace ${namespace}`; await this.deletePvcs(namespace, contexts); } if (deleteSecrets) { task.title = `Deleting Secrets in namespace ${namespace}`; await this.deleteSecrets(namespace, contexts); } } } private async logDestroyResults(title: string, results: PromiseSettledResult[]): Promise { const failures: PromiseRejectedResult[] = results.filter( (result): result is PromiseRejectedResult => result.status === 'rejected', ); if (failures.length === 0) { return; } for (const failure of failures) { this.logger.warn(`${title} failed; continuing destroy`, failure.reason); } } private async deleteSecrets(namespace: NamespaceName, contexts: Context[]): Promise { const secretsData: Array<{secret: string; context: Context}> = []; for (const context of contexts) { const secrets: Secret[] = await this.k8Factory.getK8(context).secrets().list(namespace); for (const secret of secrets) { secretsData.push({secret: secret.name, context: context}); } } const promises: Promise[] = secretsData.map(async ({context, secret}): Promise => { await this.k8Factory.getK8(context).secrets().delete(namespace, secret); }); await Promise.all(promises); } private async deletePvcs(namespace: NamespaceName, contexts: Context[]): Promise { const pvcsData: Array<{pvc: string; context: Context}> = []; for (const context of contexts) { const pvcs: string[] = await this.k8Factory.getK8(context).pvcs().list(namespace, []); for (const pvc of pvcs) { pvcsData.push({pvc, context}); } } const promises: Promise[] = pvcsData.map(async ({context, pvc}): Promise => { await this.k8Factory .getK8(context) .pvcs() .delete(PvcReference.of(namespace, PvcName.of(pvc))) .catch(); }); await Promise.all(promises); } private async crdExists(context: string, crdName: string): Promise { return await this.k8Factory.getK8(context).crds().ifExists(crdName); } /** * Ensure the PodLogs CRD from Grafana Alloy is installed */ private async ensurePodLogsCrd({contexts}: NetworkDeployConfigClass): Promise { const PODLOGS_CRD: string = 'podlogs.monitoring.grafana.com'; const CRD_FILE_PATH: string = 'operations/helm/charts/alloy/charts/crds/crds/monitoring.grafana.com_podlogs.yaml'; // Use the GitHub Contents API (api.github.com) instead of raw.githubusercontent.com. // // Why: raw.githubusercontent.com is served by the Fastly CDN and its rate-limiting // behaviour for unauthenticated requests is undocumented — adding a token there may // have no effect. The Contents API, on the other hand, is part of the GitHub REST API // (api.github.com) whose rate limits are well-documented: 60 req/hour unauthenticated // vs 5 000 req/hour when a valid token is supplied. Since GITHUB_TOKEN is injected // automatically into every GitHub Actions job, CI runs always get the higher limit, // making 429s far less likely in the first place. const CRD_URL: string = `https://api.github.com/repos/grafana/alloy/contents/${CRD_FILE_PATH}` + `?ref=${versions.GRAFANA_PODLOGS_CRD_VERSION}`; const CRD_RAW_URL: string = `https://raw.githubusercontent.com/grafana/alloy/${versions.GRAFANA_PODLOGS_CRD_VERSION}/${CRD_FILE_PATH}`; const LOCAL_CRD_FILE: string = PathEx.join( constants.ROOT_DIR, 'resources', 'crds', `monitoring.grafana.com_podlogs-${versions.GRAFANA_PODLOGS_CRD_VERSION}.yaml`, ); for (const context of contexts as string[]) { const exists: boolean = await this.crdExists(context, PODLOGS_CRD); if (exists) { this.logger.debug(`CRD ${PODLOGS_CRD} already exists in context ${context}`); continue; } this.logger.info(`Installing missing CRD ${PODLOGS_CRD} from ${CRD_URL} in context ${context}...`); const temporaryFile: string = PathEx.join( constants.SOLO_CACHE_DIR, `podlogs-crd-${versions.GRAFANA_PODLOGS_CRD_VERSION}.yaml`, ); // Download and cache the CRD YAML. The cache file is keyed by the CRD version so // it is automatically invalidated when GRAFANA_PODLOGS_CRD_VERSION is bumped. // SOLO_CACHE_DIR persists across job steps (unlike os.tmpdir() which is ephemeral), // ensuring we only make one network request per job even if multiple contexts need // the CRD installed. if (!fs.existsSync(temporaryFile)) { // Prefer a vendored CRD file to avoid external network/rate-limit failures in CI. if (fs.existsSync(LOCAL_CRD_FILE)) { fs.copyFileSync(LOCAL_CRD_FILE, temporaryFile); this.logger.debug(`Using local PodLogs CRD file: ${LOCAL_CRD_FILE}`); } else { const downloadErrors: string[] = []; // Attempt #1: GitHub Contents API. // The response is a JSON envelope with base64 content. const apiHeaders: Record = {Accept: 'application/vnd.github.v3+json'}; if (process.env.GITHUB_TOKEN) { apiHeaders['Authorization'] = `Bearer ${process.env.GITHUB_TOKEN}`; } const apiResponse: Response = await fetch(CRD_URL, {headers: apiHeaders}); if (apiResponse.ok) { const json: {content: string} = (await apiResponse.json()) as {content: string}; const yamlContent: string = Buffer.from(json.content.replaceAll(/\s/g, ''), 'base64').toString('utf8'); fs.writeFileSync(temporaryFile, yamlContent, 'utf8'); } else { const apiError: string = `${apiResponse.status} ${apiResponse.statusText}`.trim(); downloadErrors.push(`GitHub API: ${apiError}`); this.logger.warn(`Failed to download PodLogs CRD from GitHub API (${apiError}), trying raw URL fallback.`); // Attempt #2: raw.githubusercontent.com fallback. const rawHeaders: Record = {}; if (process.env.GITHUB_TOKEN) { rawHeaders['Authorization'] = `Bearer ${process.env.GITHUB_TOKEN}`; } const rawResponse: Response = await fetch(CRD_RAW_URL, {headers: rawHeaders}); if (!rawResponse.ok) { const rawError: string = `${rawResponse.status} ${rawResponse.statusText}`.trim(); downloadErrors.push(`Raw URL: ${rawError}`); throw new Error(`Failed to download CRD YAML (${downloadErrors.join('; ')})`); } const yamlContent: string = await rawResponse.text(); fs.writeFileSync(temporaryFile, yamlContent, 'utf8'); } } } await this.k8Factory.getK8(context).manifests().applyManifest(temporaryFile); } } /** * Ensure all Prometheus Operator CRDs exist; install chart only if needed. * If all CRDs are already present or monitoring support is disabled, skip installation. */ /** Ensure Prometheus Operator CRDs are present; install missing ones via the chart */ private async ensurePrometheusOperatorCrds({ clusterRefs, namespace, deployment, }: NetworkDeployConfigClass): Promise { const CRDS: {key: string; crd: string}[] = [ {key: 'alertmanagerconfigs', crd: 'alertmanagerconfigs.monitoring.coreos.com'}, {key: 'alertmanagers', crd: 'alertmanagers.monitoring.coreos.com'}, {key: 'podmonitors', crd: 'podmonitors.monitoring.coreos.com'}, {key: 'probes', crd: 'probes.monitoring.coreos.com'}, {key: 'prometheusagents', crd: 'prometheusagents.monitoring.coreos.com'}, {key: 'prometheuses', crd: 'prometheuses.monitoring.coreos.com'}, {key: 'prometheusrules', crd: 'prometheusrules.monitoring.coreos.com'}, {key: 'scrapeconfigs', crd: 'scrapeconfigs.monitoring.coreos.com'}, {key: 'servicemonitors', crd: 'servicemonitors.monitoring.coreos.com'}, {key: 'thanosrulers', crd: 'thanosrulers.monitoring.coreos.com'}, ]; // eslint-disable-next-line @typescript-eslint/no-unused-vars for (const [_, context] of clusterRefs) { let valuesArgument: string = ''; let missingCount: number = 0; for (const {key, crd} of CRDS) { const exists: boolean = await this.crdExists(context, crd); if (exists) { valuesArgument += ` --set "${key}.enabled=false"`; } else { missingCount++; } } if (missingCount === 0) { this.logger.info(`All Prometheus Operator CRDs already present in context ${context}; skipping installation.`); continue; } const setupMap: Map = new Map([ [constants.PROMETHEUS_OPERATOR_CRDS_RELEASE_NAME, constants.PROMETHEUS_OPERATOR_CRDS_CHART_URL], ]); await this.chartManager.setup(setupMap); await this.chartManager.install( namespace, constants.PROMETHEUS_OPERATOR_CRDS_RELEASE_NAME, constants.PROMETHEUS_OPERATOR_CRDS_CHART, constants.PROMETHEUS_OPERATOR_CRDS_CHART, versions.PROMETHEUS_OPERATOR_CRDS_VERSION, valuesArgument, context, ); this.eventBus.emit(new NetworkDeployedEvent(deployment)); showVersionBanner( this.logger, constants.PROMETHEUS_OPERATOR_CRDS_CHART, versions.PROMETHEUS_OPERATOR_CRDS_VERSION, ); } } /** * Patch the ServiceMonitor created by the solo-deployment helm chart so that it is discovered * by the kube-prometheus-stack Prometheus operator and targets the correct consensus node services. * * Two fixes are applied via a merge patch: * 1. Adds the `release: ` label so the Prometheus instance from * kube-prometheus-stack (which selects ServiceMonitors by `release` label) can discover it. * 2. Corrects `spec.selector.matchLabels` to `solo.hedera.com/type: network-node-svc` so the * ServiceMonitor targets the non-headless consensus-node services (which expose the prometheus * metrics port) rather than the hard-coded `network-node` value in the helm chart template. */ private async patchServiceMonitorForPrometheus(namespace: NamespaceName, context: Context): Promise { const patch: object = { apiVersion: 'monitoring.coreos.com/v1', kind: 'ServiceMonitor', metadata: { name: constants.SOLO_SERVICE_MONITOR_NAME, namespace: namespace.name, labels: { release: constants.PROMETHEUS_RELEASE_NAME, }, }, spec: { selector: { matchLabels: { 'solo.hedera.com/type': 'network-node-svc', }, }, }, }; await this.k8Factory.getK8(context).manifests().patchObject(patch); this.logger.debug( `Patched ServiceMonitor '${constants.SOLO_SERVICE_MONITOR_NAME}' in namespace '${namespace.name}': ` + `added label release=${constants.PROMETHEUS_RELEASE_NAME} and fixed selector to network-node-svc`, ); } /** Run helm install and deploy network components */ public async deploy(argv: ArgvStruct): Promise { let lease: Lock; const tasks: SoloListr = this.taskList.newTaskList( [ { title: 'Initialize', task: async (context_, task): Promise> => { this.configManager.update(argv); await this.localConfig.load(); await this.remoteConfig.loadAndValidate(argv, true, true); if (!this.oneShotState.isActive()) { lease = await this.leaseManager.create(); } const releaseTag: SemanticVersion = new SemanticVersion( this.configManager.getFlag(flags.releaseTag), ); if ( this.remoteConfig.configuration.versions.consensusNode.toString() === '0.0.0' || !new SemanticVersion(this.remoteConfig.configuration.versions.consensusNode).equals(releaseTag) ) { // if is possible block node deployed before consensus node, then use release tag as fallback this.remoteConfig.configuration.versions.consensusNode = releaseTag; await this.remoteConfig.persist(); } const currentVersion: SemanticVersion = new SemanticVersion( this.remoteConfig.configuration.versions.consensusNode.toString(), ); let tssEnabled: boolean = this.configManager.getFlag(flags.tssEnabled); const minimumVersion: SemanticVersion = new SemanticVersion( versions.MINIMUM_HIERO_PLATFORM_VERSION_FOR_TSS, ); // if platform version is insufficient for tss, disable it if (tssEnabled && new SemanticVersion(currentVersion).lessThan(minimumVersion)) { tssEnabled = false; } const wrapsEnabled: boolean = this.configManager.getFlag(flags.wrapsEnabled); this.remoteConfig.configuration.state.wrapsEnabled = wrapsEnabled; if (wrapsEnabled && new SemanticVersion(currentVersion).lessThan(minimumVersion)) { this.logger.showUser( `Consensus node version ${currentVersion} does not support TSS or Wraps. Please upgrade to version ${minimumVersion} or later to enable these features.`, ); throw new SoloError( `"--wraps" requires consensus node >= ${versions.MINIMUM_HIERO_PLATFORM_VERSION_FOR_TSS}`, ); } this.remoteConfig.configuration.state.tssEnabled = tssEnabled; await this.remoteConfig.persist(); context_.config = await this.prepareConfig(task, argv); if (!this.oneShotState.isActive()) { return ListrLock.newAcquireLockTask(lease, task); } return ListrLock.newSkippedLockTask(task); }, }, { title: 'Copy gRPC TLS Certificates', task: ( {config: {grpcTlsCertificatePath, grpcWebTlsCertificatePath, grpcTlsKeyPath, grpcWebTlsKeyPath}}, parentTask, ): SoloListr => this.certificateManager.buildCopyTlsCertificatesTasks( parentTask, grpcTlsCertificatePath, grpcWebTlsCertificatePath, grpcTlsKeyPath, grpcWebTlsKeyPath, ), skip: ({config: {grpcTlsCertificatePath, grpcWebTlsCertificatePath}}): boolean => !grpcTlsCertificatePath && !grpcWebTlsCertificatePath, }, { title: 'Prepare staging directory', task: (_, parentTask): SoloListr => { return parentTask.newListr( [ { title: 'Copy Gossip keys to staging', task: ({config: {keysDir, stagingKeysDir, nodeAliases}}): void => { this.keyManager.copyGossipKeysToStaging(keysDir, stagingKeysDir, nodeAliases); }, }, { title: 'Copy gRPC TLS keys to staging', task: ({config: {nodeAliases, keysDir, stagingKeysDir}}): void => { for (const nodeAlias of nodeAliases) { const tlsKeyFiles: PrivateKeyAndCertificateObject = this.keyManager.prepareTlsKeyFilePaths( nodeAlias, keysDir, ); this.keyManager.copyNodeKeysToStaging(tlsKeyFiles, stagingKeysDir); } }, }, ], constants.LISTR_DEFAULT_OPTIONS.DEFAULT, ); }, }, { title: 'Copy node keys to secrets', task: ({config: {stagingDir, consensusNodes, contexts}}, parentTask): SoloListr => { // set up the subtasks return parentTask.newListr( this.platformInstaller.copyNodeKeys(stagingDir, consensusNodes, contexts), constants.LISTR_DEFAULT_OPTIONS.WITH_CONCURRENCY, ); }, }, { title: 'Install monitoring CRDs', skip: ({config: {enableMonitoringSupport}}): boolean => !enableMonitoringSupport, task: (_, task): SoloListr => { const tasks: SoloListrTask[] = [ { title: 'Pod Logs CRDs', task: async ({config}): Promise => await this.ensurePodLogsCrd(config), }, { title: 'Prometheus Operator CRDs', task: async ({config}): Promise => await this.ensurePrometheusOperatorCrds(config), }, ]; return task.newListr(tasks, {concurrent: true, rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION}); }, }, { title: `Install chart '${constants.SOLO_DEPLOYMENT_CHART}'`, task: async ({config}): Promise => { const {namespace, clusterRefs, valuesArgMap, chartDirectory} = config; for (const [clusterReference] of clusterRefs) { const isInstalled: boolean = await this.chartManager.isChartInstalled( namespace, constants.SOLO_DEPLOYMENT_CHART, clusterRefs.get(clusterReference), ); if (isInstalled) { await this.chartManager.uninstall( namespace, constants.SOLO_DEPLOYMENT_CHART, clusterRefs.get(clusterReference), ); config.isUpgrade = true; } config.soloChartVersion = SemanticVersion.getValidSemanticVersion( config.soloChartVersion, false, 'Solo chart version', ); await this.chartManager.upgrade( namespace, constants.SOLO_DEPLOYMENT_CHART, constants.SOLO_DEPLOYMENT_CHART, chartDirectory || constants.SOLO_TESTING_CHART_URL, config.soloChartVersion, valuesArgMap[clusterReference], clusterRefs.get(clusterReference), ); showVersionBanner(this.logger, constants.SOLO_DEPLOYMENT_CHART, config.soloChartVersion); } }, }, { title: 'Patch ServiceMonitor for Prometheus discovery', skip: ({config: {enableMonitoringSupport}}): boolean => !enableMonitoringSupport, task: async ({config: {namespace, clusterRefs}}): Promise => { for (const [, context] of clusterRefs) { await this.patchServiceMonitorForPrometheus(namespace, context); } }, }, // TODO: Move the check for load balancer logic to a utility method or class { title: 'Check for load balancer', skip: ({config: {loadBalancerEnabled}}): boolean => loadBalancerEnabled === false, task: ({config: {consensusNodes, namespace}}, task): SoloListr => { const subTasks: SoloListrTask[] = []; //Add check for network node service to be created and load balancer to be assigned (if load balancer is enabled) for (const consensusNode of consensusNodes) { subTasks.push({ title: `Load balancer is assigned for: ${chalk.yellow(consensusNode.name)}, cluster: ${chalk.yellow(consensusNode.cluster)}`, task: async (): Promise => { let attempts: number = 0; let svc: Service[]; while (attempts < constants.LOAD_BALANCER_CHECK_MAX_ATTEMPTS) { svc = await this.k8Factory .getK8(consensusNode.context) .services() .list(namespace, Templates.renderNodeSvcLabelsFromNodeId(consensusNode.nodeId)); if (svc && svc.length > 0 && svc[0].status?.loadBalancer?.ingress?.length > 0) { let shouldContinue: boolean = false; for (let index: number = 0; index < svc[0].status.loadBalancer.ingress.length; index++) { const ingress: LoadBalancerIngress = svc[0].status.loadBalancer.ingress[index]; if (!ingress.hostname && !ingress.ip) { shouldContinue = true; // try again if there is neither a hostname nor an ip break; } } if (shouldContinue) { continue; } return; } attempts++; await sleep(Duration.ofSeconds(constants.LOAD_BALANCER_CHECK_DELAY_SECS)); } throw new SoloError('Load balancer not found'); }, }); } // set up the sub-tasks return task.newListr(subTasks, { concurrent: true, rendererOptions: { collapseSubtasks: false, }, }); }, }, // TODO: find a better solution to avoid the need to redeploy the chart { title: 'Redeploy chart with external IP address config', skip: ({config: {loadBalancerEnabled}}): boolean => loadBalancerEnabled === false, task: async ({config}, task): Promise> => { const {namespace, chartDirectory, soloChartVersion, clusterRefs} = config; // Update the valuesArgMap with the external IP addresses // This regenerates the config.txt and genesis-network.json files with the external IP addresses config.valuesArgMap = await this.prepareValuesArgMap(config); // Perform a helm upgrade for each cluster const subTasks: SoloListrTask[] = []; for (const [clusterReference] of clusterRefs) { subTasks.push({ title: `Upgrade chart for cluster: ${chalk.yellow(clusterReference)}`, task: async (): Promise => { await this.chartManager.upgrade( namespace, constants.SOLO_DEPLOYMENT_CHART, constants.SOLO_DEPLOYMENT_CHART, chartDirectory || constants.SOLO_TESTING_CHART_URL, soloChartVersion, config.valuesArgMap[clusterReference], clusterRefs.get(clusterReference), ); showVersionBanner(this.logger, constants.SOLO_DEPLOYMENT_CHART, soloChartVersion, 'Upgraded'); // TODO: Remove this code now that we have made the config dynamic and can update it without redeploying const k8: K8 = this.k8Factory.getK8(clusterRefs.get(clusterReference)); const pods: Pod[] = await k8.pods().list(namespace, ['solo.hedera.com/type=network-node']); for (const pod of pods) { await k8.pods().readByReference(pod.podReference).killPod(); } }, }); } // set up the sub-tasks return task.newListr(subTasks, { concurrent: true, rendererOptions: { collapseSubtasks: false, }, }); }, }, this.waitForNetworkPods(), { title: 'Check proxy pods are running', task: (context_, task): SoloListr => { const subTasks: SoloListrTask[] = []; const config: NetworkDeployConfigClass = context_.config; // HAProxy for (const consensusNode of config.consensusNodes) { subTasks.push({ title: `Check HAProxy for: ${chalk.yellow(consensusNode.name)}, cluster: ${chalk.yellow(consensusNode.cluster)}`, task: async (): Promise => await this.k8Factory .getK8(consensusNode.context) .pods() .waitForRunningPhase( config.namespace, ['solo.hedera.com/type=haproxy'], constants.PODS_RUNNING_MAX_ATTEMPTS, constants.PODS_RUNNING_DELAY, ), }); } // Envoy Proxy for (const consensusNode of config.consensusNodes) { subTasks.push({ title: `Check Envoy Proxy for: ${chalk.yellow(consensusNode.name)}, cluster: ${chalk.yellow(consensusNode.cluster)}`, task: async (): Promise => await this.k8Factory .getK8(consensusNode.context) .pods() .waitForRunningPhase( context_.config.namespace, ['solo.hedera.com/type=envoy-proxy'], constants.PODS_RUNNING_MAX_ATTEMPTS, constants.PODS_RUNNING_DELAY, ), }); } // set up the sub-tasks return task.newListr(subTasks, { concurrent: true, rendererOptions: { collapseSubtasks: false, }, }); }, }, { title: 'Check auxiliary pods are ready', task: (_, task): SoloListr => { const subTasks: SoloListrTask[] = [ { title: 'Check MinIO', task: async ({config: {contexts, namespace}}): Promise => { for (const context of contexts) { await this.k8Factory .getK8(context) .pods() .waitForReadyStatus( namespace, ['v1.min.io/tenant=minio'], constants.PODS_RUNNING_MAX_ATTEMPTS, constants.PODS_RUNNING_DELAY, ); } }, // skip if only cloud storage is/are used skip: ({config: {storageType}}): boolean => storageType === constants.StorageType.GCS_ONLY || storageType === constants.StorageType.AWS_ONLY || storageType === constants.StorageType.AWS_AND_GCS, }, ]; // minio // set up the subtasks return task.newListr(subTasks, { concurrent: false, // no need to run concurrently since if one node is up, the rest should be up by then rendererOptions: { collapseSubtasks: false, }, }); }, }, this.addNodesAndProxies(), { title: 'Copy wraps lib into consensus node', skip: (): boolean => !this.remoteConfig.configuration.state.wrapsEnabled, task: async ({config}): Promise => { const wraps: Wraps = this.soloConfig.tss.wraps; const extractedDirectory: string = PathEx.join(constants.SOLO_CACHE_DIR, wraps.directoryName); if (config.wrapsKeyPath) { // Use user-provided local directory containing WRAPs proving key files if (!fs.existsSync(config.wrapsKeyPath)) { throw new SoloError(`WRAPs key path does not exist: ${config.wrapsKeyPath}`); } this.logger.info(`Using WRAPs proving key files from: ${config.wrapsKeyPath}`); // Copy allowed .bin files from user path into the cache directory if (!fs.existsSync(extractedDirectory)) { fs.mkdirSync(extractedDirectory, {recursive: true}); } const allowedFiles: Set = wraps.allowedKeyFileSet; for (const file of fs.readdirSync(config.wrapsKeyPath)) { if (allowedFiles.has(file)) { fs.copyFileSync(PathEx.join(config.wrapsKeyPath, file), PathEx.join(extractedDirectory, file)); } } } else { if (fs.existsSync(extractedDirectory)) { this.logger.debug('Wraps library already installed'); } else { await this.downloader.fetchPackage( wraps.libraryDownloadUrl, 'unusued', constants.SOLO_CACHE_DIR, false, '', false, ); const tarFilePath: string = PathEx.join(constants.SOLO_CACHE_DIR, `${wraps.directoryName}.tar.gz`); // Create extraction dir fs.mkdirSync(extractedDirectory); // Extract wraps-v0.2.0.tar.gz -> wraps-v0.2.0 this.zippy.untar(tarFilePath, extractedDirectory); } // Having any files except for those inside the folder causes an error in CN const allowedFiles: Set = wraps.allowedKeyFileSet; for (const file of fs.readdirSync(extractedDirectory)) { if (!allowedFiles.has(file)) { const filePath: string = PathEx.join(extractedDirectory, file); fs.unlinkSync(filePath); // delete unwanted file } } } for (const consensusNode of config.consensusNodes) { const rootContainer: Container = await new K8Helper(consensusNode.context).getConsensusNodeRootContainer( config.namespace, consensusNode.name, ); await rootContainer.copyTo(extractedDirectory, constants.HEDERA_HAPI_PATH); } }, }, { title: `Copy ${constants.BLOCK_NODES_JSON_FILE}`, skip: ({config: {blockNodeComponents}}): boolean => blockNodeComponents.length === 0, task: async ({config: {consensusNodes}}): Promise => { try { for (const consensusNode of consensusNodes) { await createAndCopyBlockNodeJsonFileForConsensusNode(consensusNode, this.logger, this.k8Factory); } } catch (error) { throw new SoloError(`Failed while creating block-nodes configuration: ${error.message}`, error); } }, }, { title: 'Copy JFR config file to nodes', skip: ({config: {javaFlightRecorderConfiguration}}): boolean => javaFlightRecorderConfiguration.length === 0, task: async ( {config: {consensusNodes, javaFlightRecorderConfiguration}}, task, ): Promise> => { const subTasks: SoloListrTask[] = []; for (const consensusNode of consensusNodes) { subTasks.push({ title: `Copy config JFR file to node: ${chalk.yellow(consensusNode.name)}, cluster: ${chalk.yellow(consensusNode.context)}`, task: async (): Promise => { try { const container: Container = await new K8Helper( consensusNode.context, ).getConsensusNodeRootContainer(NamespaceName.of(consensusNode.namespace), consensusNode.name); await container.copyTo( javaFlightRecorderConfiguration, `${constants.HEDERA_HAPI_PATH}/data/config`, ); } catch (error) { throw new SoloError(`Failed while creating block-nodes configuration: ${error.message}`, error); } }, }); } return task.newListr(subTasks, { concurrent: true, rendererOptions: { collapseSubtasks: false, }, }); }, }, ], constants.LISTR_DEFAULT_OPTIONS.DEFAULT, undefined, 'consensus network deploy', ); if (tasks.isRoot()) { try { await tasks.run(); } catch (error) { throw new SoloError(`Error installing chart ${constants.SOLO_DEPLOYMENT_CHART}`, error); } finally { if (lease && !this.oneShotState.isActive()) { await lease.release(); } } } else { this.taskList.registerCloseFunction(async (): Promise => { if (!this.oneShotState.isActive()) { await lease?.release(); } }); } return true; } public async destroy(argv: ArgvStruct): Promise { let lease: Lock; let networkDestroySuccess: boolean = true; const tasks: SoloListr = this.taskList.newTaskList( [ { title: 'Initialize', task: async (context_, task): Promise> => { await this.localConfig.load(); const remoteConfigLoaded: boolean = await this.loadRemoteConfigOrWarn(argv); if (!this.oneShotState.isActive()) { lease = await this.leaseManager.create(); } if (!argv.force) { const confirmResult: boolean = await task.prompt(ListrInquirerPromptAdapter).run(confirmPrompt, { default: false, message: 'Are you sure you would like to destroy the network components?', }); if (!confirmResult) { throw new UserBreak('Aborted application by user prompt'); } } this.configManager.update(argv); await this.configManager.executePrompt(task, [flags.deletePvcs, flags.deleteSecrets]); context_.config = { deletePvcs: this.configManager.getFlag(flags.deletePvcs), deleteSecrets: this.configManager.getFlag(flags.deleteSecrets), deployment: this.configManager.getFlag(flags.deployment), namespace: await resolveNamespaceFromDeployment(this.localConfig, this.configManager, task), enableTimeout: this.configManager.getFlag(flags.enableTimeout), force: this.configManager.getFlag(flags.force), contexts: remoteConfigLoaded ? this.remoteConfig.getContexts() : [...this.localConfig.configuration.clusterRefs.values()].map( (context): Context => context.toString(), ), }; if (!this.oneShotState.isActive()) { return ListrLock.newAcquireLockTask(lease, task); } return ListrLock.newSkippedLockTask(task); }, }, { title: 'Running sub-tasks to destroy network', task: async ( {config: {enableTimeout, deletePvcs, deleteSecrets, namespace, contexts}}, task, ): Promise => { if (!enableTimeout) { await this.destroyTask(task, namespace, deletePvcs, deleteSecrets, contexts); return; } const onTimeoutCallback: NodeJS.Timeout = setTimeout(async (): Promise => { const message: string = `\n\nUnable to finish consensus network destroy in ${constants.NETWORK_DESTROY_WAIT_TIMEOUT} seconds\n\n`; this.logger.error(message); this.logger.showUser(chalk.red(message)); networkDestroySuccess = false; if (!deleteSecrets || !deletePvcs) { await this.remoteConfig.deleteComponents(); return; } for (const context of contexts) { await this.k8Factory.getK8(context).namespaces().delete(namespace); } }, constants.NETWORK_DESTROY_WAIT_TIMEOUT * 1000); await this.destroyTask(task, namespace, deletePvcs, deleteSecrets, contexts); clearTimeout(onTimeoutCallback); }, }, ], constants.LISTR_DEFAULT_OPTIONS.DEFAULT, undefined, 'consensus network destroy', ); if (tasks.isRoot()) { try { await tasks.run(); } catch (error) { throw new SoloError('Error destroying network', error); } finally { // If the namespace is deleted, the lease can't be released if (!this.oneShotState.isActive()) { await lease?.release().catch(); } } } else { this.taskList.registerCloseFunction(async (): Promise => { if (!this.oneShotState.isActive()) { await lease?.release(); } }); } return networkDestroySuccess; } /** Adds the consensus node, envoy and haproxy components to remote config. */ public addNodesAndProxies(): SoloListrTask { return { title: 'Add node and proxies to remote config', skip: (): boolean => !this.remoteConfig.isLoaded(), task: async ({config: {consensusNodes, namespace, isUpgrade, releaseTag}}): Promise => { for (const consensusNode of consensusNodes) { const componentId: ComponentId = Templates.renderComponentIdFromNodeAlias(consensusNode.name); const clusterReference: ClusterReferenceName = consensusNode.cluster; this.remoteConfig.configuration.components.changeNodePhase(componentId, DeploymentPhase.REQUESTED); if (isUpgrade) { this.logger.info('Do not add envoy and haproxy components again during upgrade'); } else { // do not add new envoy or haproxy components if they already exist this.remoteConfig.configuration.components.addNewComponent( this.componentFactory.createNewEnvoyProxyComponent(clusterReference, namespace), ComponentTypes.EnvoyProxy, ); this.remoteConfig.configuration.components.addNewComponent( this.componentFactory.createNewHaProxyComponent(clusterReference, namespace), ComponentTypes.HaProxy, ); } } if (releaseTag) { // update the solo chart version to match the deployed version this.remoteConfig.updateComponentVersion( ComponentTypes.ConsensusNode, new SemanticVersion(releaseTag), ); } await this.remoteConfig.persist(); }, }; } private getBlockNodes(): BlockNodeStateSchema[] { return this.remoteConfig.configuration.components.state.blockNodes; } public async close(): Promise {} // no-op }