diff --git a/package-lock.json b/package-lock.json index 5e5dd0b..d84dcef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@types/shortid": "0.0.31", "@types/sshpk": "^1.17.3", "bee-queue": "^1.6.0", + "consul": "^2.0.1", "dotenv": "^16.3.1", "dots-wrapper": "^3.11.3", "envalid": "^8.0.0", @@ -1555,6 +1556,29 @@ "url": "https://github.com/open-cli-tools/concurrently?sponsor=1" } }, + "node_modules/consul": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/consul/-/consul-2.0.1.tgz", + "integrity": "sha512-91ExUUelOJ1yyB0etYAR0w1p6Ues1VosEyBVxPcWJdnQDTKqAEFzL0MHfOqZWYI2d4HZ4FgotHZkAPW2A/xahA==", + "license": "MIT", + "dependencies": { + "papi": "^1.1.0", + "uuid": "^10.0.0" + } + }, + "node_modules/consul/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -4769,6 +4793,15 @@ "node": ">=8" } }, + "node_modules/papi": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/papi/-/papi-1.1.2.tgz", + "integrity": "sha512-cwM6pPpfAYgPe3EQi23SmB5J5s4XFS9lou9z63I5BbnMGmFaR8LAKvKboW7n1IUAKj76OtnyK0YU16JjnZrqVg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7431,6 +7464,22 @@ "yargs": "^17.7.2" } }, + "consul": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/consul/-/consul-2.0.1.tgz", + "integrity": "sha512-91ExUUelOJ1yyB0etYAR0w1p6Ues1VosEyBVxPcWJdnQDTKqAEFzL0MHfOqZWYI2d4HZ4FgotHZkAPW2A/xahA==", + "requires": { + "papi": "^1.1.0", + "uuid": "^10.0.0" + }, + "dependencies": { + "uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==" + } + } + }, "content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -9977,6 +10026,11 @@ "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-2.0.1.tgz", "integrity": "sha512-vpm09aKwq6H9phqRQzecoDpD8TmVyGw70qmWlyq5onxY7tqyTTFVvxMykxQSQKILBSFlbXpypIw2T1Ml7+DDtw==" }, + "papi": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/papi/-/papi-1.1.2.tgz", + "integrity": "sha512-cwM6pPpfAYgPe3EQi23SmB5J5s4XFS9lou9z63I5BbnMGmFaR8LAKvKboW7n1IUAKj76OtnyK0YU16JjnZrqVg==" + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", diff --git a/package.json b/package.json index 4c8e1cb..ad116e0 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "@types/shortid": "0.0.31", "@types/sshpk": "^1.17.3", "bee-queue": "^1.6.0", + "consul": "^2.0.1", "dotenv": "^16.3.1", "dots-wrapper": "^3.11.3", "envalid": "^8.0.0", diff --git a/src/consul.ts b/src/consul.ts new file mode 100644 index 0000000..6b20dd2 --- /dev/null +++ b/src/consul.ts @@ -0,0 +1,102 @@ +import Consul from 'consul'; +import { Context } from './context'; +import { GetItem } from 'consul/lib/kv'; +import InstanceStore, { InstanceGroup } from './instance_store'; + +// implments the InstanceStore interface using consul K/V API calls +// uses the got library to make HTTP requests + +export interface ConsulOptions { + host: string; + port: number; + secure: boolean; + groupsPrefix?: string; +} + +export default class ConsulStore { + private client: Consul; + private groupsPrefix = 'autoscaler/groups/'; + + constructor(options: ConsulOptions) { + this.client = new Consul(options); + if (options.groupsPrefix) { + this.groupsPrefix = options.groupsPrefix; + } + } + + async getInstanceGroup(ctx: Context, group: string): Promise { + try { + const { Value } = await this.fetch(ctx, `${this.groupsPrefix}${group}`); + return JSON.parse(Value); + } catch (err) { + ctx.logger.error(`Failed to get instance group from consul: ${err}`, { err }); + throw err; + } + } + + async getAllInstanceGroups(ctx: Context): Promise { + try { + const keys = await this.fetchInstanceGroups(ctx); + const groups = await Promise.all(keys.map((key) => this.getInstanceGroup(ctx, key))); + return groups; + } catch (err) { + ctx.logger.error(`Failed to get all instance groups from consul: ${err}`, { err }); + throw err; + } + } + + async fetchInstanceGroups(ctx: Context): Promise { + ctx.logger.debug('fetching consul k/v keys'); + const res = await this.client.kv.get({ key: this.groupsPrefix, recurse: true }); + ctx.logger.debug('received consul k/v keys', { res }); + if (!res) { + return []; + } + return Object.entries(res).map(([_k, v]) => v.Key.replace(this.groupsPrefix, '')); + } + + async upsertInstanceGroup(ctx: Context, group: InstanceGroup): Promise { + try { + await this.write(ctx, `${this.groupsPrefix}${group.name}`, JSON.stringify(group)); + return true; + } catch (err) { + ctx.logger.error(`Failed to upsert instance group into consul: ${err}`, { group: group.name, err }); + return false; + } + } + + async deleteInstanceGroup(ctx: Context, group: string): Promise { + try { + await this.delete(`${this.groupsPrefix}${group}`); + return true; + } catch (err) { + ctx.logger.error(`Failed to delete instance group from consul: ${err}`, { group, err }); + return false; + } + } + + async fetch(ctx: Context, key: string): Promise { + ctx.logger.debug(`reading consul k/v key`, { key }); + const v = await this.client.kv.get(key); + ctx.logger.debug(`received consul k/v item`, { v }); + return v; + } + + async write(ctx: Context, key: string, value: string): Promise { + try { + const res = await this.client.kv.set(key, value); + if (!res) { + ctx.logger.error(`Failed to write to consul`); + } + return res; + } catch (err) { + ctx.logger.error(`Failed to write to consul: ${err}`, { err }); + return false; + } + } + + async delete(key: string): Promise { + await this.client.kv.del(key); + return true; + } +} diff --git a/src/test/consul.ts b/src/test/consul.ts new file mode 100644 index 0000000..c41132f --- /dev/null +++ b/src/test/consul.ts @@ -0,0 +1,65 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck +import AutoscalerLogger from '../logger'; +import assert from 'node:assert'; +import test, { afterEach, describe, mock } from 'node:test'; + +import ConsulClient, { ConsulOptions } from '../consul'; + +const asLogger = new AutoscalerLogger({ logLevel: 'debug' }); +const logger = asLogger.createLogger('debug'); + +const ctx = { logger }; +ctx.logger.debug = mock.fn(); +ctx.logger.error = mock.fn(); + +const options = { + host: 'localhost', + port: 8500, + secure: false, + groupsPrefix: '_test/autoscaler/groups/', +}; +const client = new ConsulClient(options); + +const group = { + name: 'test', + type: 'test', + region: 'test', + environment: 'test', + enableScheduler: true, + tags: { + test: 'test', + }, +}; + +describe('ConsulClient', () => { + afterEach(() => { + mock.restoreAll(); + }); + + describe('testListInstanceGroups', () => { + test('will list all instance groups', async () => { + const res = await client.fetchInstanceGroups(ctx); + assert.strictEqual(res.length, 0); + }); + + test('will upsert a test group', async () => { + const res = await client.upsertInstanceGroup(ctx, group); + assert.strictEqual(res, true); + }); + + test('will find upserted group when listing all instance groups', async () => { + const res = await client.fetchInstanceGroups(ctx); + assert.strictEqual(res.length, 1); + assert.strictEqual(res[0], group.name); + + const res2 = await client.getInstanceGroup(ctx, group.name); + assert.deepEqual(res2, group); + }); + + test('will delete upserted test group', async () => { + const res = await client.deleteInstanceGroup(ctx, group.name); + assert.strictEqual(res, true); + }); + }); +});