Skip to content

Commit

Permalink
refactor: replace got with ky (#7479)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlCalzone
AlCalzone authored Dec 10, 2024
1 parent 7c5a4ac commit 205ad8d
Show file tree
Hide file tree
Showing 8 changed files with 90 additions and 243 deletions.
58 changes: 29 additions & 29 deletions packages/config/maintenance/importConfig.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,56 +182,56 @@ function updateNumberOrDefault(

/** Retrieves the list of database IDs from the OpenSmartHouse DB */
async function fetchIDsOH(): Promise<number[]> {
const { got } = await import("got");
const data = (await got.get(ohUrlIDs).json()) as any;
const { default: ky } = await import("ky");
const data = (await ky.get(ohUrlIDs).json()) as any;
return data.devices.map((d: any) => d.id);
}

/** Retrieves the definition for a specific device from the OpenSmartHouse DB */
async function fetchDeviceOH(id: number): Promise<string> {
const { got } = await import("got");
const source = (await got.get(ohUrlDevice(id)).json()) as any;
const { default: ky } = await import("ky");
const source = (await ky.get(ohUrlDevice(id)).json()) as any;
return stringify(source, "\t");
}

/** Retrieves the definition for a specific device from the Z-Wave Alliance DB */
async function fetchDeviceZWA(id: number): Promise<string> {
const { got } = await import("got");
const source = (await got.get(zwaUrlDevice(id)).json()) as any;
const { default: ky } = await import("ky");
const source = (await ky.get(zwaUrlDevice(id)).json()) as any;
return stringify(source, "\t");
}

/** Downloads ozw master archive and store it on `tmpDir` */
async function downloadOZWConfig(): Promise<string> {
console.log("downloading ozw archive...");
const { got } = await import("got");
const { default: ky } = await import("ky");

// create tmp directory if missing
await fs.mkdir(ozwTempDir, { recursive: true });

// this will return a stream in `data` that we pipe into write stream
// to store the file in `tmpDir`
const data = got.stream.get(ozwTarUrl);

return new Promise(async (resolve, reject) => {
let fileHandle: fs.FileHandle | undefined;
try {
// Create a stream to write the file
const fileDest = path.join(ozwTempDir, ozwTarName);
const handle = await fs.open(fileDest, "w");
const stream = handle.createWriteStream();
data.pipe(stream);
let hasError = false;
stream.on("error", (err) => {
hasError = true;
stream.close();
reject(err);
fileHandle = await fs.open(fileDest, "w");
const writable = new WritableStream({
async write(chunk) {
await fileHandle!.write(chunk);
},
});

stream.on("close", () => {
if (!hasError) {
resolve(fileDest);
console.log("ozw archive stored in temporary directory");
}
});
});
// And pipe the response into the stream
const response = await ky.get(ozwTarUrl);
await response.body?.pipeTo(writable);

console.log("ozw archive stored in temporary directory");
return fileDest;
} finally {
await fileHandle?.close();
}
}

/** Extract `config` folder from ozw archive in `tmpDir` */
Expand Down Expand Up @@ -1701,15 +1701,15 @@ async function retrieveZWADeviceIds(
highestDeviceOnly: boolean = true,
manufacturer: number[] = [-1],
): Promise<number[]> {
const { got } = await import("got");
const { default: ky } = await import("ky");
const deviceIdsSet = new Set<number>();

for (const manu of manufacturer) {
let page = 1;
// Page 1
let currentUrl =
`https://products.z-wavealliance.org/search/DoAdvancedSearch?productName=&productIdentifier=&productDescription=&category=-1&brand=${manu}&regionId=-1&order=&page=${page}`;
const firstPage = await got.get(currentUrl).text();
const firstPage = await ky.get(currentUrl).text();
for (const i of firstPage.match(/(?<=productId=).*?(?=[\&\"])/g)!) {
deviceIdsSet.add(i);
}
Expand All @@ -1730,7 +1730,7 @@ async function retrieveZWADeviceIds(
);
currentUrl =
`https://products.z-wavealliance.org/search/DoAdvancedSearch?productName=&productIdentifier=&productDescription=&category=-1&brand=${manu}&regionId=-1&order=&page=${page}`;
const nextPage = await got.get(currentUrl).text();
const nextPage = await ky.get(currentUrl).text();
const nextPageIds = nextPage.match(
/(?<=productId=).*?(?=[\&\"])/g,
)!;
Expand Down Expand Up @@ -1814,8 +1814,8 @@ async function downloadDevicesOH(IDs?: number[]): Promise<void> {
async function downloadManufacturersOH(): Promise<void> {
process.stdout.write("Fetching manufacturers...");

const { got } = await import("got");
const data = await got.get(ohUrlManufacturers).json();
const { default: ky } = await import("ky");
const data = await ky.get(ohUrlManufacturers).json();

// Delete the last line
process.stdout.write("\r\x1b[K");
Expand Down
2 changes: 1 addition & 1 deletion packages/config/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,8 @@
"comment-json": "^4.2.5",
"del-cli": "^6.0.0",
"es-main": "^1.3.0",
"got": "^13.0.0",
"js-levenshtein": "^1.1.6",
"ky": "^1.7.2",
"peggy": "^3.0.2",
"proxyquire": "^2.1.3",
"sinon": "^19.0.2",
Expand Down
2 changes: 1 addition & 1 deletion packages/zwave-js/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@
"alcalzone-shared": "^5.0.0",
"ansi-colors": "^4.1.3",
"execa": "^5.1.1",
"got": "^13.0.0",
"ky": "^1.7.2",
"mdns-server": "^1.0.11",
"p-queue": "^8.0.1",
"proper-lockfile": "^4.1.2",
Expand Down
57 changes: 29 additions & 28 deletions packages/zwave-js/src/lib/controller/FirmwareUpdateService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {
guessFirmwareFileFormat,
} from "@zwave-js/core";
import { Bytes, formatId } from "@zwave-js/shared";
import type { Headers, OptionsOfTextResponseBody } from "got";
import type { Options as KyOptions } from "ky";
import type PQueue from "p-queue";
import type {
FirmwareUpdateDeviceID,
Expand Down Expand Up @@ -57,17 +57,14 @@ function cleanCache() {
}
}

async function cachedGot<T>(config: OptionsOfTextResponseBody): Promise<T> {
// Replaces got's built-in cache functionality because it uses Keyv internally
// which apparently has some issues: https://github.com/zwave-js/node-zwave-js/issues/5404

async function cachedRequest<T>(url: string, config: KyOptions): Promise<T> {
const hash = Bytes.view(
await digest(
"sha-256",
Bytes.from(JSON.stringify(config.json)),
),
).toString("hex");
const cacheKey = `${config.method}:${config.url!.toString()}:${hash}`;
const cacheKey = `${config.method}:${url}:${hash}`;

// Return cached requests if they are not stale yet
if (requestCache.has(cacheKey)) {
Expand All @@ -77,13 +74,15 @@ async function cachedGot<T>(config: OptionsOfTextResponseBody): Promise<T> {
}
}

const { got } = await import("got");
const response = await got(config);
const responseJson = JSON.parse(response.body) as T;
const { default: ky } = await import("ky");
const response = await ky(url, config);
const responseJson = await response.json<T>();

// Check if we can cache the response
if (response.statusCode === 200 && response.headers["cache-control"]) {
const cacheControl = response.headers["cache-control"];
if (response.status === 200 && response.headers.has("cache-control")) {
const cacheControl = response.headers.get("cache-control")!;
const age = response.headers.get("age");
const date = response.headers.get("date");

let maxAge: number | undefined;
const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
Expand All @@ -93,10 +92,10 @@ async function cachedGot<T>(config: OptionsOfTextResponseBody): Promise<T> {

if (maxAge) {
let currentAge: number;
if (response.headers.age) {
currentAge = parseInt(response.headers.age, 10);
} else if (response.headers.date) {
currentAge = (Date.now() - Date.parse(response.headers.date))
if (age) {
currentAge = parseInt(age, 10);
} else if (date) {
currentAge = (Date.now() - Date.parse(date))
/ 1000;
} else {
currentAge = 0;
Expand Down Expand Up @@ -173,12 +172,12 @@ export async function getAvailableFirmwareUpdates(
deviceId: FirmwareUpdateDeviceID,
options: GetAvailableFirmwareUpdateOptions,
): Promise<FirmwareUpdateInfo[]> {
const headers: Headers = {
const headers = new Headers({
"User-Agent": options.userAgent,
"Content-Type": "application/json",
};
});
if (options.apiKey) {
headers["X-API-Key"] = options.apiKey;
headers.set("X-API-Key", options.apiKey);
}

const body: Record<string, string> = {
Expand All @@ -195,9 +194,9 @@ export async function getAvailableFirmwareUpdates(
// Prereleases and/or RF region-specific updates are only available in v3
const apiVersion = options.includePrereleases || !!rfRegion ? "v3" : "v1";

const config: OptionsOfTextResponseBody = {
const url = `${serviceURL()}/api/${apiVersion}/updates`;
const config: KyOptions = {
method: "POST",
url: `${serviceURL()}/api/${apiVersion}/updates`,
json: body,
// Consider re-enabling this instead of using cachedGot()
// At the moment, the built-in caching has some issues though, so we stick
Expand All @@ -216,7 +215,7 @@ export async function getAvailableFirmwareUpdates(
}
// Weird types...
const result = (
await requestQueue.add(() => cachedGot(config))
await requestQueue.add(() => cachedRequest(url, config))
) as FirmwareUpdateServiceResponse[];

// Remember the device ID in the response, so we can use it later
Expand All @@ -242,14 +241,13 @@ export async function downloadFirmwareUpdate(
// TODO: Make request abort-able (requires AbortController, Node 14.17+ / Node 16)

// Download the firmware file
const { got } = await import("got");
const downloadResponse = await got.get(file.url, {
timeout: { request: DOWNLOAD_TIMEOUT },
responseType: "buffer",
const { default: ky } = await import("ky");
const downloadResponse = await ky.get(file.url, {
timeout: DOWNLOAD_TIMEOUT,
// TODO: figure out how to do maxContentLength: MAX_FIRMWARE_SIZE,
});

const rawData = downloadResponse.body;
const rawData = new Uint8Array(await downloadResponse.arrayBuffer());

const requestedPathname = new URL(file.url).pathname;
// The response may be redirected, so the filename information may be different
Expand All @@ -263,12 +261,15 @@ export async function downloadFirmwareUpdate(

// Infer the file type from the content-disposition header or the filename
let filename: string;
const contentDisposition = downloadResponse.headers.get(
"content-disposition",
);
if (
downloadResponse.headers["content-disposition"]?.startsWith(
contentDisposition?.startsWith(
"attachment; filename=",
)
) {
filename = downloadResponse.headers["content-disposition"]
filename = contentDisposition
.split("filename=")[1]
.replace(/^"/, "")
.replace(/[";]$/, "");
Expand Down
27 changes: 15 additions & 12 deletions packages/zwave-js/src/lib/driver/UpdateConfig.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ import semverMaxSatisfying from "semver/ranges/max-satisfying.js";
export async function checkForConfigUpdates(
currentVersion: string,
): Promise<string | undefined> {
const { got } = await import("got");
const { default: ky } = await import("ky");
let registry: Record<string, unknown>;

try {
registry = await got
registry = await ky
.get("https://registry.npmjs.org/@zwave-js/config")
.json();
} catch {
Expand Down Expand Up @@ -66,11 +66,11 @@ export async function installConfigUpdate(
cacheDir: string;
},
): Promise<void> {
const { got } = await import("got");
const { default: ky } = await import("ky");

let registryInfo: any;
try {
registryInfo = await got
registryInfo = await ky
.get(`https://registry.npmjs.org/@zwave-js/config/${newVersion}`)
.json();
} catch {
Expand Down Expand Up @@ -137,16 +137,17 @@ export async function installConfigUpdate(

// Download the package tarball into the temporary directory
const tarFilename = path.join(tmpDir, "zjs-config-update.tgz");
let fileHandle: fs.FileHandle | undefined;
try {
const handle = await fs.open(tarFilename, "w");
const fstream = handle.createWriteStream({ autoClose: true });
const response = got.stream.get(url);
response.pipe(fstream);

await new Promise((resolve, reject) => {
response.on("error", reject);
response.on("end", resolve);
fileHandle = await fs.open(tarFilename, "w");
const writable = new WritableStream({
async write(chunk) {
await fileHandle!.write(chunk);
},
});

const response = await ky.get(url);
await response.body?.pipeTo(writable);
} catch (e) {
await freeLock();
throw new ZWaveError(
Expand All @@ -157,6 +158,8 @@ export async function installConfigUpdate(
}`,
ZWaveErrorCodes.Config_Update_InstallFailed,
);
} finally {
await fileHandle?.close();
}

// This should not be necessary in Docker. Leaving it here anyways in case
Expand Down
4 changes: 2 additions & 2 deletions packages/zwave-js/src/lib/telemetry/deviceConfig.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ export async function reportMissingDeviceConfig(
// // If we tried to report this file earlier, we can skip the report
// if (missingDeviceConfigCache.has(configFingerprint)) return;
// // Otherwise ask our device DB if it exists
// const { got } = await import("got");
// const { default: ky } = await import("ky");
// try {
// const data = await got
// const data = await ky
// .get(
// `https://devices.zwave-js.io/public_api/getdeviceinfo/${configFingerprint.replace(
// /:/g,
Expand Down
4 changes: 2 additions & 2 deletions packages/zwave-js/src/lib/telemetry/statistics.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,10 @@ export async function compileStatistics(
export async function sendStatistics(
statistics: Record<string, any>,
): Promise<boolean | number> {
const { got } = await import("got");
const { default: ky } = await import("ky");

try {
const data = await got
const data = await ky
.post(statisticsUrl, {
json: { data: [statistics] },
headers: { "x-api-token": apiToken },
Expand Down
Loading

0 comments on commit 205ad8d

Please sign in to comment.