From 64c66aea8e9d217a485094969d6733551e7310e5 Mon Sep 17 00:00:00 2001 From: Josh B <421772+HT154@users.noreply.github.com> Date: Fri, 17 Jan 2025 06:34:34 -0800 Subject: [PATCH] Implement SPICE-0009 External Readers (#26) * Add `EvaluatorOptions.externalModuleReaders` and `EvaluatorOptions.externalResourceReaders`. * Add `ExternalReaderClient` to host the child process side of the external reader workflow. --- .gitignore | 1 + Package.swift | 4 + .../Encoder/MessagePackEncoder.swift | 9 +- .../SingleValueEncodingContainer.swift | 11 +- Sources/PklSwift/EvaluatorManager.swift | 9 +- Sources/PklSwift/EvaluatorOptions.swift | 26 ++- Sources/PklSwift/ExternalReaderClient.swift | 208 ++++++++++++++++++ Sources/PklSwift/Message.swift | 49 ++++- Sources/PklSwift/MessageTransport.swift | 186 +++++++++++----- Sources/PklSwift/PklEvaluatorSettings.swift | 9 +- .../TestExternalReader.swift | 51 +++++ .../ExternalReaderClientTest.swift | 65 ++++++ Tests/PklSwiftTests/ProjectTest.swift | 39 +++- docs/modules/ROOT/pages/external-readers.adoc | 37 ++++ docs/nav.adoc | 1 + 15 files changed, 627 insertions(+), 78 deletions(-) create mode 100644 Sources/PklSwift/ExternalReaderClient.swift create mode 100644 Sources/test-external-reader/TestExternalReader.swift create mode 100644 Tests/PklSwiftTests/ExternalReaderClientTest.swift create mode 100644 docs/modules/ROOT/pages/external-readers.adoc diff --git a/.gitignore b/.gitignore index d6935b7..1f9c440 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store .build +.index-build /Packages /*.xcodeproj xcuserdata/ diff --git a/Package.swift b/Package.swift index ed867ec..1bb476c 100644 --- a/Package.swift +++ b/Package.swift @@ -48,6 +48,10 @@ let package = Package( ], resources: [.embedInCode("Resources/VERSION.txt")] ), + .executableTarget( + name: "test-external-reader", + dependencies: ["PklSwift"] + ), .testTarget( name: "PklSwiftTests", dependencies: [ diff --git a/Sources/MessagePack/Encoder/MessagePackEncoder.swift b/Sources/MessagePack/Encoder/MessagePackEncoder.swift index d13c9c9..45950fc 100644 --- a/Sources/MessagePack/Encoder/MessagePackEncoder.swift +++ b/Sources/MessagePack/Encoder/MessagePackEncoder.swift @@ -1,5 +1,5 @@ // ===----------------------------------------------------------------------===// -// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -44,10 +44,13 @@ public final class MessagePackEncoder { try Box(data).encode(to: encoder) case let date as Date: try Box(date).encode(to: encoder) - case let bytes as [UInt8]: - try Box<[UInt8]>(bytes).encode(to: encoder) case let url as URL: try Box(url).encode(to: encoder) + case let bytes as [UInt8]: + guard type(of: value) == [UInt8].self else { + fallthrough + } + try Box<[UInt8]>(bytes).encode(to: encoder) default: try value.encode(to: encoder) } diff --git a/Sources/MessagePack/Encoder/SingleValueEncodingContainer.swift b/Sources/MessagePack/Encoder/SingleValueEncodingContainer.swift index e129540..8f53372 100644 --- a/Sources/MessagePack/Encoder/SingleValueEncodingContainer.swift +++ b/Sources/MessagePack/Encoder/SingleValueEncodingContainer.swift @@ -1,5 +1,5 @@ // ===----------------------------------------------------------------------===// -// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -304,8 +304,6 @@ extension _MessagePackEncoder.SingleValueContainer: SingleValueEncodingContainer try self.encode(data) case let date as Date: try self.encode(date) - case let bytes as [UInt8]: - try self.encode(bytes) case let url as URL: try self.encode(url) case let bool as Bool: @@ -334,9 +332,16 @@ extension _MessagePackEncoder.SingleValueContainer: SingleValueEncodingContainer try self.encode(uint64) case let num as any BinaryInteger & Encodable: try self.encode(num) + case let bytes as [UInt8]: + guard type(of: value) == [UInt8].self else { + fallthrough + } + try self.encode(bytes) default: let writer: Writer = BufferWriter() let encoder = _MessagePackEncoder() + encoder.userInfo = userInfo + encoder.codingPath = codingPath try value.encode(to: encoder) try encoder.write(into: writer) let data = (writer as! BufferWriter).bytes diff --git a/Sources/PklSwift/EvaluatorManager.swift b/Sources/PklSwift/EvaluatorManager.swift index e8f8f9a..20b3b0b 100644 --- a/Sources/PklSwift/EvaluatorManager.swift +++ b/Sources/PklSwift/EvaluatorManager.swift @@ -1,5 +1,5 @@ // ===----------------------------------------------------------------------===// -// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -108,7 +108,7 @@ public actor EvaluatorManager { // note; when our C bindings are released, change `init()` based on compiler flags. public init() { - self.init(transport: ChildProcessMessageTransport()) + self.init(transport: ServerMessageTransport()) } // Used for testing only. @@ -264,6 +264,9 @@ public actor EvaluatorManager { guard options.http == nil || version >= pklVersion0_26 else { throw PklError("http options are not supported on Pkl versions lower than 0.26") } + guard (options.externalModuleReaders == nil && options.externalResourceReaders == nil) || version >= pklVersion0_27 else { + throw PklError("external reader options are not supported on Pkl versions lower than 0.27") + } let req = options.toMessage() guard let response = try await ask(req) as? CreateEvaluatorResponse else { throw PklBugError.invalidMessageCode( @@ -371,8 +374,10 @@ enum PklBugError: Error { let pklVersion0_25 = SemanticVersion("0.25.0")! let pklVersion0_26 = SemanticVersion("0.26.0")! +let pklVersion0_27 = SemanticVersion("0.27.0")! let supportedPklVersions = [ pklVersion0_25, pklVersion0_26, + pklVersion0_27, ] diff --git a/Sources/PklSwift/EvaluatorOptions.swift b/Sources/PklSwift/EvaluatorOptions.swift index 8251d98..ebd9c45 100644 --- a/Sources/PklSwift/EvaluatorOptions.swift +++ b/Sources/PklSwift/EvaluatorOptions.swift @@ -1,5 +1,5 @@ // ===----------------------------------------------------------------------===// -// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -32,7 +32,9 @@ public struct EvaluatorOptions { logger: Logger = Loggers.noop, projectBaseURI: URL? = nil, http: Http? = nil, - declaredProjectDependencies: [String: ProjectDependency]? = nil + declaredProjectDependencies: [String: ProjectDependency]? = nil, + externalModuleReaders: [String: ExternalReader]? = nil, + externalResourceReaders: [String: ExternalReader]? = nil ) { self.allowedModules = allowedModules self.allowedResources = allowedResources @@ -49,6 +51,8 @@ public struct EvaluatorOptions { self.projectBaseURI = projectBaseURI self.http = http self.declaredProjectDependencies = declaredProjectDependencies + self.externalModuleReaders = externalModuleReaders + self.externalResourceReaders = externalResourceReaders } /// Regular expression patterns that control what modules are allowed to be imported in a Pkl program. @@ -123,6 +127,18 @@ public struct EvaluatorOptions { /// When importing dependencies, a `PklProject.deps.json` file must exist within ``projectBaseURI`` /// that contains the project's resolved dependencies. public var declaredProjectDependencies: [String: ProjectDependency]? + + /// Registered external commands that implement module reader schemes. + /// + /// Added in Pkl 0.27. + /// If the underlying Pkl does not support external readers, evaluation will fail when a registered scheme is used. + public var externalModuleReaders: [String: ExternalReader]? + + /// Registered external commands that implement resource reader schemes. + /// + /// Added in Pkl 0.27. + /// If the underlying Pkl does not support external readers, evaluation will fail when a registered scheme is used. + public var externalResourceReaders: [String: ExternalReader]? } extension EvaluatorOptions { @@ -162,7 +178,9 @@ extension EvaluatorOptions { cacheDir: self.cacheDir, outputFormat: self.outputFormat, project: self.project(), - http: self.http + http: self.http, + externalModuleReaders: self.externalModuleReaders, + externalResourceReaders: self.externalResourceReaders ) } @@ -245,6 +263,8 @@ extension EvaluatorOptions { options.cacheDir = evaluatorSettings.noCache != nil ? nil : (evaluatorSettings.moduleCacheDir ?? self.cacheDir) options.rootDir = evaluatorSettings.rootDir ?? self.rootDir options.http = evaluatorSettings.http ?? self.http + options.externalModuleReaders = evaluatorSettings.externalModuleReaders ?? options.externalModuleReaders + options.externalResourceReaders = evaluatorSettings.externalResourceReaders ?? options.externalResourceReaders return options } diff --git a/Sources/PklSwift/ExternalReaderClient.swift b/Sources/PklSwift/ExternalReaderClient.swift new file mode 100644 index 0000000..96bd761 --- /dev/null +++ b/Sources/PklSwift/ExternalReaderClient.swift @@ -0,0 +1,208 @@ +// ===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ===----------------------------------------------------------------------===// + +import Foundation +import MessagePack + +// Example usage: +// import PklSwift +// @main +// struct Main { +// static func main() async throws { +// let client = ExternalReaderClient( +// options: ExternalReaderClientOptions( +// resourceReaders: [MyResourceReader()] +// )) +// try await client.run() +// } +// } + +public struct ExternalReaderClientOptions { + /// Reader to receive requests. + public var requestReader: Reader = FileHandle.standardInput + + /// Writer to publish responses. + public var responseWriter: Writer = FileHandle.standardOutput + + /// Readers that allow reading custom resources in Pkl. + public var moduleReaders: [ModuleReader] = [] + + /// Readers that allow importing custom modules in Pkl. + public var resourceReaders: [ResourceReader] = [] + + public init( + requestReader: Reader = FileHandle.standardInput, + responseWriter: Writer = FileHandle.standardOutput, + moduleReaders: [ModuleReader] = [], + resourceReaders: [ResourceReader] = [] + ) { + self.requestReader = requestReader + self.responseWriter = responseWriter + self.moduleReaders = moduleReaders + self.resourceReaders = resourceReaders + } +} + +public class ExternalReaderClient { + private let moduleReaders: [ModuleReader] + private let resourceReaders: [ResourceReader] + private let transport: MessageTransport + + public init(options: ExternalReaderClientOptions) { + self.moduleReaders = options.moduleReaders + self.resourceReaders = options.resourceReaders + self.transport = ExternalReaderMessageTransport( + reader: options.requestReader, writer: options.responseWriter + ) + } + + public func run() async throws { + for try await message in try self.transport.getMessages() { + switch message { + case let message as InitializeModuleReaderRequest: + try self.handleInitializeModuleReaderRequest(request: message) + case let message as InitializeResourceReaderRequest: + try self.handleInitializeResourceReaderRequest(request: message) + case let message as ReadModuleRequest: + try await self.handleReadModuleRequest(request: message) + case let message as ReadResourceRequest: + try await self.handleReadResourceRequest(request: message) + case let message as ListModulesRequest: + try await self.handleListModulesRequest(request: message) + case let message as ListResourcesRequest: + try await self.handleListResourcesRequest(request: message) + case _ as CloseExternalProcess: + self.close() + default: + throw PklBugError.unknownMessage("Got request for unknown message: \(message)") + } + } + } + + public func close() { + self.transport.close() + } + + func handleInitializeModuleReaderRequest(request: InitializeModuleReaderRequest) throws { + var response = InitializeModuleReaderResponse(requestId: request.requestId, spec: nil) + guard let reader = moduleReaders.first(where: { $0.scheme == request.scheme }) else { + try self.transport.send(response) + return + } + response.spec = reader.toMessage() + try self.transport.send(response) + } + + func handleInitializeResourceReaderRequest(request: InitializeResourceReaderRequest) throws { + var response = InitializeResourceReaderResponse(requestId: request.requestId, spec: nil) + guard let reader = resourceReaders.first(where: { $0.scheme == request.scheme }) else { + try self.transport.send(response) + return + } + response.spec = reader.toMessage() + try self.transport.send(response) + } + + func handleReadModuleRequest(request: ReadModuleRequest) async throws { + var response = ReadModuleResponse( + requestId: request.requestId, + evaluatorId: request.evaluatorId, + contents: nil, + error: nil + ) + guard let reader = moduleReaders.first(where: { $0.scheme == request.uri.scheme }) else { + response.error = "No module reader found for scheme \(request.uri.scheme!)" + try self.transport.send(response) + return + } + do { + let result = try await reader.read(url: request.uri) + response.contents = result + try self.transport.send(response) + } catch { + response.error = "\(error)" + try self.transport.send(response) + } + } + + func handleReadResourceRequest(request: ReadResourceRequest) async throws { + var response = ReadResourceResponse( + requestId: request.requestId, + evaluatorId: request.evaluatorId, + contents: nil, + error: nil + ) + guard + let reader = resourceReaders.first(where: { $0.scheme == request.uri.scheme }) else { + response.error = "No resource reader found for scheme \(request.uri.scheme!)" + try self.transport.send(response) + return + } + do { + let result = try await reader.read(url: request.uri) + response.contents = result + try self.transport.send(response) + } catch { + response.error = "\(error)" + try self.transport.send(response) + } + } + + func handleListModulesRequest(request: ListModulesRequest) async throws { + var response = ListModulesResponse( + requestId: request.requestId, + evaluatorId: request.evaluatorId, + pathElements: nil, + error: nil + ) + guard let reader = moduleReaders.first(where: { $0.scheme == request.uri.scheme }) else { + response.error = "No module reader found for scheme \(request.uri.scheme!)" + try self.transport.send(response) + return + } + do { + let elems = try await reader.listElements(uri: request.uri) + response.pathElements = elems.map { $0.toMessage() } + try self.transport.send(response) + } catch { + response.error = "\(error)" + try self.transport.send(response) + } + } + + func handleListResourcesRequest(request: ListResourcesRequest) async throws { + var response = ListResourcesResponse( + requestId: request.requestId, + evaluatorId: request.evaluatorId, + pathElements: nil, + error: nil + ) + guard + let reader = resourceReaders.first(where: { $0.scheme == request.uri.scheme }) else { + response.error = "No resource reader found for scheme \(request.uri.scheme!)" + try self.transport.send(response) + return + } + do { + let elems = try await reader.listElements(uri: request.uri) + response.pathElements = elems.map { $0.toMessage() } + try self.transport.send(response) + } catch { + response.error = "\(error)" + try self.transport.send(response) + } + } +} diff --git a/Sources/PklSwift/Message.swift b/Sources/PklSwift/Message.swift index d5834f6..ebbd9c7 100644 --- a/Sources/PklSwift/Message.swift +++ b/Sources/PklSwift/Message.swift @@ -1,5 +1,5 @@ // ===----------------------------------------------------------------------===// -// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -62,8 +62,8 @@ enum MessageType: Int, Codable { case CREATE_EVALUATOR_REQUEST = 0x20 case CREATE_EVALUATOR_RESPONSE = 0x21 case CLOSE_EVALUATOR = 0x22 - case EVALUATOR_REQUEST = 0x23 - case EVALUATOR_RESPONSE = 0x24 + case EVALUATE_REQUEST = 0x23 + case EVALUATE_RESPONSE = 0x24 case LOG_MESSAGE = 0x25 case READ_RESOURCE_REQUEST = 0x26 case READ_RESOURCE_RESPONSE = 0x27 @@ -73,6 +73,11 @@ enum MessageType: Int, Codable { case LIST_RESOURCES_RESPONSE = 0x2B case LIST_MODULES_REQUEST = 0x2C case LIST_MODULES_RESPONSE = 0x2D + case INITIALIZE_MODULE_READER_REQUEST = 0x2E + case INITIALIZE_MODULE_READER_RESPONSE = 0x2F + case INITIALIZE_RESOURCE_READER_REQUEST = 0x30 + case INITIALIZE_RESOURCE_READER_RESPONSE = 0x31 + case CLOSE_EXTERNAL_PROCESS = 0x32 } extension MessageType { @@ -85,9 +90,9 @@ extension MessageType { case is CloseEvaluatorRequest: return MessageType.CLOSE_EVALUATOR case is EvaluateRequest: - return MessageType.EVALUATOR_REQUEST + return MessageType.EVALUATE_REQUEST case is EvaluateResponse: - return MessageType.EVALUATOR_RESPONSE + return MessageType.EVALUATE_RESPONSE case is LogMessage: return MessageType.LOG_MESSAGE case is ListResourcesRequest: @@ -102,6 +107,16 @@ extension MessageType { return MessageType.READ_MODULE_RESPONSE case is ReadResourceResponse: return MessageType.READ_RESOURCE_RESPONSE + case is InitializeModuleReaderRequest: + return MessageType.INITIALIZE_MODULE_READER_REQUEST + case is InitializeModuleReaderResponse: + return MessageType.INITIALIZE_MODULE_READER_RESPONSE + case is InitializeResourceReaderRequest: + return MessageType.INITIALIZE_RESOURCE_READER_REQUEST + case is InitializeResourceReaderResponse: + return MessageType.INITIALIZE_RESOURCE_READER_RESPONSE + case is CloseExternalProcess: + return MessageType.CLOSE_EXTERNAL_PROCESS default: preconditionFailure("Unreachable code") } @@ -123,6 +138,8 @@ struct CreateEvaluatorRequest: ClientRequestMessage { var outputFormat: String? var project: ProjectOrDependency? var http: Http? + var externalModuleReaders: [String: ExternalReader]? + var externalResourceReaders: [String: ExternalReader]? } struct ProjectOrDependency: Codable { @@ -231,3 +248,25 @@ struct LogMessage: ServerOneWayMessage { // NOTE: not guaranteed to conform to URL. This might have been transformed by a stack frame transformer. let frameUri: String } + +struct InitializeModuleReaderRequest: ServerRequestMessage { + var requestId: Int64 + let scheme: String +} + +struct InitializeModuleReaderResponse: ClientResponseMessage { + var requestId: Int64 + var spec: ModuleReaderSpec? +} + +struct InitializeResourceReaderRequest: ServerRequestMessage { + var requestId: Int64 + let scheme: String +} + +struct InitializeResourceReaderResponse: ClientResponseMessage { + var requestId: Int64 + var spec: ResourceReaderSpec? +} + +struct CloseExternalProcess: ServerOneWayMessage {} diff --git a/Sources/PklSwift/MessageTransport.swift b/Sources/PklSwift/MessageTransport.swift index 978b41d..0852651 100644 --- a/Sources/PklSwift/MessageTransport.swift +++ b/Sources/PklSwift/MessageTransport.swift @@ -1,5 +1,5 @@ // ===----------------------------------------------------------------------===// -// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -31,35 +31,104 @@ protocol MessageTransport { extension Pipe: Reader { public func read(into: UnsafeMutableRawBufferPointer) throws -> Int { - let data = try fileHandleForReading.read(upToCount: into.count) - if data == nil { - return 0 - } - data!.copyBytes(to: into) - return data!.count + try fileHandleForReading.read(into: into) + } + + public func close() throws { + try fileHandleForReading.close() + } +} + +extension FileHandle: Reader { + public func read(into: UnsafeMutableRawBufferPointer) throws -> Int { + guard let data = try read(upToCount: into.count) else { return 0 } + data.copyBytes(to: into) + return data.count } public func close() throws { - fileHandleForReading.closeFile() + closeFile() } } extension Pipe: Writer { public func write(_ buffer: UnsafeRawBufferPointer) throws { - try fileHandleForWriting.write(contentsOf: buffer) + try fileHandleForWriting.write(buffer) } } -/// A ``MessageTransport`` that sends and receives messages by spawning Pkl as a child process. -public class ChildProcessMessageTransport: MessageTransport { - var reader: Pipe! - var writer: Pipe! +extension FileHandle: Writer { + public func write(_ buffer: UnsafeRawBufferPointer) throws { + try self.write(contentsOf: buffer) + } +} + +/// A ``MessageTransport`` base class that implements core message handling logic +public class BaseMessageTransport: MessageTransport { + var reader: Reader! + var writer: Writer! var encoder: MessagePackEncoder! var decoder: MessagePackDecoder! + + var running: Bool { true } + + func send(_ message: ClientMessage) throws { + debug("Sending message: \(message)") + + let messageType = MessageType.getMessageType(message) + try self.encoder.encodeArrayHeader(2) + try self.encoder.encode(messageType) + try self.encoder.encode(message) + } + + fileprivate func decodeMessage(_ messageType: MessageType) throws -> ServerMessage { + switch messageType { + case MessageType.READ_MODULE_REQUEST: + return try self.decoder.decode(as: ReadModuleRequest.self) + case MessageType.READ_RESOURCE_REQUEST: + return try self.decoder.decode(as: ReadResourceRequest.self) + case MessageType.LIST_MODULES_REQUEST: + return try self.decoder.decode(as: ListModulesRequest.self) + case MessageType.LIST_RESOURCES_REQUEST: + return try self.decoder.decode(as: ListResourcesRequest.self) + default: + throw PklBugError.unknownMessage("Received unexpected message: \(messageType)") + } + } + + func close() {} + + func getMessages() throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + Task { + while self.running { + do { + let arrayLength = try decoder.decodeArrayLength() + assert(arrayLength == 2) + let code = try decoder.decode(as: MessageType.self) + let message = try decodeMessage(code) + debug("Received message: \(message)") + continuation.yield(message) + } catch { + if self.running { + continuation.finish(throwing: error) + } + } + } + continuation.finish() + } + } + } +} + +/// A ``MessageTransport`` that sends and receives messages by spawning Pkl as a child process. +public class ServerMessageTransport: BaseMessageTransport { var process: Process? let pklCommand: [String]? - convenience init() { + override var running: Bool { self.process?.isRunning == true } + + override convenience init() { self.init(pklCommand: nil) } @@ -75,8 +144,8 @@ public class ChildProcessMessageTransport: MessageTransport { var arguments = Array(pklCommand.dropFirst()) arguments.append("server") self.process!.arguments = arguments - self.reader = .init() - self.writer = .init() + self.reader = Pipe() + self.writer = Pipe() self.encoder = .init(writer: self.writer) self.decoder = .init(reader: self.reader) self.process!.standardOutput = self.reader @@ -85,17 +154,25 @@ public class ChildProcessMessageTransport: MessageTransport { try self.process!.run() } - func send(_ message: ClientMessage) throws { + override func send(_ message: ClientMessage) throws { try self.ensureProcessStarted() - debug("Sending message: \(message)") + try super.send(message) + } - let messageType = MessageType.getMessageType(message) - try self.encoder.encodeArrayHeader(2) - try self.encoder.encode(messageType) - try self.encoder.encode(message) + override fileprivate func decodeMessage(_ messageType: MessageType) throws -> ServerMessage { + switch messageType { + case MessageType.CREATE_EVALUATOR_RESPONSE: + return try self.decoder.decode(as: CreateEvaluatorResponse.self) + case MessageType.EVALUATE_RESPONSE: + return try self.decoder.decode(as: EvaluateResponse.self) + case MessageType.LOG_MESSAGE: + return try self.decoder.decode(as: LogMessage.self) + default: + return try super.decodeMessage(messageType) + } } - func close() { + override func close() { if self.process == nil { return } @@ -111,45 +188,38 @@ public class ChildProcessMessageTransport: MessageTransport { self.process = nil } - private func decodeMessage(_ messageType: MessageType) throws -> ServerMessage { + override func getMessages() throws -> AsyncThrowingStream { + try self.ensureProcessStarted() + return try super.getMessages() + } +} + +public class ExternalReaderMessageTransport: BaseMessageTransport { + override var running: Bool { self._running } + private var _running = true + + init(reader: Reader, writer: Writer) { + super.init() + self.reader = reader + self.writer = writer + self.encoder = .init(writer: self.writer) + self.decoder = .init(reader: self.reader) + } + + override fileprivate func decodeMessage(_ messageType: MessageType) throws -> ServerMessage { switch messageType { - case MessageType.CREATE_EVALUATOR_RESPONSE: - return try self.decoder.decode(as: CreateEvaluatorResponse.self) - case MessageType.EVALUATOR_RESPONSE: - return try self.decoder.decode(as: EvaluateResponse.self) - case MessageType.READ_MODULE_REQUEST: - return try self.decoder.decode(as: ReadModuleRequest.self) - case MessageType.LOG_MESSAGE: - return try self.decoder.decode(as: LogMessage.self) - case MessageType.READ_RESOURCE_REQUEST: - return try self.decoder.decode(as: ReadResourceRequest.self) - case MessageType.LIST_MODULES_REQUEST: - return try self.decoder.decode(as: ListModulesRequest.self) - case MessageType.LIST_RESOURCES_REQUEST: - return try self.decoder.decode(as: ListResourcesRequest.self) + case MessageType.INITIALIZE_MODULE_READER_REQUEST: + return try self.decoder.decode(as: InitializeModuleReaderRequest.self) + case MessageType.INITIALIZE_RESOURCE_READER_REQUEST: + return try self.decoder.decode(as: InitializeResourceReaderRequest.self) + case MessageType.CLOSE_EXTERNAL_PROCESS: + return try self.decoder.decode(as: CloseExternalProcess.self) default: - fatalError("Unreachable code") + return try super.decodeMessage(messageType) } } - func getMessages() throws -> AsyncThrowingStream { - try self.ensureProcessStarted() - return AsyncThrowingStream { continuation in - Task { - while self.process?.isRunning == true { - do { - let arrayLength = try decoder.decodeArrayLength() - assert(arrayLength == 2) - let code = try decoder.decode(as: MessageType.self) - let message = try decodeMessage(code) - debug("Received message: \(message)") - continuation.yield(message) - } catch { - continuation.finish(throwing: error) - } - } - continuation.finish() - } - } + override func close() { + self._running = false } } diff --git a/Sources/PklSwift/PklEvaluatorSettings.swift b/Sources/PklSwift/PklEvaluatorSettings.swift index 9677515..7eb33e3 100644 --- a/Sources/PklSwift/PklEvaluatorSettings.swift +++ b/Sources/PklSwift/PklEvaluatorSettings.swift @@ -1,5 +1,5 @@ // ===----------------------------------------------------------------------===// -// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,6 +26,8 @@ public struct PklEvaluatorSettings: Decodable, Hashable { let moduleCacheDir: String? let rootDir: String? let http: Http? + let externalModuleReaders: [String: ExternalReader]? + let externalResourceReaders: [String: ExternalReader]? } /// Settings that control how Pkl talks to HTTP(S) servers. @@ -85,3 +87,8 @@ public struct Proxy: Codable, Hashable { /// ``` var noProxy: [String]? } + +public struct ExternalReader: Codable, Hashable { + var executable: String + var arguments: [String]? = nil +} diff --git a/Sources/test-external-reader/TestExternalReader.swift b/Sources/test-external-reader/TestExternalReader.swift new file mode 100644 index 0000000..e1d9665 --- /dev/null +++ b/Sources/test-external-reader/TestExternalReader.swift @@ -0,0 +1,51 @@ +// ===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ===----------------------------------------------------------------------===// + +import Foundation +import PklSwift + +@main +struct TestExternalReader { + static func main() async throws { + let client = ExternalReaderClient( + options: ExternalReaderClientOptions( + resourceReaders: [FibReader()] + )) + try await client.run() + } +} + +struct FibReader: ResourceReader { + var scheme: String { "fib" } + var isGlobbable: Bool { false } + var hasHierarchicalUris: Bool { false } + func listElements(uri: URL) async throws -> [PathElement] { throw PklError("not implemented") } + func read(url: URL) async throws -> [UInt8] { + let key = url.absoluteString.dropFirst(scheme.count + 1) + guard let n = Int(key), n > 0 else { + throw PklError("input uri must be in format fib:") + } + return Array(String(fibonacci(n: n)).utf8) + } +} + +func fibonacci(n: Int) -> Int { + var (a, b) = (0, 1) + for _ in 0..= pklVersion0_27 else { + throw XCTSkip("External readers require Pkl 0.27 or later.") + } + + let tempDir = try tempDir() + let testFile = tempDir.appendingPathComponent("test.pkl") + try #""" + import "pkl:test" + + fib5 = read("fib:5").text.toInt() + fib10 = read("fib:10").text.toInt() + fib20 = read("fib:20").text.toInt() + + fibErrA = test.catch(() -> read("fib:%20")) + fibErrB = test.catch(() -> read("fib:abc")) + fibErrC = test.catch(() -> read("fib:-10")) + """#.write(to: testFile, atomically: true, encoding: .utf8) + + let expectedResult = """ + fib5 = 5 + fib10 = 55 + fib20 = 6765 + fibErrA = "I/O error reading resource `fib:%20`. IOException: PklError(message: \\"input uri must be in format fib:\\")" + fibErrB = "I/O error reading resource `fib:abc`. IOException: PklError(message: \\"input uri must be in format fib:\\")" + fibErrC = "I/O error reading resource `fib:-10`. IOException: PklError(message: \\"input uri must be in format fib:\\")" + """ + + let opts = EvaluatorOptions( + allowedModules: ["file:", "repl:text"], + allowedResources: ["fib:", "prop:"], + externalResourceReaders: [ + "fib": ExternalReader(executable: "./.build/debug/test-external-reader"), + ] + ) + + try await withEvaluator(options: opts) { evaluator in + let result = try await evaluator.evaluateOutputText(source: .url(testFile)) + XCTAssertEqual(result.trimmingCharacters(in: .whitespacesAndNewlines), expectedResult.trimmingCharacters(in: .whitespacesAndNewlines)) + } + } +} diff --git a/Tests/PklSwiftTests/ProjectTest.swift b/Tests/PklSwiftTests/ProjectTest.swift index c29e234..051f542 100644 --- a/Tests/PklSwiftTests/ProjectTest.swift +++ b/Tests/PklSwiftTests/ProjectTest.swift @@ -1,5 +1,5 @@ // ===----------------------------------------------------------------------===// -// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -51,10 +51,38 @@ class ProjectTest: XCTestCase { } } """ + let externalReaderSettings = version < pklVersion0_27 ? "" : """ + externalModuleReaders { + ["scheme1"] { + executable = "reader1" + } + ["scheme2"] { + executable = "reader2" + arguments { "with"; "args" } + } + } + externalResourceReaders { + ["scheme3"] { + executable = "reader3" + } + ["scheme4"] { + executable = "reader4" + arguments { "with"; "args" } + } + } + """ let httpExpectation = version < pklVersion0_26 ? nil : Http( caCertificates: nil, proxy: .init(address: "http://localhost:1", noProxy: ["example.com", "foo.bar.org"]) ) + let externalModuleReadersExpectation = version < pklVersion0_27 ? nil : [ + "scheme1": ExternalReader(executable: "reader1"), + "scheme2": ExternalReader(executable: "reader2", arguments: ["with", "args"]), + ] + let externalResourceReadersExpectation = version < pklVersion0_27 ? nil : [ + "scheme3": ExternalReader(executable: "reader3"), + "scheme4": ExternalReader(executable: "reader4", arguments: ["with", "args"]), + ] try #""" amends "pkl:Project" @@ -99,6 +127,7 @@ class ProjectTest: XCTestCase { timeout = 5.min moduleCacheDir = "/bar/buzz" rootDir = "/buzzy" + \#(externalReaderSettings) \#(httpSetting) } @@ -124,7 +153,9 @@ class ProjectTest: XCTestCase { timeout: .minutes(5), moduleCacheDir: "/bar/buzz", rootDir: "/buzzy", - http: httpExpectation + http: httpExpectation, + externalModuleReaders: externalModuleReadersExpectation, + externalResourceReaders: externalResourceReadersExpectation ) let expectedPackage = PklSwift.Project.Package( name: "hawk", @@ -179,7 +210,9 @@ class ProjectTest: XCTestCase { timeout: nil, moduleCacheDir: nil, rootDir: nil, - http: nil + http: nil, + externalModuleReaders: nil, + externalResourceReaders: nil ), projectFileUri: "\(otherProjectFile)", tests: [], diff --git a/docs/modules/ROOT/pages/external-readers.adoc b/docs/modules/ROOT/pages/external-readers.adoc new file mode 100644 index 0000000..2e059c9 --- /dev/null +++ b/docs/modules/ROOT/pages/external-readers.adoc @@ -0,0 +1,37 @@ += External Readers + +pkl-swift provides APIs that aid in implementing xref:main:language-reference:index.adoc#external-readers[External Readers]. +In this mode of execution, the program built with pkl-swift runs as a child process of the Pkl evaluator, rather than a parent process. +The `PklSwift.ExternalReaderClient` type provides a set of tools for building external readers. + +Much like implementing xref:ROOT:evaluation.adoc#custom-readers[Custom Readers], external readers are implemented by providing one or more instances of the `PklSwift.ResourceReader` and `pkl.ModuleReader` protocols. + +== Example + +.main.swift +[source,swift] +---- +import Foundation +import PklSwift + +@main struct Main { + static func main() async throws { + let client = ExternalReaderClient( + options: ExternalReaderClientOptions( + resourceReaders: [MyResourceReader()] + )) + try await client.run() + } +} + +struct MyResourceReader: ResourceReader { + var scheme: String { "env2" } + var isGlobbable: Bool { false } + var hasHierarchicalUris: Bool { false } + func listElements(uri: URL) async throws -> [PathElement] { throw PklError("not implemented") } + func read(url: URL) async throws -> [UInt8] { + let key = url.absoluteString.dropFirst(scheme.count + 1) + return Array((ProcessInfo.processInfo.environment[String(key)] ?? "").utf8) + } +} +---- diff --git a/docs/nav.adoc b/docs/nav.adoc index 4c0662d..09776fa 100644 --- a/docs/nav.adoc +++ b/docs/nav.adoc @@ -1,4 +1,5 @@ * xref:ROOT:quickstart.adoc[Quickstart] * xref:ROOT:evaluation.adoc[Evaluator API] * xref:ROOT:codegen.adoc[Code Generation] +* xref:ROOT:external-readers.adoc[External Readers] * xref:ROOT:CHANGELOG.adoc[Changelog]