feat: vitri CM awareness

README.md

@@ -1,4 +1,15 @@
-# Finorza
-A Minecraft 1.16 mod for simulating autonomous agents. Requires Fabric Loader & Fabric API.
+<h1>
+    <img src="https://raw.githubusercontent.com/vitri-ent/finorza/main/src/main/resources/assets/icon.png" width="24">
+    <span>Fiиorza</span>
+</h1>
+A Minecraft 1.16 mod for simulating autonomous agents.
 
 https://github.com/vitri-ent/finorza/assets/61946759/54c3932f-b414-45c1-8dd4-257cc4f28bff
+
+## Requirements
+- **Minecraft** 1.16.5 w/ [Fabric](https://fabricmc.net)
+- [**Fabric API** v0.42](https://modrinth.com/mod/fabric-api/version/0.42.0+1.16)
+- [**Cloth Config** v4.17](https://modrinth.com/mod/cloth-config/version/4.17.101)
+- [**Mod Menu** v1.16](https://modrinth.com/mod/modmenu/version/1.16.23)
+
+Also verified to run with Sodium, Lithium, and LazyDFU. Most other optimization mods should work.

src/main/java/io/pyke/vitri/finorza/inference/rpc/RemoteControlService.java

@@ -15,11 +15,14 @@
 import io.grpc.stub.ServerCallStreamObserver;
 import io.grpc.stub.StreamObserver;
 import net.minecraft.client.Minecraft;
-import net.minecraft.client.gui.screens.ChatScreen;
+import net.minecraft.client.gui.screens.DeathScreen;
 import net.minecraft.client.gui.screens.TitleScreen;
 import net.minecraft.client.gui.screens.inventory.AbstractContainerScreen;
 import net.minecraft.client.multiplayer.ServerData;
+import net.minecraft.client.player.LocalPlayer;
 import net.minecraft.client.resources.language.I18n;
+import net.minecraft.core.Registry;
+import net.minecraft.world.level.LightLayer;
 import net.minecraft.world.level.storage.LevelStorageException;
 import net.minecraft.world.level.storage.LevelSummary;
 import org.jetbrains.annotations.NotNull;
@@ -143,18 +146,29 @@ public void updateHumanControl(UpdateHumanControlRequest request, StreamObserver
         response.onCompleted();
     }
 
+    @Override
+    public void respawn(Empty request, StreamObserver<Empty> response) {
+        if (this.minecraft.player != null && this.minecraft.player.isDeadOrDying()) {
+            this.minecraft.player.respawn();
+        }
+        response.onNext(Empty.getDefaultInstance());
+        response.onCompleted();
+    }
+
     @Override
     public void observe(Empty request, StreamObserver<ObserveResponse> plainResponse) {
         final boolean isGuiOpen = minecraft.screen != null;
         final boolean isPaused = isGuiOpen && !ScreenUtil.isAgentControllableScreen(minecraft.screen);
         final ObserveResponse.Builder builder = ObserveResponse.newBuilder()
-            .setDone(false)
+            .setDone(minecraft.screen instanceof DeathScreen)
             .setGuiOpen(isGuiOpen)
             .setPaused(isPaused || !Controller.getInstance().hasAgentControl());
 
         final ServerCallStreamObserver<ObserveResponse> response = (ServerCallStreamObserver<ObserveResponse>)plainResponse;
 
         if (!isPaused) {
+            assert this.minecraft.level != null;
+
             if (ConfigManager.getConfig().compressObservation) {
                 response.setCompression("gzip");
             }
@@ -167,7 +181,26 @@ public void observe(Empty request, StreamObserver<ObserveResponse> plainResponse
                 RenderSystem.recordRenderCall(() -> frameFuture.complete(this.screenshot.read()));
 
                 frameFuture.join(); // just join onto the gRPC thread
-                builder.setData(ByteString.copyFrom(this.screenshot.resize(Screenshot.InterpolationMethod.NEAREST_NEIGHBOR)));
+                final ByteString frame = ByteString.copyFrom(this.screenshot.resize(Screenshot.InterpolationMethod.NEAREST_NEIGHBOR));
+
+                final LocalPlayer player = this.minecraft.player;
+                assert player != null;
+
+	            final ObservationData.Builder dataBuilder = ObservationData.newBuilder()
+                    .setFrame(
+                        ObservationFrame.newBuilder()
+                            .setData(frame)
+                            .setWidth(this.screenshot.getResizeWidth())
+                            .setHeight(this.screenshot.getResizeHeight())
+                            .build()
+                    )
+                    .setAboveGround(this.minecraft.level.getBrightness(LightLayer.SKY, player.blockPosition()) >= 4)
+                    .setBiome(this.minecraft.level.registryAccess().registryOrThrow(Registry.BIOME_REGISTRY).getKey(this.minecraft.level.getBiome(player.blockPosition())).toString())
+                    .setHealth(player.getHealth())
+                    .setOnGround(player.isOnGround())
+                    .setRaining(player.isInWaterOrRain() && !player.isInWater())
+                    .setSwimming(player.isSwimming());
+                builder.setData(dataBuilder);
             }
         }
 

src/main/java/io/pyke/vitri/finorza/inference/util/Screenshot.java

@@ -23,6 +23,14 @@ public void setResizeSize(Integer resizeWidth, Integer resizeHeight) {
 		this.resizeHeight = resizeHeight != null ? resizeHeight : this.height;
 	}
 
+	public int getResizeWidth() {
+		return resizeWidth;
+	}
+
+	public int getResizeHeight() {
+		return resizeHeight;
+	}
+
 	public @NotNull ByteBuffer read() {
 		final RenderTarget target = Minecraft.getInstance().getMainRenderTarget();
 		if (target.width != width || target.height != height) {

src/main/proto/finorza/proto/v1/control.proto

@@ -8,6 +8,7 @@ option java_package = "io.pyke.vitri.finorza.inference.rpc.proto.v1";
 import "google/protobuf/empty.proto";
 import "finorza/proto/v1/keyboard.proto";
 import "finorza/proto/v1/mouse.proto";
+import "finorza/proto/v1/observation.proto";
 
 message World {
 	string id = 1;
@@ -28,7 +29,7 @@ message EnterServerRequest {
 }
 
 message ObserveResponse {
-	optional bytes data = 1;
+	optional finorza.proto.v1.ObservationData data = 1;
 	bool done = 2;
 	bool gui_open = 3;
 	bool paused = 4;
@@ -48,6 +49,7 @@ service RemoteControlService {
 	rpc EnterWorld(EnterWorldRequest) returns (google.protobuf.Empty);
 	rpc EnterServer(EnterServerRequest) returns (google.protobuf.Empty);
 
+	rpc Respawn(google.protobuf.Empty) returns (google.protobuf.Empty);
 	rpc Observe(google.protobuf.Empty) returns (ObserveResponse);
 	rpc Act(ActRequest) returns (google.protobuf.Empty);
 	rpc UpdateHumanControl(UpdateHumanControlRequest) returns (google.protobuf.Empty);

src/main/proto/finorza/proto/v1/observation.proto

@@ -0,0 +1,22 @@
+syntax = "proto3";
+
+package finorza.proto.v1;
+
+option java_multiple_files = true;
+option java_package = "io.pyke.vitri.finorza.inference.rpc.proto.v1";
+
+message ObservationFrame {
+  bytes data = 1;
+  uint32 width = 2;
+  uint32 height = 3;
+}
+
+message ObservationData {
+  ObservationFrame frame = 1;
+  optional string biome = 2;
+  optional bool above_ground = 3;
+  optional bool on_ground = 4;
+  optional bool swimming = 5;
+  optional bool raining = 6;
+  optional float health = 7;
+}

src/main/resources/fabric.mod.json

@@ -2,7 +2,7 @@
 	"schemaVersion": 1,
 	"id": "${name}",
 	"version": "${version}",
-	"name": "Finorza Inference",
+	"name": "Fiиorza",
 	"description": "${description}",
 	"authors": [
 		"Carson M",
@@ -32,6 +32,10 @@
 	"depends": {
 		"fabricloader": ">=0.8.0",
 		"minecraft": "=1.16.5",
+		"cloth-config2": "~4.17.101",
 		"fabric": "~0.42.0"
+	},
+	"recommends": {
+		"mod-menu": "~1.16"
 	}
 }