APIv5 change: vlob_update now contains a realm_id parameter for c…

…onsistency with `vlob_create`

libparsec/crates/client/src/user/sync.rs

@@ -161,6 +161,7 @@ async fn upload_manifest(
         } else {
             use authenticated_cmds::latest::vlob_update::{Rep, Req};
             let req = Req {
+                realm_id: ops.device.user_realm_id,
                 vlob_id: ops.device.user_realm_id,
                 version: to_sync_um.version,
                 key_index: 0,
@@ -203,6 +204,8 @@ async fn upload_manifest(
                 bad_rep @ (
                     // A user realm is never shared, and hence it initial owner cannot lose access to it
                     Rep::AuthorNotAllowed
+                    // `outbound_sync_inner` ensures the realm is created before calling us
+                    | Rep::RealmNotFound
                     // The vlob must exists since we are at version > 1
                     | Rep::VlobNotFound
                     // The user realm is never supposed to do key rotation

libparsec/crates/client/src/workspace/transactions/outbound_sync.rs

@@ -508,6 +508,7 @@ async fn upload_manifest<M: RemoteManifest>(
             use authenticated_cmds::latest::vlob_update::{Rep, Req};
             let req = Req {
                 key_index,
+                realm_id: ops.realm_id,
                 vlob_id: to_upload.id(),
                 version: to_upload.version(),
                 timestamp: to_upload.timestamp(),
@@ -557,8 +558,10 @@ async fn upload_manifest<M: RemoteManifest>(
 
                 // Unexpected errors :(
                 bad_rep @ (
+                    // Already checked the realm exists when we called `CertificateOps::encrypt_for_realm`
+                    Rep::RealmNotFound
                     // Already checked the vlob exists since the manifest has version > 0
-                    Rep::VlobNotFound
+                    | Rep::VlobNotFound
                     // Don't know what to do with this status :/
                     | Rep::UnknownStatus { .. }
                 ) => {

libparsec/crates/protocol/schema/authenticated_cmds/vlob_update.json5

@@ -6,6 +6,10 @@
         "req": {
             "cmd": "vlob_update",
             "fields": [
+                {
+                    "name": "realm_id",
+                    "type": "VlobID"
+                },
                 {
                     "name": "vlob_id",
                     "type": "VlobID"
@@ -55,6 +59,9 @@
                     }
                 ]
             },
+            {
+                "status": "realm_not_found"
+            },
             {
                 "status": "vlob_not_found"
             },

libparsec/crates/protocol/tests/authenticated_cmds/v5/vlob_update.rs

@@ -12,22 +12,25 @@ use super::authenticated_cmds;
 // Request
 
 pub fn req() {
-    // Generated from Parsec v3.0.0-b.11+dev
+    // Generated from Parsec 3.2.4-a.0+dev
     // Content:
-    //   timestamp: ext(1, 946774800.0)
-    //   version: 8
-    //   blob: hex!("666f6f626172")
-    //   cmd: "vlob_update"
+    //   cmd: 'vlob_update'
+    //   realm_id: ext(2, 0x1d3353157d7d4e95ad2fdea7b3bd19c5)
+    //   vlob_id: ext(2, 0x2b5f314728134a12863da1ce49c112f6)
     //   key_index: 8
-    //   vlob_id: ext(2, hex!("2b5f314728134a12863da1ce49c112f6"))
+    //   timestamp: ext(1, 946774800000000) i.e. 2000-01-02T02:00:00Z
+    //   version: 8
+    //   blob: 0x666f6f626172
     let raw: &[u8] = hex!(
-        "87a3636d64ab766c6f625f757064617465a7766c6f625f6964d8022b5f314728134a12"
-        "863da1ce49c112f6a96b65795f696e64657808a974696d657374616d70d70100035d16"
-        "2fa2e400a776657273696f6e08a4626c6f62c406666f6f626172ae7365717565737465"
-        "725f626c6f62c0"
+        "87a3636d64ab766c6f625f757064617465a87265616c6d5f6964d8021d3353157d7d4e"
+        "95ad2fdea7b3bd19c5a7766c6f625f6964d8022b5f314728134a12863da1ce49c112f6"
+        "a96b65795f696e64657808a974696d657374616d70d70100035d162fa2e400a7766572"
+        "73696f6e08a4626c6f62c406666f6f626172"
     )
     .as_ref();
+
     let expected = authenticated_cmds::vlob_update::Req {
+        realm_id: VlobID::from_hex("1d3353157d7d4e95ad2fdea7b3bd19c5").unwrap(),
         key_index: 8,
         vlob_id: VlobID::from_hex("2b5f314728134a12863da1ce49c112f6").unwrap(),
         timestamp: "2000-1-2T01:00:00Z".parse().unwrap(),
@@ -95,6 +98,26 @@ pub fn rep_author_not_allowed() {
     p_assert_eq!(data2, expected);
 }
 
+pub fn rep_realm_not_found() {
+    // Generated from Parsec 3.2.4-a.0+dev
+    // Content:
+    //   status: 'realm_not_found'
+    let raw = hex!("81a6737461747573af7265616c6d5f6e6f745f666f756e64");
+
+    let expected = authenticated_cmds::vlob_update::Rep::RealmNotFound;
+
+    let data = authenticated_cmds::vlob_update::Rep::load(&raw).unwrap();
+
+    p_assert_eq!(data, expected);
+
+    // Also test serialization round trip
+    let raw2 = data.dump().unwrap();
+
+    let data2 = authenticated_cmds::vlob_update::Rep::load(&raw2).unwrap();
+
+    p_assert_eq!(data2, expected);
+}
+
 pub fn rep_vlob_not_found() {
     // Generated from Rust implementation (Parsec v3.0.0+dev)
     // Content:

server/parsec/backend.py

@@ -403,6 +403,7 @@ def _get_device_verify_key(device_id: DeviceID) -> VerifyKey:
                         now=event.timestamp,
                         organization_id=org_id,
                         author=event.author,
+                        realm_id=event.realm,
                         vlob_id=event.vlob_id,
                         key_index=event.key_index,
                         version=event.version,

server/parsec/components/memory/datamodel.py

@@ -100,7 +100,6 @@ class MemoryOrganization:
     greeting_attempts: dict[GreetingAttemptID, MemoryGreetingAttempt] = field(default_factory=dict)
     pki_enrollments: dict[EnrollmentID, MemoryPkiEnrollment] = field(default_factory=dict)
     realms: dict[VlobID, MemoryRealm] = field(default_factory=dict)
-    vlobs: dict[VlobID, list[MemoryVlobAtom]] = field(default_factory=dict)
     blocks: dict[BlockID, MemoryBlock] = field(default_factory=dict)
     block_store: dict[BlockID, bytes] = field(default_factory=dict, repr=False)
     # The user id is the author of the shamir recovery
@@ -515,6 +514,7 @@ class MemoryRealm:
     renames: list[MemoryRealmRename] = field(default_factory=list)
     archivings: list[MemoryRealmArchiving] = field(default_factory=list)
     last_vlob_timestamp: DateTime | None = None
+    vlobs: dict[VlobID, list[MemoryVlobAtom]] = field(default_factory=dict)
 
     def get_current_role_for(self, user_id: UserID) -> RealmRole | None:
         for role in reversed(self.roles):
@@ -579,7 +579,6 @@ class MemoryRealmUserChange:
 
 @dataclass(slots=True)
 class MemoryVlobAtom:
-    realm_id: VlobID
     vlob_id: VlobID
     key_index: int
     version: int

server/parsec/components/memory/organization.py

@@ -281,8 +281,9 @@ def _stats(
         for realm in org.realms.values():
             if realm.created_on <= at:
                 realms += 1
-        for vlob in org.vlobs.values():
-            metadata_size += sum(len(atom.blob) for atom in vlob if atom.created_on <= at)
+        for realm in org.realms.values():
+            for vlob in realm.vlobs.values():
+                metadata_size += sum(len(atom.blob) for atom in vlob if atom.created_on <= at)
         for block in org.blocks.values():
             if block.created_on <= at:
                 data_size += block.block_size

server/parsec/components/memory/vlob.py

@@ -114,7 +114,7 @@ async def create(
             if len(realm.key_rotations) != key_index:
                 return BadKeyIndex(last_realm_certificate_timestamp=realm_topic_last_timestamp)
 
-            if vlob_id in org.vlobs:
+            if vlob_id in realm.vlobs:
                 return VlobCreateBadOutcome.VLOB_ALREADY_EXISTS
 
             maybe_error = timestamps_in_the_ballpark(timestamp, now)
@@ -160,15 +160,14 @@ async def create(
                     realm.last_vlob_timestamp = max(previous_last_vlob_timestamp, timestamp)
 
             vlob_atom = MemoryVlobAtom(
-                realm_id=realm_id,
                 vlob_id=vlob_id,
                 key_index=key_index,
                 version=1,
                 blob=blob,
                 author=author,
                 created_on=timestamp,
             )
-            org.vlobs[vlob_id] = [vlob_atom]
+            realm.vlobs[vlob_id] = [vlob_atom]
             realm_change_checkpoint = len(realm.vlob_updates) + 1
             realm.vlob_updates.append(
                 MemoryRealmVlobUpdate(
@@ -196,6 +195,7 @@ async def update(
         now: DateTime,
         organization_id: OrganizationID,
         author: DeviceID,
+        realm_id: VlobID,
         vlob_id: VlobID,
         key_index: int,
         version: int,
@@ -229,10 +229,14 @@ async def update(
                 return VlobUpdateBadOutcome.AUTHOR_REVOKED
 
             try:
-                vlobs = org.vlobs[vlob_id]
+                realm = org.realms[realm_id]
+            except KeyError:
+                return VlobUpdateBadOutcome.REALM_NOT_FOUND
+
+            try:
+                vlobs = realm.vlobs[vlob_id]
             except KeyError:
                 return VlobUpdateBadOutcome.VLOB_NOT_FOUND
-            realm_id = vlobs[0].realm_id
 
             async with org.topics_lock(read=[("realm", realm_id)]) as (realm_topic_last_timestamp,):
                 realm = org.realms[realm_id]
@@ -298,7 +302,6 @@ async def update(
 
                 version = len(vlobs) + 1
                 vlob_atom = MemoryVlobAtom(
-                    realm_id=realm.realm_id,
                     vlob_id=vlob_id,
                     key_index=key_index,
                     version=version,
@@ -376,7 +379,7 @@ async def read_versions(
         output = []
         for vlob_id, version in items:
             try:
-                vlob = org.vlobs[vlob_id]
+                vlob = realm.vlobs[vlob_id]
             except KeyError:
                 # An unkown vlob ID was provided
                 continue
@@ -454,7 +457,7 @@ async def read_batch(
         output = []
         for vlob_id in vlobs:
             try:
-                vlob = org.vlobs[vlob_id]
+                vlob = realm.vlobs[vlob_id]
             except KeyError:
                 # An unkown vlob ID was provided
                 continue
@@ -542,11 +545,12 @@ async def poll_changes(
     @override
     async def test_dump_vlobs(
         self, organization_id: OrganizationID
-    ) -> dict[VlobID, list[tuple[DeviceID, DateTime, VlobID, bytes]]]:
+    ) -> dict[VlobID, dict[VlobID, list[tuple[DeviceID, DateTime, bytes]]]]:
         org = self._data.organizations[organization_id]
         return {
-            vlob_id: [
-                (atom.author, atom.created_on, atom.realm_id, atom.blob) for atom in vlob_atoms
-            ]
-            for vlob_id, vlob_atoms in org.vlobs.items()
+            realm_id: {
+                vlob_id: [(atom.author, atom.created_on, atom.blob) for atom in vlob_atoms]
+                for vlob_id, vlob_atoms in realm.vlobs.items()
+            }
+            for realm_id, realm in org.realms.items()
         }

server/parsec/components/postgresql/vlob.py

@@ -123,6 +123,7 @@ async def update(
         now: DateTime,
         organization_id: OrganizationID,
         author: DeviceID,
+        realm_id: VlobID,
         vlob_id: VlobID,
         key_index: int,
         version: int,
@@ -143,6 +144,7 @@ async def update(
             now,
             organization_id,
             author,
+            realm_id,
             vlob_id,
             key_index,
             version,
@@ -191,5 +193,5 @@ async def read_versions(
     @no_transaction
     async def test_dump_vlobs(
         self, conn: AsyncpgConnection, organization_id: OrganizationID
-    ) -> dict[VlobID, list[tuple[DeviceID, DateTime, VlobID, bytes]]]:
+    ) -> dict[VlobID, dict[VlobID, list[tuple[DeviceID, DateTime, bytes]]]]:
         return await vlob_test_dump_vlobs(conn, organization_id)

server/parsec/components/postgresql/vlob_test_dump_vlobs.py

@@ -33,21 +33,28 @@
 
 async def vlob_test_dump_vlobs(
     conn: AsyncpgConnection, organization_id: OrganizationID
-) -> dict[VlobID, list[tuple[DeviceID, DateTime, VlobID, bytes]]]:
+) -> dict[VlobID, dict[VlobID, list[tuple[DeviceID, DateTime, bytes]]]]:
     rows = await conn.fetch(*_q_dump_vlobs(organization_id=organization_id.str))
-    result: dict[VlobID, list[tuple[DeviceID, DateTime, VlobID, bytes]]] = {}
+    realms: dict[VlobID, dict[VlobID, list[tuple[DeviceID, DateTime, bytes]]]] = {}
     for row in rows:
         vlob_id = VlobID.from_hex(row["vlob_id"])
         realm_id = VlobID.from_hex(row["realm_id"])
         author = DeviceID.from_hex(row["author"])
-        if vlob_id not in result:
-            result[vlob_id] = []
-        result[vlob_id].append(
-            (
-                author,
-                row["created_on"],
-                realm_id,
-                row["blob"],
-            )
+
+        try:
+            realm = realms[realm_id]
+        except KeyError:
+            realm = realms[realm_id] = {}
+
+        item = (
+            author,
+            row["created_on"],
+            row["blob"],
         )
-    return result
+
+        try:
+            realm[vlob_id].append(item)
+        except KeyError:
+            realm[vlob_id] = [item]
+
+    return realms