Pack TBN into 6 bytes

Instead of using 8 bytes (1 unused) with 8-bit precision for individual
components, we now use 10-10-10 format for normal and 8-8 format for
tangent (using octahedral encoding). This reduces vertex size from 20 to
16 bytes and improves normal quality.

While it is possible to pack the normal and tangent further (e.g. normal
could use 20 bits instead of 30 via octahedral encoding; tangent can use
angle or diamond encoding to fit in the remaining 10 bits), due to
padding this would not help reduce vertex size further atm.

src/scene.cpp

@@ -100,11 +100,10 @@ static bool loadObj(std::vector<Vertex>& vertices, const char* path)
 			v.vx = meshopt_quantizeHalf(obj->positions[gi.p * 3 + 0]);
 			v.vy = meshopt_quantizeHalf(obj->positions[gi.p * 3 + 1]);
 			v.vz = meshopt_quantizeHalf(obj->positions[gi.p * 3 + 2]);
-			v.nx = uint8_t(obj->normals[gi.n * 3 + 0] * 127.f + 127.5f);
-			v.ny = uint8_t(obj->normals[gi.n * 3 + 1] * 127.f + 127.5f);
-			v.nz = uint8_t(obj->normals[gi.n * 3 + 2] * 127.f + 127.5f);
-			v.tx = v.ty = v.tz = 127;
-			v.tw = 254;
+			v.tp = 0;
+			v.np = (meshopt_quantizeSnorm(obj->normals[gi.n * 3 + 0], 10) + 511) |
+			       (meshopt_quantizeSnorm(obj->normals[gi.n * 3 + 1], 10) + 511) << 10 |
+			       (meshopt_quantizeSnorm(obj->normals[gi.n * 3 + 1], 10) + 511) << 20;
 			v.tu = meshopt_quantizeHalf(obj->texcoords[gi.t * 2 + 0]);
 			v.tv = meshopt_quantizeHalf(obj->texcoords[gi.t * 2 + 1]);
 		}
@@ -154,7 +153,7 @@ static void appendMesh(Geometry& result, std::vector<Vertex>& vertices, std::vec
 	for (size_t i = 0; i < vertices.size(); ++i)
 	{
 		Vertex& v = vertices[i];
-		normals[i] = vec3(v.nx / 127.f - 1.f, v.ny / 127.f - 1.f, v.nz / 127.f - 1.f);
+		normals[i] = vec3((v.np & 1023) / 511.f - 1.f, ((v.np >> 10) & 1023) / 511.f - 1.f, ((v.np >> 20) & 1023) / 511.f - 1.f);
 	}
 
 	vec3 center = vec3(0);
@@ -311,9 +310,11 @@ static void loadVertices(std::vector<Vertex>& vertices, const cgltf_primitive& p
 
 		for (size_t j = 0; j < vertexCount; ++j)
 		{
-			vertices[j].nx = uint8_t(scratch[j * 3 + 0] * 127.f + 127.5f);
-			vertices[j].ny = uint8_t(scratch[j * 3 + 1] * 127.f + 127.5f);
-			vertices[j].nz = uint8_t(scratch[j * 3 + 2] * 127.f + 127.5f);
+			float nx = scratch[j * 3 + 0], ny = scratch[j * 3 + 1], nz = scratch[j * 3 + 2];
+
+			vertices[j].np = (meshopt_quantizeSnorm(nx, 10) + 511) |
+			                 (meshopt_quantizeSnorm(ny, 10) + 511) << 10 |
+			                 (meshopt_quantizeSnorm(nz, 10) + 511) << 20;
 		}
 	}
 
@@ -324,10 +325,13 @@ static void loadVertices(std::vector<Vertex>& vertices, const cgltf_primitive& p
 
 		for (size_t j = 0; j < vertexCount; ++j)
 		{
-			vertices[j].tx = uint8_t(scratch[j * 4 + 0] * 127.f + 127.5f);
-			vertices[j].ty = uint8_t(scratch[j * 4 + 1] * 127.f + 127.5f);
-			vertices[j].tz = uint8_t(scratch[j * 4 + 2] * 127.f + 127.5f);
-			vertices[j].tw = uint8_t(scratch[j * 4 + 3] * 127.f + 127.5f);
+			float tx = scratch[j * 4 + 0], ty = scratch[j * 4 + 1], tz = scratch[j * 4 + 2];
+			float tsum = fabsf(tx) + fabsf(ty) + fabsf(tz);
+			float tu = tz >= 0 ? tx / tsum : (1 - fabsf(ty / tsum)) * (tx >= 0 ? 1 : -1);
+			float tv = tz >= 0 ? ty / tsum : (1 - fabsf(tx / tsum)) * (ty >= 0 ? 1 : -1);
+
+			vertices[j].tp = (meshopt_quantizeSnorm(tu, 8) + 127) | (meshopt_quantizeSnorm(tv, 8) + 127) << 8;
+			vertices[j].np |= (scratch[j * 4 + 3] >= 0 ? 0 : 1) << 30;
 		}
 	}
 

src/scene.h

@@ -46,9 +46,9 @@ struct alignas(16) MeshDraw
 
 struct Vertex
 {
-	uint16_t vx, vy, vz, vw;
-	uint8_t nx, ny, nz, nw;
-	uint8_t tx, ty, tz, tw;
+	uint16_t vx, vy, vz;
+	uint16_t tp; // packed tangent: 8-8 octahedral
+	uint32_t np; // packed normal: 10-10-10-2 vector + bitangent sign
 	uint16_t tu, tv;
 };
 

src/shaders/math.h

@@ -75,3 +75,10 @@ float gradientNoise(vec2 uv)
 {
 	return fract(52.9829189 * fract(dot(uv, vec2(0.06711056, 0.00583715))));
 }
+
+void unpackTBN(uint np, uint tp, out vec3 normal, out vec4 tangent)
+{
+	normal = ((ivec3(np) >> ivec3(0, 10, 20)) & ivec3(1023)) / 511.0 - 1.0;
+	tangent.xyz = decodeOct(((ivec2(tp) >> ivec2(0, 8)) & ivec2(255)) / 127.0 - 1.0);
+	tangent.w = (np & (1 << 30)) != 0 ? -1.0 : 1.0;
+}

src/shaders/mesh.h

@@ -2,9 +2,9 @@
 
 struct Vertex
 {
-	float16_t vx, vy, vz, vw;
-	uint8_t nx, ny, nz, nw;
-	uint8_t tx, ty, tz, tw;
+	float16_t vx, vy, vz;
+	uint16_t tp; // packed tangent: 8-8 octahedral
+	uint np;     // packed normal: 10-10-10-2 vector + bitangent sign
 	float16_t tu, tv;
 };
 

src/shaders/mesh.vert.glsl

@@ -43,10 +43,12 @@ void main()
 
 	uint vi = gl_VertexIndex;
 	vec3 position = vec3(vertices[vi].vx, vertices[vi].vy, vertices[vi].vz);
-	vec3 normal = vec3(int(vertices[vi].nx), int(vertices[vi].ny), int(vertices[vi].nz)) / 127.0 - 1.0;
-	vec4 tangent = vec4(int(vertices[vi].tx), int(vertices[vi].ty), int(vertices[vi].tz), int(vertices[vi].tw)) / 127.0 - 1.0;
 	vec2 texcoord = vec2(vertices[vi].tu, vertices[vi].tv);
 
+	vec3 normal;
+	vec4 tangent;
+	unpackTBN(vertices[vi].np, uint(vertices[vi].tp), normal, tangent);
+
 	normal = rotateQuat(normal, meshDraw.orientation);
 	tangent.xyz = rotateQuat(tangent.xyz, meshDraw.orientation);
 

src/shaders/meshlet.mesh.glsl

@@ -121,10 +121,12 @@ void main()
 		uint vi = meshletData[vertexOffset + i] + baseVertex;
 
 		vec3 position = vec3(vertices[vi].vx, vertices[vi].vy, vertices[vi].vz);
-		vec3 normal = vec3(int(vertices[vi].nx), int(vertices[vi].ny), int(vertices[vi].nz)) / 127.0 - 1.0;
-		vec4 tangent = vec4(int(vertices[vi].tx), int(vertices[vi].ty), int(vertices[vi].tz), int(vertices[vi].tw)) / 127.0 - 1.0;
 		vec2 texcoord = vec2(vertices[vi].tu, vertices[vi].tv);
 
+		vec3 normal;
+		vec4 tangent;
+		unpackTBN(vertices[vi].np, uint(vertices[vi].tp), normal, tangent);
+
 		normal = rotateQuat(normal, meshDraw.orientation);
 		tangent.xyz = rotateQuat(tangent.xyz, meshDraw.orientation);