[decompiler] Update vector ops, reduce casts (#3849)

Update the decompiler to use the new vf macros.

Also, fix a bunch of silly casting issues where accessing inline fields
with an offset of 0 would be better than a cast:

![image](https://github.com/user-attachments/assets/885bbb07-634f-47b8-99f5-5a947941cdde)

---------

Co-authored-by: water111 <[email protected]>

decompiler/IR2/Form.cpp

@@ -2148,6 +2148,15 @@ DerefToken to_token(const FieldReverseLookupOutput::Token& in) {
   }
 }
 
+std::vector<DerefToken> to_tokens(const std::vector<FieldReverseLookupOutput::Token>& in) {
+  std::vector<DerefToken> ret;
+  ret.reserve(in.size());
+  for (auto& x : in) {
+    ret.push_back(to_token(x));
+  }
+  return ret;
+}
+
 DerefElement::DerefElement(Form* base, bool is_addr_of, DerefToken token)
     : m_base(base), m_is_addr_of(is_addr_of), m_tokens({std::move(token)}) {
   m_base->parent_element = this;

decompiler/IR2/Form.h

@@ -1274,6 +1274,7 @@ class DerefToken {
 };
 
 DerefToken to_token(const FieldReverseLookupOutput::Token& in);
+std::vector<DerefToken> to_tokens(const std::vector<FieldReverseLookupOutput::Token>& in);
 
 class DerefElement : public FormElement {
  public:

decompiler/IR2/FormExpressionAnalysis.cpp

@@ -489,6 +489,38 @@ Form* cast_form(Form* in,
   return pool.form<CastElement>(new_type, in);
 }
 
+Form* cast_form_from(Form* in,
+                     const TypeSpec& old_type,
+                     const TypeSpec& new_type,
+                     FormPool& pool,
+                     const Env& env,
+                     bool tc_pass = false) {
+  auto& ts = env.dts->ts;
+
+  // sometimes, accessing a field is a no-op but changes the type. For example, accessing an inlined
+  // structure at the start of a structure. To detect this, look up all the possible derefs with no
+  // deref or offset, then find the highest scoring one that is the right type.
+  FieldReverseLookupInput lookup_input;
+  lookup_input.deref = std::nullopt;
+  lookup_input.offset = 0;
+  lookup_input.stride = 0;
+  lookup_input.base_type = old_type;
+  auto lookup_result = ts.reverse_field_multi_lookup(lookup_input);
+  if (lookup_result.success) {
+    for (auto& result : lookup_result.results) {
+      if (result.result_type == new_type) {
+        // it's a match!
+        auto form = pool.form<DerefElement>(in, result.addr_of, to_tokens(result.tokens));
+        form->try_as_element<DerefElement>()->inline_nested();
+        return form;
+      }
+    }
+  }
+
+  // fall back to normal casts.
+  return cast_form(in, new_type, pool, env, tc_pass);
+}
+
 /*!
  * Pop each variable in the input list into a form. The variables should be given in the order
  * they are evaluated in the source. It is safe to put the result of these in the same expression.
@@ -660,7 +692,7 @@ Form* make_cast_if_needed(Form* in,
     return try_cast_simplify(in, TypeSpec("time-frame"), pool, env, true);
   }
 
-  return cast_form(in, out_type, pool, env);
+  return cast_form_from(in, in_type, out_type, pool, env);
 }
 
 std::vector<Form*> make_casts_if_needed(const std::vector<Form*>& in,
@@ -1605,7 +1637,8 @@ void SimpleExpressionElement::update_from_stack_vector_plus_minus_cross(
   for (int i = 0; i < 3; i++) {
     auto arg_type = env.get_types_before_op(m_my_idx).get(m_expr.get_arg(i).var().reg());
     if (arg_type.typespec() != TypeSpec("vector")) {
-      popped_args.at(i) = cast_form(popped_args.at(i), TypeSpec("vector"), pool, env);
+      popped_args.at(i) =
+          cast_form_from(popped_args.at(i), arg_type.typespec(), TypeSpec("vector"), pool, env);
     }
   }
 
@@ -1629,7 +1662,8 @@ void SimpleExpressionElement::update_from_stack_vector_plus_float_times(
     auto arg_type = env.get_types_before_op(m_my_idx).get(m_expr.get_arg(i).var().reg());
     TypeSpec desired_type(i == 3 ? "float" : "vector");
     if (arg_type.typespec() != desired_type) {
-      popped_args.at(i) = cast_form(popped_args.at(i), desired_type, pool, env);
+      popped_args.at(i) =
+          cast_form_from(popped_args.at(i), arg_type.typespec(), desired_type, pool, env);
     }
   }
 
@@ -1654,7 +1688,8 @@ void SimpleExpressionElement::update_from_stack_vector_float_product(
     auto arg_type = env.get_types_before_op(m_my_idx).get(m_expr.get_arg(i).var().reg());
     TypeSpec desired_type(i == 2 ? "float" : "vector");
     if (arg_type.typespec() != desired_type) {
-      popped_args.at(i) = cast_form(popped_args.at(i), desired_type, pool, env);
+      popped_args.at(i) =
+          cast_form_from(popped_args.at(i), arg_type.typespec(), desired_type, pool, env);
     }
   }
 
@@ -1680,7 +1715,8 @@ void SimpleExpressionElement::update_from_stack_vectors_in_common(FixedOperatorK
   for (int i = 0; i < m_expr.args(); i++) {
     auto arg_type = env.get_types_before_op(m_my_idx).get(m_expr.get_arg(i).var().reg());
     if (arg_type.typespec() != TypeSpec("vector")) {
-      popped_args.at(i) = cast_form(popped_args.at(i), TypeSpec("vector"), pool, env);
+      popped_args.at(i) =
+          cast_form_from(popped_args.at(i), arg_type.typespec(), TypeSpec("vector"), pool, env);
     }
   }
 
@@ -3554,8 +3590,8 @@ void FunctionCallElement::update_from_stack(const Env& env,
 
       auto desired_arg_type = function_type.get_arg(arg_id);
       if (env.dts->should_attempt_cast_simplify(desired_arg_type, actual_arg_type)) {
-        arg_forms.push_back(cast_form(val, desired_arg_type, pool, env,
-                                      env.dts->ts.tc(desired_arg_type, actual_arg_type)));
+        arg_forms.push_back(cast_form_from(val, actual_arg_type, desired_arg_type, pool, env,
+                                           env.dts->ts.tc(desired_arg_type, actual_arg_type)));
       } else {
         arg_forms.push_back(val);
       }
@@ -4449,11 +4485,11 @@ void DerefElement::update_from_stack(const Env& env,
 }
 
 void DerefElement::inline_nested() {
-  auto as_deref = dynamic_cast<DerefElement*>(m_base->try_as_single_element());
-  if (as_deref) {
-    if (!m_is_addr_of && !as_deref->is_addr_of()) {
-      m_tokens.insert(m_tokens.begin(), as_deref->tokens().begin(), as_deref->tokens().end());
-      m_base = as_deref->m_base;
+  auto inner = dynamic_cast<DerefElement*>(m_base->try_as_single_element());
+  if (inner) {
+    if (!inner->is_addr_of()) {
+      m_tokens.insert(m_tokens.begin(), inner->tokens().begin(), inner->tokens().end());
+      m_base = inner->m_base;
     }
   }
 }
@@ -6001,7 +6037,7 @@ void ReturnElement::push_to_stack(const Env& env, FormPool& pool, FormStack& sta
         as_cast->set_type(func_type);
         return_code->push_back(as_cast);
       } else {
-        return_code = cast_form(return_code, func_type, pool, env);
+        return_code = cast_form_from(return_code, *return_type, func_type, pool, env);
         return_code->parent_element = this;
       }
     }

decompiler/IR2/OpenGoalMapping.cpp

@@ -207,6 +207,13 @@ std::string OpenGOALAsm::full_function_name() {
       func_name = fmt::format(fmt::runtime(func_name), bc);
     }
   }
+
+  // Handle destination masks
+  if (func.allows_modifier(MOD::DEST_MASK) && m_instr.cop2_dest != 0xff &&
+      m_instr.cop2_dest != 15) {
+    func_name += m_instr.cop2_dest_to_char();
+  }
+
   return func_name;
 }
 
@@ -270,13 +277,6 @@ std::vector<goos::Object> OpenGOALAsm::get_args(const std::vector<DecompilerLabe
     args.push_back(pretty_print::to_symbol("acc"));
   }
 
-  // Handle destination masks
-  if (func.allows_modifier(MOD::DEST_MASK) && m_instr.cop2_dest != 0xff &&
-      m_instr.cop2_dest != 15) {
-    named_args.push_back(
-        pretty_print::to_symbol(fmt::format(":mask #b{:b}", m_instr.cop2_dest_mask_intel())));
-  }
-
   // Some functions are configured, or its easiest to swap the source args
   // NOTE - this currently assumes it is the first two args that must be swapped
   if (func.allows_modifier(MOD::SWAP_FIRST_TWO_SOURCE_ARGS)) {

goal_src/jak3/engine/ai/enemy.gc

@@ -269,7 +269,7 @@
     (if *target*
         (look-at!
           (-> *target* neck)
-          (the-as vector (-> this root root-prim prim-core))
+          (-> this root root-prim prim-core world-sphere)
           (if (logtest? (-> this enemy-flags) (enemy-flag cam-attack-mode))
               'attacking
               )
@@ -2896,7 +2896,7 @@
          (s3-0 (new 'stack-no-clear 'matrix))
          (s5-0 (quaternion->matrix (new 'stack-no-clear 'matrix) (-> this root quat)))
          )
-    (quaternion-normalize! (quaternion*! s1-0 (the-as quaternion (-> s2-0 0)) s1-0))
+    (quaternion-normalize! (quaternion*! s1-0 (-> s2-0 0 quat) s1-0))
     (quaternion->matrix s3-0 s1-0)
     (if (logtest? (-> s4-0 ragdoll-flags) (ragdoll-flag mirror))
         (matrix*! s3-0 s3-0 (-> s4-0 mirror))

goal_src/jak3/engine/ambient/ambient.gc

@@ -505,7 +505,7 @@
                         )
                       (begin
                         (dotimes (s5-2 4)
-                          (when (and (string-charp= (-> self message name) (the-as (pointer uint8) (-> *sound-iop-info* stream-name s5-2)))
+                          (when (and (string-charp= (-> self message name) (-> *sound-iop-info* stream-name s5-2 name))
                                      (= (-> self voice-id) (-> *sound-iop-info* stream-id s5-2))
                                      (logtest? (-> *sound-iop-info* stream-status s5-2) (stream-status ss6))
                                      )

goal_src/jak3/engine/anim/aligner.gc

@@ -87,25 +87,19 @@
       (set! (-> v1-25 0 fvec quad) a2-6)
       (set! (-> v1-25 0 trans quad) a3-3)
       )
-    (vector*! (the-as vector (-> this transform)) (-> s5-1 bone transform trans) (-> this process root scale))
-    )
-  (vector-!
-    (the-as vector (-> this delta))
-    (the-as vector (-> this transform))
-    (the-as vector (-> this transform 1))
+    (vector*! (-> this transform 0 trans) (-> s5-1 bone transform trans) (-> this process root scale))
     )
+  (vector-! (-> this delta trans) (-> this transform 0 trans) (-> this transform 1 trans))
   (set-vector!
     (-> this align scale)
-    (vector-length (the-as vector (-> this matrix)))
+    (vector-length (-> this matrix 0 rvec))
     (vector-length (-> this matrix 0 uvec))
     (vector-length (-> this matrix 0 fvec))
     1.0
     )
   (vector-! (-> this delta scale) (-> this align scale) (-> this transform 1 scale))
   (let ((a2-7 (matrix-inv-scale! (new 'stack-no-clear 'matrix) (-> this align scale))))
-    (quaternion-normalize!
-      (matrix->quaternion (-> this align quat) (matrix*! a2-7 (the-as matrix (-> this matrix)) a2-7))
-      )
+    (quaternion-normalize! (matrix->quaternion (-> this align quat) (matrix*! a2-7 (-> this matrix 0) a2-7)))
     )
   (let ((a1-27 (quaternion-inverse! (new 'stack-no-clear 'quaternion) (the-as quaternion (-> this transform 1 rot))))
         )

goal_src/jak3/engine/anim/joint-exploder.gc

@@ -399,9 +399,9 @@
                        (v1-8
                          (set! sv-48 (remove-joint-from-list this arg0 s2-0))
                          (set! sv-64 (-> this lists data))
-                         (add-joint-to-list this (the-as joint-exploder-list sv-64) s2-0)
+                         (add-joint-to-list this (-> sv-64 0) s2-0)
                          (set! s2-0 sv-48)
-                         (update-bbox-for-joint this (the-as joint-exploder-list sv-64) sv-80)
+                         (update-bbox-for-joint this (-> sv-64 0) sv-80)
                          (set! (-> arg0 bbox-valid?) s0-0)
                          (set! (-> arg0 bbox min quad) (-> s1-0 min quad))
                          (vector-copy! (-> arg0 bbox max) (-> s1-0 max))

goal_src/jak3/engine/anim/joint-mod-h.gc

@@ -375,7 +375,7 @@ Unlike jak 2, this actually multiplies the scale, instead of adding."
   "Callback for joint-mod-add-local. See comment on that type."
   (let ((s4-0 (the-as joint-mod-add-local (-> bone-cspace param1))))
     (if (logtest? (-> s4-0 flags) (joint-mod-base-flags trans))
-        (vector+! (-> joint-transform trans) (-> joint-transform trans) (the-as vector (-> s4-0 transform)))
+        (vector+! (-> joint-transform trans) (-> joint-transform trans) (-> s4-0 transform trans))
         )
     (when (logtest? (-> s4-0 flags) (joint-mod-base-flags quat))
       (quaternion*! (-> joint-transform quat) (-> joint-transform quat) (-> s4-0 transform quat))
@@ -485,9 +485,9 @@ Then, apply the normal parented transform."
   "Callback for joint-mod-blend-local. See comment on that type."
   (let ((gp-0 (the-as joint-mod-blend-local (-> bone-cspace param1))))
     (vector-lerp!
-      (the-as vector (-> gp-0 blend-transform))
+      (-> gp-0 blend-transform trans)
       (-> joint-transform trans)
-      (the-as vector (-> gp-0 transform))
+      (-> gp-0 transform trans)
       (-> gp-0 blend)
       )
     (vector-lerp!
@@ -560,18 +560,13 @@ Then, apply the normal parented transform."
       (matrix*! (-> s3-0 mat2) (-> s3-0 mat1) (-> bone-cspace parent bone transform))
       (set-vector!
         (-> s3-0 vec)
-        (vector-length (the-as vector (-> s3-0 mat2)))
+        (vector-length (-> s3-0 mat2 rvec))
         (vector-length (-> s3-0 mat2 uvec))
         (vector-length (-> s3-0 mat2 fvec))
         1.0
         )
       (vector*! (-> gp-0 blend-transform scale) (-> s3-0 vec) (-> joint-transform scale))
-      (vector-lerp!
-        (the-as vector (-> gp-0 blend-transform))
-        (-> s3-0 mat2 trans)
-        (the-as vector (-> gp-0 transform))
-        f30-0
-        )
+      (vector-lerp! (-> gp-0 blend-transform trans) (-> s3-0 mat2 trans) (-> gp-0 transform trans) f30-0)
       (vector-lerp! (-> gp-0 blend-transform scale) (-> gp-0 blend-transform scale) (-> gp-0 transform scale) f26-0)
       (quaternion-slerp!
         (-> gp-0 blend-transform quat)