Add Ghidra scripts for decompiled normalization

scripting/ghidra-extraction-scripts/clang-parser/ghidra-normalizer-header-pre.sh

+#!/bin/bash
+
+if [ $# -eq 0 ]; then
+  echo "No arguments supplied"
+  exit 1
+fi
+
+filename=$1
+
+voidline=$(grep -n "#define code   void" $filename | sed -n -e "s/^\([0-9]*\):.*$/\1/p")
+insertline=$((voidline + 1))
+echo $voidline
+echo $insertline
+sed -i -e ""$insertline"r header-footer.h" $filename
+
+>missing-commas.txt
+grep -n "undefined FUN_" $filename | sed -n -e "s/^\([0-9]*\):.*$/\1/p"&> missing-commas.txt
+for line in `cat missing-commas.txt`
+do
+  sed -i -e "${line}s/$/;/" $filename
+done
+rm missing-commas.txt
+
+sed -i -e 's/FUN_00402750(puParm2, puParm3, local_50)/FUN_00402750(puParm2, puParm3, local_50, local_50)/g' $filename
+sed -i -e 's/undefined  \[16\] FUN_00401870(ulong uParm1,undefined8 \*puParm2)/void FUN_00401870(ulong uParm1,undefined8 \*puParm2)/g' $filename
+sed -i -e 's/undefined  \[16\] FUN_004018f0(uint uParm1,undefined8 \*puParm2)/void FUN_004018f0(uint uParm1,undefined8 \*puParm2);/g' $filename
+sed -i -e 's/undefined  \[16\] FUN_00401a40(uint uParm1,undefined8 \*puParm2)/void FUN_00401a40(uint uParm1,undefined8 \*puParm2);/g' $filename
+sed -i -e 's/undefined  \[16\] FUN_/void FUN_/g' $filename
+sed -i -e 's/undefined  \[16\]FUN_/void FUN_/g' $filename

scripting/ghidra-extraction-scripts/clang-parser/ghidra-normalizer-pre.sh

+#!/bin/bash
+
+if [ $# -eq 0 ]; then
+  echo "No arguments supplied"
+  exit 1
+fi
+
+filename=$1
+
+# Ad hoc fix for base32 while
+sed -i -e 's/FUN_00402750(puParm2, puParm3, local_50)/FUN_00402750(puParm2, puParm3, local_50, local_50)/g' $filename
+sed -i -e 's/FUN_004026f0(puParm2, puParm3, local_50)/FUN_004026f0(puParm2, puParm3, local_50, local_50)/g' $filename
+
+#ad hoc fix for basename
+sed -i -e 's/undefined\[16\] FUN_00401870(ulong uParm1, undefined8 \*puParm2)/void FUN_00401870(ulong uParm1,undefined8 \*puParm2)/g' $filename
+sed -i -e 's/undefined\[16\] FUN_004018f0(uint uParm1, undefined8 \*puParm2)/void FUN_004018f0(uint uParm1, undefined8 \*puParm2)/g' $filename
+sed -i -e 's/undefined\[16\] FUN_004018f0(uint uParm1, undefined8 \*puParm2)/void FUN_004018f0(uint uParm1, undefined8 \*puParm2)/g' $filename
+sed -i -e 's/undefined\[16\] FUN_/void FUN_/g' $filename
+
+sed -i -e 's/FUN_00403940(0, 0, "missing operand");/FUN_00403940(0, 0, "missing operand", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);/g' $filename
+
+# Manual fix kill executable
+sed -i -e 's/uVar4 = FUN_00404bb0((ulong)uParm1, puParm2,//g' $filename
+sed -i -e 's/local_24 = FUN_00405948((ulong)uParm1, puParm2,//g' $filename
+sed -i -e 's/uVar4 = FUN_004049c0((ulong)uParm1, puParm2,//g' $filename
+sed -i -e 's/uVar4 = FUN_00404bf0((ulong)uParm1, puParm2,//g' $filename
+sed -i -e 's/uVar4 = FUN_00405cb0((ulong)uParm1, puParm2,//g' $filename
+sed -i -e 's/"0::1::2::3::4::5::6::7::8::9::A::B::C::D::E::F::G::H::I::J::K::M::N::O::P::Q::R::S::T::U::V::W::X::Y::Z::Lln:s:t", null_ARRAY_0060f240, 0);//g' $filename
+sed -i -e 's/"0::1::2::3::4::5::6::7::8::9::A::B::C::D::E::F::G::H::I::J::K::M::N::O::P::Q::R::S::T::U::V::W::X::Y::Z::Lln:s:t", null_ARRAY_00412440, 0);//g' $filename
+sed -i -e 's/"0::1::2::3::4::5::6::7::8::9::A::B::C::D::E::F::G::H::I::J::K::M::N::O::P::Q::R::S::T::U::V::W::X::Y::Z::Lln:s:t", null_ARRAY_0040da00, 0);//g' $filename
+sed -i -e 's/"0::1::2::3::4::5::6::7::8::9::A::B::C::D::E::F::G::H::I::J::K::M::N::O::P::Q::R::S::T::U::V::W::X::Y::Z::Lln:s:t", null_ARRAY_0040d400, 0);//g' $filename
+sed -i -e 's/"0::1::2::3::4::5::6::7::8::9::A::B::C::D::E::F::G::H::I::J::K::M::N::O::P::Q::R::S::T::U::V::W::X::Y::Z::Lln:s:t", null_ARRAY_0040e740, 0);//g' $filename
+
+# Manual fix for shred exe
+sed -i -e 's/local_40 = local_40 & 0xff00000000000000 | (ulong)(uint7)local_40;/imperfection_wrapper();/g' $filename
+
+# Manual fix for sort exe
+sed -i -e 's/auVar25 = CONCAT114.*/imperfection_wrapper();/g' $filename
+sed -i -e 's/auVar.* = CONCAT114.*/imperfection_wrapper();/g' $filename
+
+# Manual fix tail executable
+sed -i -e 's/DAT_006176a8 = (long \*)FUN_00408cf0(DAT_00617998, 0, 0xffffffffffffffff, 0x414614,/;/g' $filename
+sed -i -e 's/^[ ]*"invalid maximum number of unchanged stats between opens", 0);/;/g' $filename
+sed -i -e 's/DAT_006174b0 = (long \*)FUN_00408cc0(DAT_00617798, 0, 0xffffffffffffffff, 0x414a8c,/;/g' $filename
+sed -i -e 's/DAT_00619830 = (long \*)FUN_0040a5a0(DAT_00619b18, 0, 0xffffffffffffffff, 0x416fac,/;/g' $filename
+
+# Manual fix for tr exe
+sed -i -e 's/uint3 uVar4;/long uVar4;/g' $filename
+
+# Add an empty line after each label, to avoid clang complaints
+sed -i -e "s/^.*\(LAB_.*:\)/\1\n;/g" $filename
+sed -i -e "s/^.*\(joined_.*:\)/\1\n;/g" $filename
+
+# Fix empty case situation
+sed -i -e "s/^.*\(case 0x40:\)/\1\n;/g" $filename
+sed -i -e "s/\(case \)(undefined \*)\(0x40.*\)/\1\2/g" $filename

scripting/ghidra-extraction-scripts/clang-parser/ghidra-normalizer-wrapper.sh

+#!/bin/bash
+
+if [ $# -eq 0 ]; then
+  echo "No arguments supplied"
+  exit 1
+fi
+
+filename=$1
+iteration=0
+max_iteration=15
+
+# Invoke a script which performs ad-hoc normalization for the header.
+./ghidra-normalizer-header-pre.sh $filename
+
+# Invoke a script which performs ad-hoc normalization for this decompiled output.
+./ghidra-normalizer-pre.sh $filename
+
+# First invocation of the compiler
+clang-7 -ferror-limit=0 -S -emit-llvm -w $filename &> clang.log
+
+# While we have errors output from the compiler, continue our fixed-point purge.
+while [ -s clang.log ]; do
+
+  # Invoke the script that, on the basis of the compiler output generated at the previous iteration, purges the decompiled file.
+  ./ghidra-normalizer.sh $filename clang.log
+
+  # Invoke the compiler.
+  clang-7 -ferror-limit=0 -S -emit-llvm -w $filename &> clang.log
+  iteration=$((iteration + 1))
+
+  # Exit if we did not reach stability in an high number of runs (cheap way to detect instability).
+  if [ "$iteration" -gt "$max_iteration" ]; then
+    echo "Stability not reached"
+    break
+  fi
+done
+
+# Finally, create the output adding an adequate suffix.
+cp $filename $filename.normalized.c

scripting/ghidra-extraction-scripts/clang-parser/ghidra-normalizer.sh

+#!/bin/bash
+
+if [ $# -eq 0 ]; then
+  echo "No arguments supplied"
+  exit 1
+fi
+
+filename=$1
+logname=$2
+
+# Ad hoc fix for base32 while
+sed -i -e 's/FUN_00402750(puParm2, puParm3, local_50)/FUN_00402750(puParm2, puParm3, local_50, local_50)/g' $filename
+
+# Declare undeclared variables
+>declarations.tmp
+>declarations.txt
+>unique-declarations.txt
+grep "error: use of undeclared identifier" $logname | sed -n -e "s/^.*identifier '\([^;]*\)'.*$/\1/p" &> declarations.txt
+
+for variable in `cat declarations.txt`
+do
+  echo "long $variable;" >> declarations.tmp
+done
+
+cat declarations.tmp | sort | uniq > unique-declarations.txt
+rm declarations.txt declarations.tmp
+
+# Fix erroneous function declarations
+> mismatching-functions.txt
+> additional-mismatching-functions.txt
+grep "error: too many arguments to function call" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> many-arguments.txt
+
+grep "error: too few arguments to function call" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> few-arguments.txt
+
+cat few-arguments.txt many-arguments.txt | sort | uniq > mismatching-functions.txt
+
+# Fix erroneous function calls (indirect jumps in ghidra)
+> void-pointer.txt
+grep "error: called object type 'void' is not a function or function pointer" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> void-pointer.txt
+grep "error: called object type 'long' is not a function or function pointer" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> long-pointer.txt
+cat void-pointer.txt long-pointer.txt > function-pointer.txt
+rm void-pointer.txt long-pointer.txt
+
+# Fix incomatible types assignements
+> incompatible-type.txt
+grep "from incompatible type 'void'" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> incompatible-type.txt
+
+# Remove void parameter from functions
+sed -i -e "s/(void)/()/g" $filename
+
+# Remove variables with names containing a .
+>variables-with-point.txt
+grep "member reference base type" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> variables-with-point.txt
+
+for line in `cat variables-with-point.txt`
+do
+  sed -i -e "${line}s/\.//" $filename
+done
+rm variables-with-point.txt
+
+# Remove "where arithmetic or pointer" errors
+grep "^.* where arithmetic or pointer type is required" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> operand-types.txt
+grep "error: array type .* is not assignable" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> array-type.txt
+grep "error: indirection requires pointer operand" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> indirection.txt
+grep "error: invalid operands to binary expression" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> operands.txt
+grep "error: operand of type 'double' cannot be cast to a pointer type" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> operands-double.txt
+grep "error: pointer cannot be cast to type 'double'" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> operands-pointer-double.txt
+grep "error: subscripted value is not an array, pointer, or vector" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> subscripted-array.txt
+grep "error: void function" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> return-value.txt
+grep "error: hex escape sequence out of range" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> hex-escape.txt
+cat operand-types.txt array-type.txt indirection.txt operands.txt operands-double.txt operands-pointer-double.txt subscripted-array.txt return-value.txt incompatible-type.txt function-pointer.txt mismatching-functions.txt hex-escape.txt label-compound.txt > final-to-comment.txt
+rm operand-types.txt array-type.txt indirection.txt operands.txt operands-double.txt operands-pointer-double.txt subscripted-array.txt return-value.txt incompatible-type.txt function-pointer.txt mismatching-functions.txt hex-escape.txt label-compound.txt
+
+cat final-to-comment.txt | sort | uniq > unique-final-to-comment.txt
+rm final-to-comment.txt
+
+for line in `cat unique-final-to-comment.txt`
+do
+  broken_line=`sed -n ${line}p $filename`
+  echo "$broken_line"
+  if echo "$broken_line" | grep "if (.*{"
+  then
+    echo "matched"
+    sed -i -e "${line}s/if (.*{/if (1) {/" $filename
+  elif echo "$broken_line" | grep "while (.*{"
+  then
+    echo "matched"
+    sed -i -e "${line}s/while (.*{/while (1) {/" $filename
+  elif echo "$broken_line" | grep "if (.*)"
+  then
+    echo "matched"
+    sed -i -e "${line}s/if (.*)/if (1)/" $filename
+  elif echo "$broken_line" | grep "} while (.*);"
+  then
+    echo "matched"
+    sed -i -e "${line}s/} while (.*);/} while (1);/" $filename
+  elif echo "$broken_line" | grep "switch (.*) {"
+  then
+    echo "matched"
+    sed -i -e "${line}s/switch (.*) {/switch (1) {/" $filename
+  else
+    sed -i -e "${line}s/^/ imperfection_wrapper(); \/\//" $filename
+  fi
+
+done
+rm unique-final-to-comment.txt
+
+# Correct erroneous pointer casts
+grep "error: pointer cannot be cast to type 'double'" $logname | sed -n -e "s/^.*\.c:\([0-9]*\):.*$/\1/p" &> pointer-casts.txt
+for line in `cat pointer-casts.txt`
+do
+  sed -i -e "${line}s/(double)pplVar15/1/" $filename
+  sed -i -e "${line}s/(double)pplVar16/1/" $filename
+  sed -i -e "${line}s/(long \*\*)local_a58/local_a58/" $filename
+done
+rm pointer-casts.txt
+
+# Ad hoc fix for no statement after a goto label
+sed -i -e "s/^.*LAB_004020c6:/LAB_004020c6:\nlVar13 = lVar13;/g" $filename
+sed -i -e "s/^.*LAB_00409d1c:/LAB_00409d1c:\npuVar14 = puVar14;/g" $filename
+sed -i -e "s/^.*LAB_0040af5c:/LAB_0040af5c:\npuVar14 = puVar14;/g" $filename
+sed -i -e "s/^.*LAB_0040b47c:/LAB_0040b47c:\npuVar14 = puVar14;/g" $filename
+sed -i -e "s/^.*LAB_004029b2:/LAB_004029b2:\nbVar1 = bVar1;/g" $filename
+sed -i -e "s/^.*LAB_0040668c:/LAB_0040668c:\lVar8 =lVar8;/g" $filename
+sed -i -e "s/^.*LAB_00401a7b:/LAB_00401a7b:\npuVar14 = puVar14;/g" $filename
+sed -i -e "s/^.*LAB_00404656:/LAB_00404656:\nuVar8 = uVar8;/g" $filename
+sed -i -e "s/^.*LAB_00404656:/LAB_00404656:\nuVar8 = uVar8;/g" $filename
+sed -i -e "s/^.*LAB_00404656:/LAB_00404656:\nuVar8 = uVar8;/g" $filename
+sed -i -e "s/^.*LAB_00404656:/LAB_00404656:\nuVar8 = uVar8;/g" $filename
+
+# Add the missing variable declarations (we need to do it as last thing, to
+# to avoid changing the lines number for the other operations)
+sed -i -e "2r unique-declarations.txt" $filename
+rm unique-declarations.txt