rts/CNF: Fix fixup comparison function

Previously we would implicitly convert the difference between two words
to an int, resulting in an integer overflow on 64-bit machines.

Fixes #16992

(cherry picked from commit c00c81a5)

libraries/ghc-compact/tests/T16992.hs

+import Data.Bifunctor
+import Foreign.Ptr
+import qualified Data.ByteString as BS
+import qualified Data.ByteString.Unsafe as BS
+import qualified GHC.Compact as Compact
+import qualified GHC.Compact.Serialized as CompactSerialize
+
+-- | Minimal test case for reproducing compactFixupPointers# bug for large compact regions.
+-- See Issue #16992.
+main :: IO ()
+main = do
+  let
+    large = 1024 * 1024 * 128
+    largeString = replicate large 'A'
+
+  region <- Compact.compact largeString
+
+  Just deserialized <- CompactSerialize.withSerializedCompact region $ \s -> do
+    blks <- mapM (BS.unsafePackCStringLen . bimap castPtr fromIntegral) (CompactSerialize.serializedCompactBlockList s)
+    CompactSerialize.importCompactByteStrings s blks
+
+  print (Compact.getCompact deserialized == largeString)

libraries/ghc-compact/tests/T16992.stdout

+True

libraries/ghc-compact/tests/all.T

@@ -22,3 +22,8 @@ test('compact_share', omit_ways(['ghci', 'profasm', 'profthreaded']),
 test('compact_bench', [ ignore_stdout, extra_run_opts('100') ],
                        compile_and_run, [''])
 test('T17044', normal, compile_and_run, [''])
+# N.B. Sanity check times out due to large list.
+test('T16992', [when(wordsize(32), skip), # Resource limit exceeded on 32-bit
+                high_memory_usage,
+                run_timeout_multiplier(5),
+                omit_ways(['sanity'])], compile_and_run, [''])

rts/sm/CNF.c

@@ -1016,8 +1016,9 @@ cmp_fixup_table_item (const void *e1, const void *e2)
 {
     const StgWord *w1 = e1;
     const StgWord *w2 = e2;
-
-    return *w1 - *w2;
+    if (*w1 > *w2) return +1;
+    else if (*w1 < *w2) return -1;
+    else return 0;
 }
 
 static StgWord *