diff --git a/hadrian/src/Settings/Packages.hs b/hadrian/src/Settings/Packages.hs
index 6c994b5995827a443c41150517c7ed18a12f01bc..7c4041d6f53f7019eeb7550b8d68cf95c74ba83c 100644
--- a/hadrian/src/Settings/Packages.hs
+++ b/hadrian/src/Settings/Packages.hs
@@ -309,10 +309,8 @@ rtsPackageArgs = package rts ? do
 
           -- We're after pur performance here. So make sure fast math and
           -- vectorization is enabled.
-          , input "**/xxhash.c" ? pure
-            [ "-O3"
-            , "-ffast-math"
-            , "-ftree-vectorize" ]
+          , input "**/Hash.c" ? pure
+            [ "-O3" ]
 
             , inputs ["**/Evac.c", "**/Evac_thr.c"] ? arg "-funroll-loops"
 
diff --git a/rts/Hash.c b/rts/Hash.c
index 1588dfc97527d13a534597df0be7b2e8a1a5fdd1..5a688ccb60fc71c7b904cff15c5b2fbebc9eb4c2 100644
--- a/rts/Hash.c
+++ b/rts/Hash.c
@@ -13,6 +13,19 @@
 
 #include "Hash.h"
 #include "RtsUtils.h"
+
+/* This file needs to be compiled with vectorization enabled.  Unfortunately
+   since we compile these things these days with cabal we can no longer
+   specify optimization per file.  So we have to resort to pragmas.  */
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC push_options
+#pragma GCC optimize ("O3")
+#endif
+
+#define XXH_NAMESPACE __rts_
+#define XXH_STATIC_LINKING_ONLY   /* access advanced declarations */
+#define XXH_PRIVATE_API
+
 #include "xxhash.h"
 
 #include <string.h>
diff --git a/rts/ghc.mk b/rts/ghc.mk
index a1610fc1f4295cac54a3ba10d5c1a78a8706e43d..ef5fc7bbb4c8eca8c35f0ef863ce5a47630c03f6 100644
--- a/rts/ghc.mk
+++ b/rts/ghc.mk
@@ -45,7 +45,10 @@ else
 ALL_DIRS += posix
 endif
 
-rts_C_SRCS := $(wildcard rts/*.c $(foreach dir,$(ALL_DIRS),rts/$(dir)/*.c))
+tmp_rts_C_SRCS := $(wildcard rts/*.c $(foreach dir,$(ALL_DIRS),rts/$(dir)/*.c))
+# We shouldn't include this file in the binary, it's a common function so
+# it will likely clash with something later. See #19948
+rts_C_SRCS = $(filter-out rts/xxhash.c, $(tmp_rts_C_SRCS))
 rts_C_HOOK_SRCS := $(wildcard rts/hooks/*.c)
 rts_CMM_SRCS := $(wildcard rts/*.cmm)
 
@@ -428,7 +431,7 @@ rts/RtsUtils_CC_OPTS += -DTargetVendor=\"$(TargetVendor_CPP)\"
 rts/RtsUtils_CC_OPTS += -DGhcUnregisterised=\"$(GhcUnregisterised)\"
 rts/RtsUtils_CC_OPTS += -DTablesNextToCode=\"$(TablesNextToCode)\"
 #
-rts/xxhash_CC_OPTS += -O3 -ffast-math -ftree-vectorize
+rts/Hash_CC_OPTS += -O3
 
 # Compile various performance-critical pieces *without* -fPIC -dynamic
 # even when building a shared library.  If we don't do this, then the
diff --git a/rts/rts.cabal.in b/rts/rts.cabal.in
index a24e4ff1f80622e8380bbfe7421ab6b0d908dabe..cebf6795f352932050d07cc6fdffd938de789ec1 100644
--- a/rts/rts.cabal.in
+++ b/rts/rts.cabal.in
@@ -497,10 +497,14 @@ library
                sm/Scav_thr.c
                sm/Storage.c
                sm/Sweep.c
-               xxhash.c
                fs.c
                -- I wish we had wildcards..., this would be:
                -- *.c hooks/**/*.c sm/**/*.c eventlog/**/*.c linker/**/*.c
+
+    extra-source-files:
+               -- This file needs to be in the package but shouldn't be compiled on its own.
+               xxhash.c
+
     if os(windows)
        c-sources: win32/AsyncIO.c
                   win32/AwaitEvent.c