From 20784d3da2eb1bb22123b8bbf2ffa8a721c5a5f7 Mon Sep 17 00:00:00 2001
From: Vincent Rabaud <vrabaud@google.com>
Date: Thu, 20 Jul 2023 15:29:34 +0200
Subject: [PATCH 1/2] Fix undefined behavior with wrong function pointers
 called.

Details here: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=58006
runtime error: call to function (unknown) through pointer to incorrect function type 'void (*)(const unsigned char **, const int *, unsigned char **, const int *, int, int)'
---
 modules/core/src/channels.cpp | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/channels.cpp b/modules/core/src/channels.cpp
index 6ceed44a28..fd6600686d 100644
--- a/modules/core/src/channels.cpp
+++ b/modules/core/src/channels.cpp
@@ -53,25 +53,25 @@ static void mixChannels8u( const uchar** src, const int* sdelta,
     mixChannels_(src, sdelta, dst, ddelta, len, npairs);
 }
 
-static void mixChannels16u( const ushort** src, const int* sdelta,
-                            ushort** dst, const int* ddelta,
+static void mixChannels16u( const uchar** src, const int* sdelta,
+                            uchar** dst, const int* ddelta,
                             int len, int npairs )
 {
-    mixChannels_(src, sdelta, dst, ddelta, len, npairs);
+    mixChannels_((const ushort**)src, sdelta, (ushort**)dst, ddelta, len, npairs);
 }
 
-static void mixChannels32s( const int** src, const int* sdelta,
-                            int** dst, const int* ddelta,
+static void mixChannels32s( const uchar** src, const int* sdelta,
+                            uchar** dst, const int* ddelta,
                             int len, int npairs )
 {
-    mixChannels_(src, sdelta, dst, ddelta, len, npairs);
+    mixChannels_((const int**)src, sdelta, (int**)dst, ddelta, len, npairs);
 }
 
-static void mixChannels64s( const int64** src, const int* sdelta,
-                            int64** dst, const int* ddelta,
+static void mixChannels64s( const uchar** src, const int* sdelta,
+                            uchar** dst, const int* ddelta,
                             int len, int npairs )
 {
-    mixChannels_(src, sdelta, dst, ddelta, len, npairs);
+    mixChannels_((const int64**)src, sdelta, (int64**)dst, ddelta, len, npairs);
 }
 
 typedef void (*MixChannelsFunc)( const uchar** src, const int* sdelta,
@@ -81,9 +81,9 @@ static MixChannelsFunc getMixchFunc(int depth)
 {
     static MixChannelsFunc mixchTab[] =
     {
-        (MixChannelsFunc)mixChannels8u, (MixChannelsFunc)mixChannels8u, (MixChannelsFunc)mixChannels16u,
-        (MixChannelsFunc)mixChannels16u, (MixChannelsFunc)mixChannels32s, (MixChannelsFunc)mixChannels32s,
-        (MixChannelsFunc)mixChannels64s, 0
+        mixChannels8u, mixChannels8u, mixChannels16u,
+        mixChannels16u, mixChannels32s, mixChannels32s,
+        mixChannels64s, 0
     };
 
     return mixchTab[depth];

From 423ab8ddb8bf039e69ebbf98a53925562ad6962a Mon Sep 17 00:00:00 2001
From: Vincent Rabaud <vrabaud@google.com>
Date: Thu, 20 Jul 2023 15:53:57 +0200
Subject: [PATCH 2/2] Use void*

---
 modules/core/src/channels.cpp | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/channels.cpp b/modules/core/src/channels.cpp
index fd6600686d..efaeb91068 100644
--- a/modules/core/src/channels.cpp
+++ b/modules/core/src/channels.cpp
@@ -46,36 +46,36 @@ mixChannels_( const T** src, const int* sdelta,
 }
 
 
-static void mixChannels8u( const uchar** src, const int* sdelta,
-                           uchar** dst, const int* ddelta,
+static void mixChannels8u( const void** src, const int* sdelta,
+                           void** dst, const int* ddelta,
                            int len, int npairs )
 {
-    mixChannels_(src, sdelta, dst, ddelta, len, npairs);
+    mixChannels_((const uchar**)src, sdelta, (uchar**)dst, ddelta, len, npairs);
 }
 
-static void mixChannels16u( const uchar** src, const int* sdelta,
-                            uchar** dst, const int* ddelta,
+static void mixChannels16u( const void** src, const int* sdelta,
+                            void** dst, const int* ddelta,
                             int len, int npairs )
 {
     mixChannels_((const ushort**)src, sdelta, (ushort**)dst, ddelta, len, npairs);
 }
 
-static void mixChannels32s( const uchar** src, const int* sdelta,
-                            uchar** dst, const int* ddelta,
+static void mixChannels32s( const void** src, const int* sdelta,
+                            void** dst, const int* ddelta,
                             int len, int npairs )
 {
     mixChannels_((const int**)src, sdelta, (int**)dst, ddelta, len, npairs);
 }
 
-static void mixChannels64s( const uchar** src, const int* sdelta,
-                            uchar** dst, const int* ddelta,
+static void mixChannels64s( const void** src, const int* sdelta,
+                            void** dst, const int* ddelta,
                             int len, int npairs )
 {
     mixChannels_((const int64**)src, sdelta, (int64**)dst, ddelta, len, npairs);
 }
 
-typedef void (*MixChannelsFunc)( const uchar** src, const int* sdelta,
-        uchar** dst, const int* ddelta, int len, int npairs );
+typedef void (*MixChannelsFunc)( const void** src, const int* sdelta,
+        void** dst, const int* ddelta, int len, int npairs );
 
 static MixChannelsFunc getMixchFunc(int depth)
 {
@@ -158,7 +158,7 @@ void cv::mixChannels( const Mat* src, size_t nsrcs, Mat* dst, size_t ndsts, cons
         for( int t = 0; t < total; t += blocksize )
         {
             int bsz = std::min(total - t, blocksize);
-            func( srcs, sdelta, dsts, ddelta, bsz, (int)npairs );
+            func( (const void**)srcs, sdelta, (void **)dsts, ddelta, bsz, (int)npairs );
 
             if( t + blocksize < total )
                 for( k = 0; k < npairs; k++ )