imgproc: disable buggy inplace processing in convexHull

modules/imgproc/include/opencv2/imgproc.hpp

@@ -3977,6 +3977,8 @@ returns convex hull points. Otherwise, it returns indices of the convex hull poi
 output array is std::vector, the flag is ignored, and the output depends on the type of the
 vector: std::vector\<int\> implies returnPoints=false, std::vector\<Point\> implies
 returnPoints=true.
+
+@note `points` and `hull` should be different arrays, inplace processing isn't supported.
  */
 CV_EXPORTS_W void convexHull( InputArray points, OutputArray hull,
                               bool clockwise = false, bool returnPoints = true );

modules/imgproc/src/convhull.cpp

@@ -130,6 +130,7 @@ void convexHull( InputArray _points, OutputArray _hull, bool clockwise, bool ret
 {
     CV_INSTRUMENT_REGION()
 
+    CV_Assert(_points.getObj() != _hull.getObj());
     Mat points = _points.getMat();
     int i, total = points.checkVector(2), depth = points.depth(), nout = 0;
     int miny_ind = 0, maxy_ind = 0;