Merge pull request #27499 from vpisarev:image_io_with_metadata

Extend image I/O API with metadata support #27499

Covered with the PR:
* AVIF encoder can write exif, xmp, icc
* AVIF decoder can read exif
* JPEG encoder can write exif
* JPEG decoder can read exif
* PNG encoder can write exif
* PNG decoder can read exif

This PR is a sort of preamble for #27488. I suggest to merge this one first to OpenCV 4.x, then promote this change to OpenCV 5.x and then provide extra API to read and write metadata in 5.x (or maybe 4.x) in a style similar to #27488. Maybe in that PR exif packing/unpacking should be done using a separate external API. That is, metadata reading and writing can/should be done in 2 steps:

 * [1] pack and then [2] embed exif into image at the encoding stage.
 * [1] extract and then [2] unpack exif at the decoding stage.

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake

modules/imgcodecs/src/exif.cpp

@@ -94,6 +94,10 @@ ExifEntry_t ExifReader::getTag(const ExifTagName tag) const
     return entry;
 }
 
+const std::vector<unsigned char>& ExifReader::getData() const
+{
+    return m_data;
+}
 
 /**
  * @brief Parsing the exif data buffer and prepare (internal) exif directory

modules/imgcodecs/src/exif.hpp

@@ -175,6 +175,10 @@ public:
      */
     ExifEntry_t getTag( const ExifTagName tag ) const;
 
+    /**
+     * @brief Get the whole exif buffer
+     */
+    const std::vector<unsigned char>& getData() const;
 
 private:
     std::vector<unsigned char> m_data;

modules/imgcodecs/src/grfmt_avif.cpp

@@ -68,8 +68,8 @@ avifResult CopyToMat(const avifImage *image, int channels, bool useRGB , Mat *ma
   return avifImageYUVToRGB(image, &rgba);
 }
 
-AvifImageUniquePtr ConvertToAvif(const cv::Mat &img, bool lossless,
-                                 int bit_depth) {
+AvifImageUniquePtr ConvertToAvif(const cv::Mat &img, bool lossless, int bit_depth,
+                                 const std::vector<std::vector<uchar> >& metadata) {
   CV_Assert(img.depth() == CV_8U || img.depth() == CV_16U);
 
   const int width = img.cols;
@@ -112,6 +112,18 @@ AvifImageUniquePtr ConvertToAvif(const cv::Mat &img, bool lossless,
     result->yuvRange = AVIF_RANGE_FULL;
   }
 
+  if (!metadata.empty()) {
+    const std::vector<uchar>& metadata_exif = metadata[IMAGE_METADATA_EXIF];
+    const std::vector<uchar>& metadata_xmp = metadata[IMAGE_METADATA_XMP];
+    const std::vector<uchar>& metadata_iccp = metadata[IMAGE_METADATA_ICCP];
+    if (!metadata_exif.empty())
+      avifImageSetMetadataExif(result, (const uint8_t*)metadata_exif.data(), metadata_exif.size());
+    if (!metadata_exif.empty())
+      avifImageSetMetadataXMP(result, (const uint8_t*)metadata_xmp.data(), metadata_xmp.size());
+    if (!metadata_iccp.empty())
+      avifImageSetProfileICC(result, (const uint8_t*)metadata_iccp.data(), metadata_iccp.size());
+  }
+
   avifRGBImage rgba;
   avifRGBImageSetDefaults(&rgba, result);
   if (img.channels() == 3) {
@@ -120,7 +132,7 @@ AvifImageUniquePtr ConvertToAvif(const cv::Mat &img, bool lossless,
     CV_Assert(img.channels() == 4);
     rgba.format = AVIF_RGB_FORMAT_BGRA;
   }
-  rgba.rowBytes = img.step[0];
+  rgba.rowBytes = (uint32_t)img.step[0];
   rgba.depth = bit_depth;
   rgba.pixels =
       const_cast<uint8_t *>(reinterpret_cast<const uint8_t *>(img.data));
@@ -287,6 +299,10 @@ bool AvifDecoder::nextPage() {
 AvifEncoder::AvifEncoder() {
   m_description = "AVIF files (*.avif)";
   m_buf_supported = true;
+  m_support_metadata.assign((size_t)IMAGE_METADATA_MAX + 1, false);
+  m_support_metadata[(size_t)IMAGE_METADATA_EXIF] = true;
+  m_support_metadata[(size_t)IMAGE_METADATA_XMP] = true;
+  m_support_metadata[(size_t)IMAGE_METADATA_ICCP] = true;
   encoder_ = avifEncoderCreate();
 }
 
@@ -349,7 +365,7 @@ bool AvifEncoder::writeanimation(const Animation& animation,
              img.channels() == 1 || img.channels() == 3 || img.channels() == 4,
              "AVIF only supports 1, 3, 4 channels");
 
-    images.emplace_back(ConvertToAvif(img, do_lossless, bit_depth));
+    images.emplace_back(ConvertToAvif(img, do_lossless, bit_depth, m_metadata));
   }
 
   for (size_t i = 0; i < images.size(); i++)

modules/imgcodecs/src/grfmt_base.cpp

@@ -58,11 +58,30 @@ BaseImageDecoder::BaseImageDecoder()
     m_frame_count = 1;
 }
 
+bool BaseImageDecoder::haveMetadata(ImageMetadataType type) const
+{
+    if (type == IMAGE_METADATA_EXIF)
+        return !m_exif.getData().empty();
+    return false;
+}
+
+Mat BaseImageDecoder::getMetadata(ImageMetadataType type) const
+{
+    if (type == IMAGE_METADATA_EXIF) {
+        const std::vector<unsigned char>& exif = m_exif.getData();
+        if (!exif.empty()) {
+            Mat exifmat(1, (int)exif.size(), CV_8U, (void*)exif.data());
+            return exifmat;
+        }
+    }
+    return Mat();
+}
 
 ExifEntry_t BaseImageDecoder::getExifTag(const ExifTagName tag) const
 {
     return m_exif.getTag(tag);
 }
+
 bool BaseImageDecoder::setSource( const String& filename )
 {
     m_filename = filename;
@@ -140,6 +159,23 @@ bool BaseImageEncoder::setDestination( std::vector<uchar>& buf )
     return true;
 }
 
+bool BaseImageEncoder::addMetadata(ImageMetadataType type, const Mat& metadata)
+{
+    CV_Assert_N(type >= IMAGE_METADATA_EXIF, type <= IMAGE_METADATA_MAX);
+    if (metadata.empty())
+        return true;
+    size_t itype = (size_t)type;
+    if (itype >= m_support_metadata.size() || !m_support_metadata[itype])
+        return false;
+    if (m_metadata.empty())
+        m_metadata.resize((size_t)IMAGE_METADATA_MAX+1);
+    CV_Assert(metadata.elemSize() == 1);
+    CV_Assert(metadata.isContinuous());
+    const unsigned char* data = metadata.ptr<unsigned char>();
+    m_metadata[itype].assign(data, data + metadata.total());
+    return true;
+}
+
 bool BaseImageEncoder::write(const Mat &img, const std::vector<int> &params) {
     std::vector<Mat> img_vec(1, img);
     return writemulti(img_vec, params);

modules/imgcodecs/src/grfmt_base.hpp

@@ -69,6 +69,20 @@ public:
      */
     virtual int type() const { return m_type; }
 
+    /**
+     * @brief Checks whether file contains metadata of the certain type.
+     * @param type The type of metadata to look for
+     */
+    virtual bool haveMetadata(ImageMetadataType type) const;
+
+    /**
+     * @brief Retrieves metadata (if any) of the certain kind.
+     * If there is no such metadata, the method returns empty array.
+     *
+     * @param type The type of metadata to look for
+     */
+    virtual Mat getMetadata(ImageMetadataType type) const;
+
     /**
      * @brief Fetch a specific EXIF tag from the image's metadata.
      * @param tag The EXIF tag to retrieve.
@@ -205,6 +219,13 @@ public:
      */
     virtual bool setDestination(std::vector<uchar>& buf);
 
+    /**
+     * @brief Sets the metadata to write together with the image data
+     * @param type The type of metadata to add
+     * @param metadata The packed metadata (Exif, XMP, ...)
+     */
+    virtual bool addMetadata(ImageMetadataType type, const Mat& metadata);
+
     /**
      * @brief Encode and write the image data.
      * @param img The Mat object containing the image data to be encoded.
@@ -243,6 +264,8 @@ public:
     virtual void throwOnError() const;
 
 protected:
+    std::vector<std::vector<unsigned char> > m_metadata; // see IMAGE_METADATA_...
+    std::vector<bool> m_support_metadata;
     String m_description;    ///< Description of the encoder (e.g., format name, capabilities).
     String m_filename;       ///< Destination file name for encoded data.
     std::vector<uchar>* m_buf; ///< Pointer to the buffer for encoded data if using memory-based destination.

modules/imgcodecs/src/grfmt_jpeg.cpp

@@ -600,6 +600,8 @@ JpegEncoder::JpegEncoder()
 {
     m_description = "JPEG files (*.jpeg;*.jpg;*.jpe)";
     m_buf_supported = true;
+    m_support_metadata.assign((size_t)IMAGE_METADATA_MAX + 1, false);
+    m_support_metadata[(size_t)IMAGE_METADATA_EXIF] = true;
 }
 
 
@@ -815,6 +817,22 @@ bool JpegEncoder::write( const Mat& img, const std::vector<int>& params )
 
         jpeg_start_compress( &cinfo, TRUE );
 
+        if (!m_metadata.empty()) {
+            const std::vector<uchar>& metadata_exif = m_metadata[IMAGE_METADATA_EXIF];
+            size_t exif_size = metadata_exif.size();
+            if (exif_size > 0u) {
+                const char app1_exif_prefix[] = {'E', 'x', 'i', 'f', '\0', '\0'};
+                size_t app1_exif_prefix_size = sizeof(app1_exif_prefix);
+                size_t data_size = exif_size + app1_exif_prefix_size;
+
+                std::vector<uchar> metadata_app1(data_size);
+                uchar* data = metadata_app1.data();
+                memcpy(data, app1_exif_prefix, app1_exif_prefix_size);
+                memcpy(data + app1_exif_prefix_size, metadata_exif.data(), exif_size);
+                jpeg_write_marker(&cinfo, JPEG_APP0 + 1, data, (unsigned)data_size);
+            }
+        }
+
         if( doDirectWrite )
         {
             for( int y = 0; y < height; y++ )

modules/imgcodecs/src/grfmt_png.cpp

@@ -858,6 +858,8 @@ PngEncoder::PngEncoder()
 {
     m_description = "Portable Network Graphics files (*.png;*.apng)";
     m_buf_supported = true;
+    m_support_metadata.assign((size_t)IMAGE_METADATA_MAX+1, false);
+    m_support_metadata[IMAGE_METADATA_EXIF] = true;
     op_zstream1.zalloc = NULL;
     op_zstream2.zalloc = NULL;
     next_seq_num = 0;
@@ -989,6 +991,16 @@ bool  PngEncoder::write( const Mat& img, const std::vector<int>& params )
                     for( y = 0; y < height; y++ )
                         buffer[y] = img.data + y*img.step;
 
+                    if (!m_metadata.empty()) {
+                        std::vector<uchar>& exif = m_metadata[IMAGE_METADATA_EXIF];
+                        if (!exif.empty()) {
+                            writeChunk(f, "eXIf", exif.data(), (uint32_t)exif.size());
+                        }
+                        // [TODO] add xmp and icc. They need special handling,
+                        // see https://dev.exiv2.org/projects/exiv2/wiki/The_Metadata_in_PNG_files and
+                        // https://www.libpng.org/pub/png/spec/1.2/PNG-Chunks.html.
+                    }
+
                     png_write_image( png_ptr, buffer.data() );
                     png_write_end( png_ptr, info_ptr );
 

modules/imgcodecs/src/loadsave.cpp

@@ -410,6 +410,76 @@ static void ApplyExifOrientation(ExifEntry_t orientationTag, OutputArray img)
     }
 }
 
+static void readMetadata(ImageDecoder& decoder,
+                         std::vector<int>* metadata_types,
+                         OutputArrayOfArrays metadata)
+{
+    if (!metadata_types)
+        return;
+    int kind = metadata.kind();
+    void* obj = metadata.getObj();
+    std::vector<Mat>* matvector = nullptr;
+    std::vector<std::vector<uchar> >* vecvector = nullptr;
+    if (kind == _InputArray::STD_VECTOR_MAT) {
+        matvector = (std::vector<Mat>*)obj;
+    } else if (kind == _InputArray::STD_VECTOR_VECTOR) {
+        int elemtype = metadata.type(0);
+        CV_Assert(elemtype == CV_8UC1 || elemtype == CV_8SC1);
+        vecvector = (std::vector<std::vector<uint8_t> >*)obj;
+    } else {
+        CV_Error(Error::StsBadArg,
+                 "unsupported metadata type, should be a vector of matrices or vector of byte vectors");
+    }
+    std::vector<Mat> src_metadata;
+    for (int m = (int)IMAGE_METADATA_EXIF; m <= (int)IMAGE_METADATA_MAX; m++) {
+        Mat mm = decoder->getMetadata((ImageMetadataType)m);
+        if (!mm.empty()) {
+            CV_Assert(mm.isContinuous());
+            CV_Assert(mm.elemSize() == 1u);
+            metadata_types->push_back(m);
+            src_metadata.push_back(mm);
+        }
+    }
+    size_t nmetadata = metadata_types->size();
+    if (matvector) {
+        matvector->resize(nmetadata);
+        for (size_t m = 0; m < nmetadata; m++)
+            src_metadata[m].copyTo(matvector->at(m));
+    } else {
+        vecvector->resize(nmetadata);
+        for (size_t m = 0; m < nmetadata; m++) {
+            const Mat& mm = src_metadata[m];
+            const uchar* data = (uchar*)mm.data;
+            vecvector->at(m).assign(data, data + mm.total());
+        }
+    }
+}
+
+static const char* metadataTypeToString(ImageMetadataType type)
+{
+    return type == IMAGE_METADATA_EXIF ? "Exif" :
+           type == IMAGE_METADATA_XMP ? "XMP" :
+           type == IMAGE_METADATA_ICCP ? "ICC Profile" : "???";
+}
+
+static void addMetadata(ImageEncoder& encoder,
+                        const std::vector<int>& metadata_types,
+                        InputArrayOfArrays metadata)
+{
+    size_t nmetadata_chunks = metadata_types.size();
+    for (size_t i = 0; i < nmetadata_chunks; i++) {
+        ImageMetadataType metadata_type = (ImageMetadataType)metadata_types[i];
+        bool ok = encoder->addMetadata(metadata_type, metadata.getMat((int)i));
+        if (!ok) {
+            std::string desc = encoder->getDescription();
+            CV_LOG_WARNING(NULL, "Imgcodecs: metadata of type '"
+                           << metadataTypeToString(metadata_type)
+                           << "' is not supported when encoding '"
+                           << desc << "'");
+        }
+    }
+}
+
 /**
  * Read an image into memory and return the information
  *
@@ -419,11 +489,15 @@ static void ApplyExifOrientation(ExifEntry_t orientationTag, OutputArray img)
  *
 */
 static bool
-imread_( const String& filename, int flags, OutputArray mat )
+imread_( const String& filename, int flags, OutputArray mat,
+         std::vector<int>* metadata_types, OutputArrayOfArrays metadata)
 {
     /// Search for the relevant decoder to handle the imagery
     ImageDecoder decoder;
 
+    if (metadata_types)
+        metadata_types->clear();
+
 #ifdef HAVE_GDAL
     if(flags != IMREAD_UNCHANGED && (flags & IMREAD_LOAD_GDAL) == IMREAD_LOAD_GDAL ){
         decoder = GdalDecoder().newDecoder();
@@ -509,6 +583,8 @@ imread_( const String& filename, int flags, OutputArray mat )
             CV_CheckTrue(original_ptr == real_mat.data, "Internal imread issue");
             success = true;
         }
+
+        readMetadata(decoder, metadata_types, metadata);
     }
     catch (const cv::Exception& e)
     {
@@ -662,7 +738,24 @@ Mat imread( const String& filename, int flags )
     Mat img;
 
     /// load the data
-    imread_( filename, flags, img );
+    imread_( filename, flags, img, nullptr, noArray() );
+
+    /// return a reference to the data
+    return img;
+}
+
+Mat imreadWithMetadata( const String& filename,
+                        std::vector<int>& metadata_types,
+                        OutputArrayOfArrays metadata,
+                        int flags )
+{
+    CV_TRACE_FUNCTION();
+
+    /// create the basic container
+    Mat img;
+
+    /// load the data
+    imread_( filename, flags, img, &metadata_types, metadata );
 
     /// return a reference to the data
     return img;
@@ -673,7 +766,7 @@ void imread( const String& filename, OutputArray dst, int flags )
     CV_TRACE_FUNCTION();
 
     /// load the data
-    imread_(filename, flags, dst);
+    imread_(filename, flags, dst, nullptr, noArray());
 }
 
 /**
@@ -946,6 +1039,8 @@ size_t imcount(const String& filename, int flags)
 
 
 static bool imwrite_( const String& filename, const std::vector<Mat>& img_vec,
+                      const std::vector<int>& metadata_types,
+                      InputArrayOfArrays metadata,
                       const std::vector<int>& params_, bool flipv )
 {
     bool isMultiImg = img_vec.size() > 1;
@@ -981,6 +1076,8 @@ static bool imwrite_( const String& filename, const std::vector<Mat>& img_vec,
     }
 
     encoder->setDestination( filename );
+    addMetadata(encoder, metadata_types, metadata);
+
 #if CV_VERSION_MAJOR < 5 && defined(HAVE_IMGCODEC_HDR)
     bool fixed = false;
     std::vector<int> params_pair(2);
@@ -1055,7 +1152,26 @@ bool imwrite( const String& filename, InputArray _img,
         img_vec.push_back(_img.getMat());
 
     CV_Assert(!img_vec.empty());
-    return imwrite_(filename, img_vec, params, false);
+    return imwrite_(filename, img_vec, {}, noArray(), params, false);
+}
+
+bool imwriteWithMetadata( const String& filename, InputArray _img,
+                          const std::vector<int>& metadata_types,
+                          InputArrayOfArrays metadata,
+                          const std::vector<int>& params )
+{
+    CV_TRACE_FUNCTION();
+
+    CV_Assert(!_img.empty());
+
+    std::vector<Mat> img_vec;
+    if (_img.isMatVector() || _img.isUMatVector())
+        _img.getMatVector(img_vec);
+    else
+        img_vec.push_back(_img.getMat());
+
+    CV_Assert(!img_vec.empty());
+    return imwrite_(filename, img_vec, metadata_types, metadata, params, false);
 }
 
 static bool imwriteanimation_(const String& filename, const Animation& animation, const std::vector<int>& params)
@@ -1140,8 +1256,13 @@ bool imencodeanimation(const String& ext, const Animation& animation, std::vecto
 }
 
 static bool
-imdecode_( const Mat& buf, int flags, Mat& mat )
+imdecode_( const Mat& buf, int flags, Mat& mat,
+           std::vector<int>* metadata_types,
+           OutputArrayOfArrays metadata )
 {
+    if (metadata_types)
+        metadata_types->clear();
+
     CV_Assert(!buf.empty());
     CV_Assert(buf.isContinuous());
     CV_Assert(buf.checkVector(1, CV_8U) > 0);
@@ -1231,6 +1352,7 @@ imdecode_( const Mat& buf, int flags, Mat& mat )
     {
         if (decoder->readData(mat))
             success = true;
+        readMetadata(decoder, metadata_types, metadata);
     }
     catch (const cv::Exception& e)
     {
@@ -1274,7 +1396,7 @@ Mat imdecode( InputArray _buf, int flags )
     CV_TRACE_FUNCTION();
 
     Mat buf = _buf.getMat(), img;
-    if (!imdecode_(buf, flags, img))
+    if (!imdecode_(buf, flags, img, nullptr, noArray()))
         img.release();
 
     return img;
@@ -1286,12 +1408,24 @@ Mat imdecode( InputArray _buf, int flags, Mat* dst )
 
     Mat buf = _buf.getMat(), img;
     dst = dst ? dst : &img;
-    if (imdecode_(buf, flags, *dst))
+    if (imdecode_(buf, flags, *dst, nullptr, noArray()))
         return *dst;
     else
         return cv::Mat();
 }
 
+Mat imdecodeWithMetadata( InputArray _buf, std::vector<int>& metadata_types,
+                          OutputArrayOfArrays metadata, int flags )
+{
+    CV_TRACE_FUNCTION();
+
+    Mat buf = _buf.getMat(), img;
+    if (!imdecode_(buf, flags, img, &metadata_types, metadata))
+        img.release();
+
+    return img;
+}
+
 static bool
 imdecodemulti_(const Mat& buf, int flags, std::vector<Mat>& mats, int start, int count)
 {
@@ -1447,8 +1581,10 @@ bool imdecodemulti(InputArray _buf, int flags, CV_OUT std::vector<Mat>& mats, co
     }
 }
 
-bool imencode( const String& ext, InputArray _img,
-               std::vector<uchar>& buf, const std::vector<int>& params_ )
+bool imencodeWithMetadata( const String& ext, InputArray _img,
+                           const std::vector<int>& metadata_types,
+                           InputArrayOfArrays metadata,
+                           std::vector<uchar>& buf, const std::vector<int>& params_ )
 {
     CV_TRACE_FUNCTION();
 
@@ -1517,6 +1653,7 @@ bool imencode( const String& ext, InputArray _img,
         code = encoder->setDestination(filename);
         CV_Assert( code );
     }
+    addMetadata(encoder, metadata_types, metadata);
 
     try {
         if (!isMultiImg)
@@ -1553,6 +1690,12 @@ bool imencode( const String& ext, InputArray _img,
     return code;
 }
 
+bool imencode( const String& ext, InputArray img,
+               std::vector<uchar>& buf, const std::vector<int>& params_ )
+{
+    return imencodeWithMetadata(ext, img, {}, noArray(), buf, params_);
+}
+
 bool imencodemulti( const String& ext, InputArrayOfArrays imgs,
                     std::vector<uchar>& buf, const std::vector<int>& params)
 {