release: OpenCV 4.6.0

Merge pull request #22047 from jlopezr:trackbar-ordered-in-mac
* In Mac highgui now shows trackbars in creation order * In Mac highgui trackbars show current value * Remove trailing spaces in objectivec code
2026-07-23 04:13:04 +04:00 · 2022-06-05 15:32:44 +00:00 · 2022-06-05 15:30:16 +00:00 · 2022-06-04 19:10:35 +00:00 · 2022-06-04 17:33:08 +03:00 · 2022-06-03 09:03:33 +00:00
1697 changed files with 254450 additions and 179192 deletions
@@ -34,11 +34,11 @@ This is a template helping you to create an issue which can be processed as quic
 - [ ] I report the issue, it's not a question
   <!--
   OpenCV team works with forum.opencv.org, Stack Overflow and other communities
-   to discuss problems. Tickets with question without real issue statement will be
+   to discuss problems. Tickets with questions without a real issue statement will be
   closed.
   -->
 - [ ] I checked the problem with documentation, FAQ, open issues,
-       forum.opencv.org, Stack Overflow, etc and have not found solution
+       forum.opencv.org, Stack Overflow, etc and have not found any solution
   <!--
   Places to check:
   * OpenCV documentation: https://docs.opencv.org
@@ -47,11 +47,11 @@ This is a template helping you to create an issue which can be processed as quic
   * OpenCV issue tracker: https://github.com/opencv/opencv/issues?q=is%3Aissue
   * Stack Overflow branch: https://stackoverflow.com/questions/tagged/opencv
   -->
- - [ ] I updated to latest OpenCV version and the issue is still there
+ - [ ] I updated to the latest OpenCV version and the issue is still there
   <!--
   master branch for OpenCV 4.x and 3.4 branch for OpenCV 3.x releases.
-   OpenCV team supports only latest release for each branch.
-   The ticket is closed, if the problem is not reproduced with modern version.
+   OpenCV team supports only the latest release for each branch.
+   The ticket is closed if the problem is not reproduced with the modern version.
   -->
 - [ ] There is reproducer code and related data files: videos, images, onnx, etc
   <!--
@@ -61,9 +61,9 @@ This is a template helping you to create an issue which can be processed as quic
     to reduce attachment size
   * Use PNG for images, if you report some CV related bug, but not image reader
     issue
-   * Attach the image as archive to the ticket, if you report some reader issue.
+   * Attach the image as an archive to the ticket, if you report some reader issue.
     Image hosting services compress images and it breaks the repro code.
-   * Provide ONNX file for some public model or ONNX file with with random weights,
+   * Provide ONNX file for some public model or ONNX file with random weights,
     if you report ONNX parsing or handling issue. Architecture details diagram
     from netron tool can be very useful too. See https://lutzroeder.github.io/netron/
   -->
@@ -3,9 +3,9 @@
 See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

 - [ ] I agree to contribute to the project under Apache 2 License.
- [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or other license that is incompatible with OpenCV
- [ ] The PR is proposed to proper branch
- [ ] There is reference to original bug report and related work
+- [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
+- [ ] The PR is proposed to the proper branch
+- [ ] There is a reference to the original bug report and related work
 - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
 - [ ] The feature is well documented and sample code can be built with the project CMake
@@ -0,0 +1,19 @@
+name: PR:4.x
+
+on:
+  pull_request:
+    branches:
+      - 4.x
+
+jobs:
+  ARM64:
+    uses: opencv/ci-gha-workflow/.github/workflows/OCV-PR-4.x-ARM64.yaml@main
+
+  U20:
+    uses: opencv/ci-gha-workflow/.github/workflows/OCV-PR-4.x-U20.yaml@main
+
+  W10:
+    uses: opencv/ci-gha-workflow/.github/workflows/OCV-PR-4.x-W10.yaml@main
+
+  TIM-VX:
+    uses: opencv/ci-gha-workflow/.github/workflows/OCV-timvx-backend-tests-4.x.yml@main
@@ -0,0 +1,25 @@
+name: lint_python
+on: workflow_dispatch
+jobs:
+  lint_python:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install --upgrade pip wheel
+      - run: pip install bandit black codespell flake8 flake8-2020 flake8-bugbear
+                         flake8-comprehensions isort mypy pytest pyupgrade safety
+      - run: bandit --recursive --skip B101 . || true  # B101 is assert statements
+      - run: black --check . || true
+      - run: codespell || true  # --ignore-words-list="" --skip="*.css,*.js,*.lock"
+      - run: flake8 . --count --select=E9,F63,F7 --show-source --statistics
+      - run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88
+                      --show-source --statistics
+      - run: isort --check-only --profile black . || true
+      - run: pip install -r requirements.txt || pip install --editable . || true
+      - run: mkdir --parents --verbose .mypy_cache
+      - run: mypy --ignore-missing-imports --install-types --non-interactive . || true
+      - run: pytest . || true
+      - run: pytest --doctest-modules . || true
+      - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true
+      - run: safety check
@@ -1778,30 +1778,30 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast<size_t>(range.
    : CV_HAL_ERROR_NOT_IMPLEMENTED \
 )

-#define TEGRA_CVT2PYUVTOBGR(src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \
+#define TEGRA_CVT2PYUVTOBGR_EX(y_data, y_step, uv_data, uv_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \
 ( \
    CAROTENE_NS::isSupportedConfiguration() ? \
        dcn == 3 ? \
            uIdx == 0 ? \
                (swapBlue ? \
                    CAROTENE_NS::yuv420i2rgb(CAROTENE_NS::Size2D(dst_width, dst_height), \
-                                             src_data, src_step, \
-                                             src_data + src_step * dst_height, src_step, \
+                                             y_data, y_step, \
+                                             uv_data, uv_step, \
                                             dst_data, dst_step) : \
                    CAROTENE_NS::yuv420i2bgr(CAROTENE_NS::Size2D(dst_width, dst_height), \
-                                             src_data, src_step, \
-                                             src_data + src_step * dst_height, src_step, \
+                                             y_data, y_step, \
+                                             uv_data, uv_step, \
                                             dst_data, dst_step)), \
                CV_HAL_ERROR_OK : \
            uIdx == 1 ? \
                (swapBlue ? \
                    CAROTENE_NS::yuv420sp2rgb(CAROTENE_NS::Size2D(dst_width, dst_height), \
-                                              src_data, src_step, \
-                                              src_data + src_step * dst_height, src_step, \
+                                              y_data, y_step, \
+                                              uv_data, uv_step, \
                                              dst_data, dst_step) : \
                    CAROTENE_NS::yuv420sp2bgr(CAROTENE_NS::Size2D(dst_width, dst_height), \
-                                              src_data, src_step, \
-                                              src_data + src_step * dst_height, src_step, \
+                                              y_data, y_step, \
+                                              uv_data, uv_step, \
                                              dst_data, dst_step)), \
                CV_HAL_ERROR_OK : \
            CV_HAL_ERROR_NOT_IMPLEMENTED : \
@@ -1809,29 +1809,32 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast<size_t>(range.
            uIdx == 0 ? \
                (swapBlue ? \
                    CAROTENE_NS::yuv420i2rgbx(CAROTENE_NS::Size2D(dst_width, dst_height), \
-                                              src_data, src_step, \
-                                              src_data + src_step * dst_height, src_step, \
+                                              y_data, y_step, \
+                                              uv_data, uv_step, \
                                              dst_data, dst_step) : \
                    CAROTENE_NS::yuv420i2bgrx(CAROTENE_NS::Size2D(dst_width, dst_height), \
-                                              src_data, src_step, \
-                                              src_data + src_step * dst_height, src_step, \
+                                              y_data, y_step, \
+                                              uv_data, uv_step, \
                                              dst_data, dst_step)), \
                CV_HAL_ERROR_OK : \
            uIdx == 1 ? \
                (swapBlue ? \
                    CAROTENE_NS::yuv420sp2rgbx(CAROTENE_NS::Size2D(dst_width, dst_height), \
-                                               src_data, src_step, \
-                                               src_data + src_step * dst_height, src_step, \
+                                               y_data, y_step, \
+                                               uv_data, uv_step, \
                                               dst_data, dst_step) : \
                    CAROTENE_NS::yuv420sp2bgrx(CAROTENE_NS::Size2D(dst_width, dst_height), \
-                                               src_data, src_step, \
-                                               src_data + src_step * dst_height, src_step, \
+                                               y_data, y_step, \
+                                               uv_data, uv_step, \
                                               dst_data, dst_step)), \
                CV_HAL_ERROR_OK : \
            CV_HAL_ERROR_NOT_IMPLEMENTED : \
        CV_HAL_ERROR_NOT_IMPLEMENTED \
    : CV_HAL_ERROR_NOT_IMPLEMENTED \
 )
+#define TEGRA_CVT2PYUVTOBGR(src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \
+    TEGRA_CVT2PYUVTOBGR_EX(src_data, src_step, src_data + src_step * dst_height, src_step, dst_data, dst_step, \
+            dst_width, dst_height, dcn, swapBlue, uIdx);

 #undef cv_hal_cvtBGRtoBGR
 #define cv_hal_cvtBGRtoBGR TEGRA_CVTBGRTOBGR
@@ -1841,12 +1844,18 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast<size_t>(range.
 #define cv_hal_cvtBGRtoGray TEGRA_CVTBGRTOGRAY
 #undef cv_hal_cvtGraytoBGR
 #define cv_hal_cvtGraytoBGR TEGRA_CVTGRAYTOBGR
+#if 0  // bit-exact tests are failed
 #undef cv_hal_cvtBGRtoYUV
 #define cv_hal_cvtBGRtoYUV TEGRA_CVTBGRTOYUV
+#endif
 #undef cv_hal_cvtBGRtoHSV
 #define cv_hal_cvtBGRtoHSV TEGRA_CVTBGRTOHSV
+#if 0  // bit-exact tests are failed
 #undef cv_hal_cvtTwoPlaneYUVtoBGR
 #define cv_hal_cvtTwoPlaneYUVtoBGR TEGRA_CVT2PYUVTOBGR
+#undef cv_hal_cvtTwoPlaneYUVtoBGREx
+#define cv_hal_cvtTwoPlaneYUVtoBGREx TEGRA_CVT2PYUVTOBGR_EX
+#endif

 #endif // OPENCV_IMGPROC_HAL_INTERFACE_H

@@ -1,9 +1,9 @@
-# Binaries branch name: ffmpeg/master_20210303
-# Binaries were created for OpenCV: 7ac6abe02a33bef445a5b77214ad31964e2c5cc1
-ocv_update(FFMPEG_BINARIES_COMMIT "629590c3ba09fb0c8eaa9ab858ff13d3a84ca1aa")
-ocv_update(FFMPEG_FILE_HASH_BIN32 "638065d5a0dab8a828879942375dcac4")
-ocv_update(FFMPEG_FILE_HASH_BIN64 "7f10ae2e6a080ba3714f7a38ee03ae15")
-ocv_update(FFMPEG_FILE_HASH_CMAKE "f8e65dbe4a3b4eedc0d2997e07c3f3fd")
+# Binaries branch name: ffmpeg/4.x_20220524
+# Binaries were created for OpenCV: d6e9616256b46bd59be0a93d397f6ab958d39cd2
+ocv_update(FFMPEG_BINARIES_COMMIT "65ec04d4573dcdfa4531f0b9e67f35d8ffff873e")
+ocv_update(FFMPEG_FILE_HASH_BIN32 "5573e2262ad1298e603122b7759fc2f6")
+ocv_update(FFMPEG_FILE_HASH_BIN64 "5f9e2b2e04c15f080f40e844de80c867")
+ocv_update(FFMPEG_FILE_HASH_CMAKE "8862c87496e2e8c375965e1277dee1c7")

 function(download_win_ffmpeg script_var)
  set(${script_var} "" PARENT_SCOPE)
@@ -54,6 +54,7 @@ set_target_properties(${ITT_LIBRARY} PROPERTIES
    )

 ocv_warnings_disable(CMAKE_C_FLAGS -Wundef -Wsign-compare)
+ocv_warnings_disable(CMAKE_C_FLAGS -Wstrict-prototypes) # clang15

 if(ENABLE_SOLUTION_FOLDERS)
  set_target_properties(${ITT_LIBRARY} PROPERTIES FOLDER "3rdparty")
@@ -3,10 +3,10 @@ project(${JPEG_LIBRARY} C)
 ocv_warnings_disable(CMAKE_C_FLAGS -Wunused-parameter -Wsign-compare -Wshorten-64-to-32 -Wimplicit-fallthrough)

 set(VERSION_MAJOR 2)
-set(VERSION_MINOR 0)
-set(VERSION_REVISION 6)
+set(VERSION_MINOR 1)
+set(VERSION_REVISION 2)
 set(VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_REVISION})
-set(LIBJPEG_TURBO_VERSION_NUMBER 2000006)
+set(LIBJPEG_TURBO_VERSION_NUMBER 2001002)

 string(TIMESTAMP BUILD "opencv-${OPENCV_VERSION}-libjpeg-turbo")
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -46,7 +46,6 @@ if(UNIX)
  ocv_update(HAVE_UNSIGNED_SHORT 1)
  # undef INCOMPLETE_TYPES_BROKEN
  ocv_update(RIGHT_SHIFT_IS_UNSIGNED 0)
-  ocv_update(__CHAR_UNSIGNED__ 0)
 endif()


@@ -91,7 +91,7 @@ best of our understanding.
 The Modified (3-clause) BSD License
 ===================================

-Copyright (C)2009-2020 D. R. Commander.  All Rights Reserved.
+Copyright (C)2009-2021 D. R. Commander.  All Rights Reserved.<br>
 Copyright (C)2015 Viktor Szathmáry.  All Rights Reserved.

 Redistribution and use in source and binary forms, with or without
@@ -128,7 +128,7 @@ with respect to this software, its quality, accuracy, merchantability, or
 fitness for a particular purpose.  This software is provided "AS IS", and you,
 its user, assume the entire risk as to its quality and accuracy.

-This software is copyright (C) 1991-2016, Thomas G. Lane, Guido Vollbeding.
+This software is copyright (C) 1991-2020, Thomas G. Lane, Guido Vollbeding.
 All Rights Reserved except as specified below.

 Permission is hereby granted to use, copy, modify, and distribute this
@@ -159,19 +159,6 @@ commercial products, provided that all warranty or liability claims are
 assumed by the product vendor.


-The IJG distribution formerly included code to read and write GIF files.
-To avoid entanglement with the Unisys LZW patent (now expired), GIF reading
-support has been removed altogether, and the GIF writer has been simplified
-to produce "uncompressed GIFs".  This technique does not use the LZW
-algorithm; the resulting GIF files are larger than usual, but are readable
-by all standard GIF decoders.
-
-We are required to state that
-    "The Graphics Interchange Format(c) is the Copyright property of
-    CompuServe Incorporated.  GIF(sm) is a Service Mark property of
-    CompuServe Incorporated."
-
-
 REFERENCES
 ==========

@@ -3,7 +3,7 @@ Background

 libjpeg-turbo is a JPEG image codec that uses SIMD instructions to accelerate
 baseline JPEG compression and decompression on x86, x86-64, Arm, PowerPC, and
-MIPS systems, as well as progressive JPEG compression on x86 and x86-64
+MIPS systems, as well as progressive JPEG compression on x86, x86-64, and Arm
 systems.  On such systems, libjpeg-turbo is generally 2-6x as fast as libjpeg,
 all else being equal.  On other types of systems, libjpeg-turbo can still
 outperform libjpeg by a significant amount, by virtue of its highly-optimized
@@ -61,11 +61,6 @@
   unsigned. */
 #cmakedefine RIGHT_SHIFT_IS_UNSIGNED 1

-/* Define to 1 if type `char' is unsigned and you are not using gcc.  */
-#ifndef __CHAR_UNSIGNED__
-  #cmakedefine __CHAR_UNSIGNED__ 1
-#endif
-
 /* Define to empty if `const' does not conform to ANSI C. */
 /* #undef const */

@@ -18,7 +18,6 @@
 #define HAVE_UNSIGNED_SHORT
 #undef INCOMPLETE_TYPES_BROKEN
 #undef RIGHT_SHIFT_IS_UNSIGNED
-#undef __CHAR_UNSIGNED__

 /* Define "boolean" as unsigned char, not int, per Windows custom */
 #ifndef __RPCNDR_H__            /* don't conflict if rpcndr.h already read */
@@ -40,3 +40,13 @@
 #define HAVE_BITSCANFORWARD
 #endif
 #endif
+
+#if defined(__has_attribute)
+#if __has_attribute(fallthrough)
+#define FALLTHROUGH  __attribute__((fallthrough));
+#else
+#define FALLTHROUGH
+#endif
+#else
+#define FALLTHROUGH
+#endif
@@ -48,9 +48,9 @@ rgb_ycc_convert_internal(j_compress_ptr cinfo, JSAMPARRAY input_buf,
    outptr2 = output_buf[2][output_row];
    output_row++;
    for (col = 0; col < num_cols; col++) {
-      r = GETJSAMPLE(inptr[RGB_RED]);
-      g = GETJSAMPLE(inptr[RGB_GREEN]);
-      b = GETJSAMPLE(inptr[RGB_BLUE]);
+      r = inptr[RGB_RED];
+      g = inptr[RGB_GREEN];
+      b = inptr[RGB_BLUE];
      inptr += RGB_PIXELSIZE;
      /* If the inputs are 0..MAXJSAMPLE, the outputs of these equations
       * must be too; we do not need an explicit range-limiting operation.
@@ -100,9 +100,9 @@ rgb_gray_convert_internal(j_compress_ptr cinfo, JSAMPARRAY input_buf,
    outptr = output_buf[0][output_row];
    output_row++;
    for (col = 0; col < num_cols; col++) {
-      r = GETJSAMPLE(inptr[RGB_RED]);
-      g = GETJSAMPLE(inptr[RGB_GREEN]);
-      b = GETJSAMPLE(inptr[RGB_BLUE]);
+      r = inptr[RGB_RED];
+      g = inptr[RGB_GREEN];
+      b = inptr[RGB_BLUE];
      inptr += RGB_PIXELSIZE;
      /* Y */
      outptr[col] = (JSAMPLE)((ctab[r + R_Y_OFF] + ctab[g + G_Y_OFF] +
@@ -135,9 +135,9 @@ rgb_rgb_convert_internal(j_compress_ptr cinfo, JSAMPARRAY input_buf,
    outptr2 = output_buf[2][output_row];
    output_row++;
    for (col = 0; col < num_cols; col++) {
-      outptr0[col] = GETJSAMPLE(inptr[RGB_RED]);
-      outptr1[col] = GETJSAMPLE(inptr[RGB_GREEN]);
-      outptr2[col] = GETJSAMPLE(inptr[RGB_BLUE]);
+      outptr0[col] = inptr[RGB_RED];
+      outptr1[col] = inptr[RGB_GREEN];
+      outptr2[col] = inptr[RGB_BLUE];
      inptr += RGB_PIXELSIZE;
    }
  }
@@ -392,11 +392,11 @@ cmyk_ycck_convert(j_compress_ptr cinfo, JSAMPARRAY input_buf,
    outptr3 = output_buf[3][output_row];
    output_row++;
    for (col = 0; col < num_cols; col++) {
-      r = MAXJSAMPLE - GETJSAMPLE(inptr[0]);
-      g = MAXJSAMPLE - GETJSAMPLE(inptr[1]);
-      b = MAXJSAMPLE - GETJSAMPLE(inptr[2]);
+      r = MAXJSAMPLE - inptr[0];
+      g = MAXJSAMPLE - inptr[1];
+      b = MAXJSAMPLE - inptr[2];
      /* K passes through as-is */
-      outptr3[col] = inptr[3];  /* don't need GETJSAMPLE here */
+      outptr3[col] = inptr[3];
      inptr += 4;
      /* If the inputs are 0..MAXJSAMPLE, the outputs of these equations
       * must be too; we do not need an explicit range-limiting operation.
@@ -438,7 +438,7 @@ grayscale_convert(j_compress_ptr cinfo, JSAMPARRAY input_buf,
    outptr = output_buf[0][output_row];
    output_row++;
    for (col = 0; col < num_cols; col++) {
-      outptr[col] = inptr[0];   /* don't need GETJSAMPLE() here */
+      outptr[col] = inptr[0];
      inptr += instride;
    }
  }
@@ -497,7 +497,7 @@ null_convert(j_compress_ptr cinfo, JSAMPARRAY input_buf, JSAMPIMAGE output_buf,
        inptr = *input_buf;
        outptr = output_buf[ci][output_row];
        for (col = 0; col < num_cols; col++) {
-          outptr[col] = inptr[ci]; /* don't need GETJSAMPLE() here */
+          outptr[col] = inptr[ci];
          inptr += nc;
        }
      }
@@ -381,19 +381,19 @@ convsamp(JSAMPARRAY sample_data, JDIMENSION start_col, DCTELEM *workspace)
    elemptr = sample_data[elemr] + start_col;

 #if DCTSIZE == 8                /* unroll the inner loop */
-    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;
-    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;
-    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;
-    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;
-    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;
-    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;
-    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;
-    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;
+    *workspaceptr++ = (*elemptr++) - CENTERJSAMPLE;
+    *workspaceptr++ = (*elemptr++) - CENTERJSAMPLE;
+    *workspaceptr++ = (*elemptr++) - CENTERJSAMPLE;
+    *workspaceptr++ = (*elemptr++) - CENTERJSAMPLE;
+    *workspaceptr++ = (*elemptr++) - CENTERJSAMPLE;
+    *workspaceptr++ = (*elemptr++) - CENTERJSAMPLE;
+    *workspaceptr++ = (*elemptr++) - CENTERJSAMPLE;
+    *workspaceptr++ = (*elemptr++) - CENTERJSAMPLE;
 #else
    {
      register int elemc;
      for (elemc = DCTSIZE; elemc > 0; elemc--)
-        *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;
+        *workspaceptr++ = (*elemptr++) - CENTERJSAMPLE;
    }
 #endif
  }
@@ -533,20 +533,19 @@ convsamp_float(JSAMPARRAY sample_data, JDIMENSION start_col,
  for (elemr = 0; elemr < DCTSIZE; elemr++) {
    elemptr = sample_data[elemr] + start_col;
 #if DCTSIZE == 8                /* unroll the inner loop */
-    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);
-    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);
-    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);
-    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);
-    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);
-    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);
-    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);
-    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);
+    *workspaceptr++ = (FAST_FLOAT)((*elemptr++) - CENTERJSAMPLE);
+    *workspaceptr++ = (FAST_FLOAT)((*elemptr++) - CENTERJSAMPLE);
+    *workspaceptr++ = (FAST_FLOAT)((*elemptr++) - CENTERJSAMPLE);
+    *workspaceptr++ = (FAST_FLOAT)((*elemptr++) - CENTERJSAMPLE);
+    *workspaceptr++ = (FAST_FLOAT)((*elemptr++) - CENTERJSAMPLE);
+    *workspaceptr++ = (FAST_FLOAT)((*elemptr++) - CENTERJSAMPLE);
+    *workspaceptr++ = (FAST_FLOAT)((*elemptr++) - CENTERJSAMPLE);
+    *workspaceptr++ = (FAST_FLOAT)((*elemptr++) - CENTERJSAMPLE);
 #else
    {
      register int elemc;
      for (elemc = DCTSIZE; elemc > 0; elemc--)
-        *workspaceptr++ = (FAST_FLOAT)
-                          (GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);
+        *workspaceptr++ = (FAST_FLOAT)((*elemptr++) - CENTERJSAMPLE);
    }
 #endif
  }
@@ -4,8 +4,10 @@
 * This file was part of the Independent JPEG Group's software:
 * Copyright (C) 1991-1997, Thomas G. Lane.
 * libjpeg-turbo Modifications:
- * Copyright (C) 2009-2011, 2014-2016, 2018-2019, D. R. Commander.
+ * Copyright (C) 2009-2011, 2014-2016, 2018-2021, D. R. Commander.
 * Copyright (C) 2015, Matthieu Darbois.
+ * Copyright (C) 2018, Matthias Räncker.
+ * Copyright (C) 2020, Arm Limited.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 *
@@ -43,14 +45,19 @@
 */

 /* NOTE: Both GCC and Clang define __GNUC__ */
-#if defined(__GNUC__) && (defined(__arm__) || defined(__aarch64__))
+#if (defined(__GNUC__) && (defined(__arm__) || defined(__aarch64__))) || \
+    defined(_M_ARM) || defined(_M_ARM64)
 #if !defined(__thumb__) || defined(__thumb2__)
 #define USE_CLZ_INTRINSIC
 #endif
 #endif

 #ifdef USE_CLZ_INTRINSIC
+#if defined(_MSC_VER) && !defined(__clang__)
+#define JPEG_NBITS_NONZERO(x)  (32 - _CountLeadingZeros(x))
+#else
 #define JPEG_NBITS_NONZERO(x)  (32 - __builtin_clz(x))
+#endif
 #define JPEG_NBITS(x)          (x ? JPEG_NBITS_NONZERO(x) : 0)
 #else
 #include "jpeg_nbits_table.h"
@@ -65,32 +72,43 @@
 * but must not be updated permanently until we complete the MCU.
 */

+#if defined(__x86_64__) && defined(__ILP32__)
+typedef unsigned long long bit_buf_type;
+#else
+typedef size_t bit_buf_type;
+#endif
+
+/* NOTE: The more optimal Huffman encoding algorithm is only used by the
+ * intrinsics implementation of the Arm Neon SIMD extensions, which is why we
+ * retain the old Huffman encoder behavior when using the GAS implementation.
+ */
+#if defined(WITH_SIMD) && !(defined(__arm__) || defined(__aarch64__) || \
+                            defined(_M_ARM) || defined(_M_ARM64))
+typedef unsigned long long simd_bit_buf_type;
+#else
+typedef bit_buf_type simd_bit_buf_type;
+#endif
+
+#if (defined(SIZEOF_SIZE_T) && SIZEOF_SIZE_T == 8) || defined(_WIN64) || \
+    (defined(__x86_64__) && defined(__ILP32__))
+#define BIT_BUF_SIZE  64
+#elif (defined(SIZEOF_SIZE_T) && SIZEOF_SIZE_T == 4) || defined(_WIN32)
+#define BIT_BUF_SIZE  32
+#else
+#error Cannot determine word size
+#endif
+#define SIMD_BIT_BUF_SIZE  (sizeof(simd_bit_buf_type) * 8)
+
 typedef struct {
-  size_t put_buffer;                    /* current bit-accumulation buffer */
-  int put_bits;                         /* # of bits now in it */
+  union {
+    bit_buf_type c;
+    simd_bit_buf_type simd;
+  } put_buffer;                         /* current bit accumulation buffer */
+  int free_bits;                        /* # of bits available in it */
+                                        /* (Neon GAS: # of bits now in it) */
  int last_dc_val[MAX_COMPS_IN_SCAN];   /* last DC coef for each component */
 } savable_state;

-/* This macro is to work around compilers with missing or broken
- * structure assignment.  You'll need to fix this code if you have
- * such a compiler and you change MAX_COMPS_IN_SCAN.
- */
-
-#ifndef NO_STRUCT_ASSIGN
-#define ASSIGN_STATE(dest, src)  ((dest) = (src))
-#else
-#if MAX_COMPS_IN_SCAN == 4
-#define ASSIGN_STATE(dest, src) \
-  ((dest).put_buffer = (src).put_buffer, \
-   (dest).put_bits = (src).put_bits, \
-   (dest).last_dc_val[0] = (src).last_dc_val[0], \
-   (dest).last_dc_val[1] = (src).last_dc_val[1], \
-   (dest).last_dc_val[2] = (src).last_dc_val[2], \
-   (dest).last_dc_val[3] = (src).last_dc_val[3])
-#endif
-#endif
-
-
 typedef struct {
  struct jpeg_entropy_encoder pub; /* public fields */

@@ -123,6 +141,7 @@ typedef struct {
  size_t free_in_buffer;        /* # of byte spaces remaining in buffer */
  savable_state cur;            /* Current bit buffer & DC state */
  j_compress_ptr cinfo;         /* dump_buffer needs access to this */
+  int simd;
 } working_state;


@@ -201,8 +220,17 @@ start_pass_huff(j_compress_ptr cinfo, boolean gather_statistics)
  }

  /* Initialize bit buffer to empty */
-  entropy->saved.put_buffer = 0;
-  entropy->saved.put_bits = 0;
+  if (entropy->simd) {
+    entropy->saved.put_buffer.simd = 0;
+#if defined(__aarch64__) && !defined(NEON_INTRINSICS)
+    entropy->saved.free_bits = 0;
+#else
+    entropy->saved.free_bits = SIMD_BIT_BUF_SIZE;
+#endif
+  } else {
+    entropy->saved.put_buffer.c = 0;
+    entropy->saved.free_bits = BIT_BUF_SIZE;
+  }

  /* Initialize restart stuff */
  entropy->restarts_to_go = cinfo->restart_interval;
@@ -287,6 +315,7 @@ jpeg_make_c_derived_tbl(j_compress_ptr cinfo, boolean isDC, int tblno,
   * this lets us detect duplicate VAL entries here, and later
   * allows emit_bits to detect any attempt to emit such symbols.
   */
+  MEMZERO(dtbl->ehufco, sizeof(dtbl->ehufco));
  MEMZERO(dtbl->ehufsi, sizeof(dtbl->ehufsi));

  /* This is also a convenient place to check for out-of-range
@@ -334,94 +363,94 @@ dump_buffer(working_state *state)

 /* Outputting bits to the file */

-/* These macros perform the same task as the emit_bits() function in the
- * original libjpeg code.  In addition to reducing overhead by explicitly
- * inlining the code, additional performance is achieved by taking into
- * account the size of the bit buffer and waiting until it is almost full
- * before emptying it.  This mostly benefits 64-bit platforms, since 6
- * bytes can be stored in a 64-bit bit buffer before it has to be emptied.
+/* Output byte b and, speculatively, an additional 0 byte.  0xFF must be
+ * encoded as 0xFF 0x00, so the output buffer pointer is advanced by 2 if the
+ * byte is 0xFF.  Otherwise, the output buffer pointer is advanced by 1, and
+ * the speculative 0 byte will be overwritten by the next byte.
 */
-
-#define EMIT_BYTE() { \
-  JOCTET c; \
-  put_bits -= 8; \
-  c = (JOCTET)GETJOCTET(put_buffer >> put_bits); \
-  *buffer++ = c; \
-  if (c == 0xFF)  /* need to stuff a zero byte? */ \
-    *buffer++ = 0; \
+#define EMIT_BYTE(b) { \
+  buffer[0] = (JOCTET)(b); \
+  buffer[1] = 0; \
+  buffer -= -2 + ((JOCTET)(b) < 0xFF); \
 }

-#define PUT_BITS(code, size) { \
-  put_bits += size; \
-  put_buffer = (put_buffer << size) | code; \
-}
+/* Output the entire bit buffer.  If there are no 0xFF bytes in it, then write
+ * directly to the output buffer.  Otherwise, use the EMIT_BYTE() macro to
+ * encode 0xFF as 0xFF 0x00.
+ */
+#if BIT_BUF_SIZE == 64

-#if SIZEOF_SIZE_T != 8 && !defined(_WIN64)
-
-#define CHECKBUF15() { \
-  if (put_bits > 15) { \
-    EMIT_BYTE() \
-    EMIT_BYTE() \
+#define FLUSH() { \
+  if (put_buffer & 0x8080808080808080 & ~(put_buffer + 0x0101010101010101)) { \
+    EMIT_BYTE(put_buffer >> 56) \
+    EMIT_BYTE(put_buffer >> 48) \
+    EMIT_BYTE(put_buffer >> 40) \
+    EMIT_BYTE(put_buffer >> 32) \
+    EMIT_BYTE(put_buffer >> 24) \
+    EMIT_BYTE(put_buffer >> 16) \
+    EMIT_BYTE(put_buffer >>  8) \
+    EMIT_BYTE(put_buffer      ) \
+  } else { \
+    buffer[0] = (JOCTET)(put_buffer >> 56); \
+    buffer[1] = (JOCTET)(put_buffer >> 48); \
+    buffer[2] = (JOCTET)(put_buffer >> 40); \
+    buffer[3] = (JOCTET)(put_buffer >> 32); \
+    buffer[4] = (JOCTET)(put_buffer >> 24); \
+    buffer[5] = (JOCTET)(put_buffer >> 16); \
+    buffer[6] = (JOCTET)(put_buffer >> 8); \
+    buffer[7] = (JOCTET)(put_buffer); \
+    buffer += 8; \
  } \
 }

-#endif
-
-#define CHECKBUF31() { \
-  if (put_bits > 31) { \
-    EMIT_BYTE() \
-    EMIT_BYTE() \
-    EMIT_BYTE() \
-    EMIT_BYTE() \
-  } \
-}
-
-#define CHECKBUF47() { \
-  if (put_bits > 47) { \
-    EMIT_BYTE() \
-    EMIT_BYTE() \
-    EMIT_BYTE() \
-    EMIT_BYTE() \
-    EMIT_BYTE() \
-    EMIT_BYTE() \
-  } \
-}
-
-#if !defined(_WIN32) && !defined(SIZEOF_SIZE_T)
-#error Cannot determine word size
-#endif
-
-#if SIZEOF_SIZE_T == 8 || defined(_WIN64)
-
-#define EMIT_BITS(code, size) { \
-  CHECKBUF47() \
-  PUT_BITS(code, size) \
-}
-
-#define EMIT_CODE(code, size) { \
-  temp2 &= (((JLONG)1) << nbits) - 1; \
-  CHECKBUF31() \
-  PUT_BITS(code, size) \
-  PUT_BITS(temp2, nbits) \
-}
-
 #else

-#define EMIT_BITS(code, size) { \
-  PUT_BITS(code, size) \
-  CHECKBUF15() \
-}
-
-#define EMIT_CODE(code, size) { \
-  temp2 &= (((JLONG)1) << nbits) - 1; \
-  PUT_BITS(code, size) \
-  CHECKBUF15() \
-  PUT_BITS(temp2, nbits) \
-  CHECKBUF15() \
+#define FLUSH() { \
+  if (put_buffer & 0x80808080 & ~(put_buffer + 0x01010101)) { \
+    EMIT_BYTE(put_buffer >> 24) \
+    EMIT_BYTE(put_buffer >> 16) \
+    EMIT_BYTE(put_buffer >>  8) \
+    EMIT_BYTE(put_buffer      ) \
+  } else { \
+    buffer[0] = (JOCTET)(put_buffer >> 24); \
+    buffer[1] = (JOCTET)(put_buffer >> 16); \
+    buffer[2] = (JOCTET)(put_buffer >> 8); \
+    buffer[3] = (JOCTET)(put_buffer); \
+    buffer += 4; \
+  } \
 }

 #endif

+/* Fill the bit buffer to capacity with the leading bits from code, then output
+ * the bit buffer and put the remaining bits from code into the bit buffer.
+ */
+#define PUT_AND_FLUSH(code, size) { \
+  put_buffer = (put_buffer << (size + free_bits)) | (code >> -free_bits); \
+  FLUSH() \
+  free_bits += BIT_BUF_SIZE; \
+  put_buffer = code; \
+}
+
+/* Insert code into the bit buffer and output the bit buffer if needed.
+ * NOTE: We can't flush with free_bits == 0, since the left shift in
+ * PUT_AND_FLUSH() would have undefined behavior.
+ */
+#define PUT_BITS(code, size) { \
+  free_bits -= size; \
+  if (free_bits < 0) \
+    PUT_AND_FLUSH(code, size) \
+  else \
+    put_buffer = (put_buffer << size) | code; \
+}
+
+#define PUT_CODE(code, size) { \
+  temp &= (((JLONG)1) << nbits) - 1; \
+  temp |= code << nbits; \
+  nbits += size; \
+  PUT_BITS(temp, nbits) \
+}
+

 /* Although it is exceedingly rare, it is possible for a Huffman-encoded
 * coefficient block to be larger than the 128-byte unencoded block.  For each
@@ -444,6 +473,7 @@ dump_buffer(working_state *state)

 #define STORE_BUFFER() { \
  if (localbuf) { \
+    size_t bytes, bytestocopy; \
    bytes = buffer - _buffer; \
    buffer = _buffer; \
    while (bytes > 0) { \
@@ -466,20 +496,46 @@ dump_buffer(working_state *state)
 LOCAL(boolean)
 flush_bits(working_state *state)
 {
-  JOCTET _buffer[BUFSIZE], *buffer;
-  size_t put_buffer;  int put_bits;
-  size_t bytes, bytestocopy;  int localbuf = 0;
+  JOCTET _buffer[BUFSIZE], *buffer, temp;
+  simd_bit_buf_type put_buffer;  int put_bits;
+  int localbuf = 0;
+
+  if (state->simd) {
+#if defined(__aarch64__) && !defined(NEON_INTRINSICS)
+    put_bits = state->cur.free_bits;
+#else
+    put_bits = SIMD_BIT_BUF_SIZE - state->cur.free_bits;
+#endif
+    put_buffer = state->cur.put_buffer.simd;
+  } else {
+    put_bits = BIT_BUF_SIZE - state->cur.free_bits;
+    put_buffer = state->cur.put_buffer.c;
+  }

-  put_buffer = state->cur.put_buffer;
-  put_bits = state->cur.put_bits;
  LOAD_BUFFER()

-  /* fill any partial byte with ones */
-  PUT_BITS(0x7F, 7)
-  while (put_bits >= 8) EMIT_BYTE()
+  while (put_bits >= 8) {
+    put_bits -= 8;
+    temp = (JOCTET)(put_buffer >> put_bits);
+    EMIT_BYTE(temp)
+  }
+  if (put_bits) {
+    /* fill partial byte with ones */
+    temp = (JOCTET)((put_buffer << (8 - put_bits)) | (0xFF >> put_bits));
+    EMIT_BYTE(temp)
+  }

-  state->cur.put_buffer = 0;    /* and reset bit-buffer to empty */
-  state->cur.put_bits = 0;
+  if (state->simd) {                    /* and reset bit buffer to empty */
+    state->cur.put_buffer.simd = 0;
+#if defined(__aarch64__) && !defined(NEON_INTRINSICS)
+    state->cur.free_bits = 0;
+#else
+    state->cur.free_bits = SIMD_BIT_BUF_SIZE;
+#endif
+  } else {
+    state->cur.put_buffer.c = 0;
+    state->cur.free_bits = BIT_BUF_SIZE;
+  }
  STORE_BUFFER()

  return TRUE;
@@ -493,7 +549,7 @@ encode_one_block_simd(working_state *state, JCOEFPTR block, int last_dc_val,
                      c_derived_tbl *dctbl, c_derived_tbl *actbl)
 {
  JOCTET _buffer[BUFSIZE], *buffer;
-  size_t bytes, bytestocopy;  int localbuf = 0;
+  int localbuf = 0;

  LOAD_BUFFER()

@@ -509,53 +565,41 @@ LOCAL(boolean)
 encode_one_block(working_state *state, JCOEFPTR block, int last_dc_val,
                 c_derived_tbl *dctbl, c_derived_tbl *actbl)
 {
-  int temp, temp2, temp3;
-  int nbits;
-  int r, code, size;
+  int temp, nbits, free_bits;
+  bit_buf_type put_buffer;
  JOCTET _buffer[BUFSIZE], *buffer;
-  size_t put_buffer;  int put_bits;
-  int code_0xf0 = actbl->ehufco[0xf0], size_0xf0 = actbl->ehufsi[0xf0];
-  size_t bytes, bytestocopy;  int localbuf = 0;
+  int localbuf = 0;

-  put_buffer = state->cur.put_buffer;
-  put_bits = state->cur.put_bits;
+  free_bits = state->cur.free_bits;
+  put_buffer = state->cur.put_buffer.c;
  LOAD_BUFFER()

  /* Encode the DC coefficient difference per section F.1.2.1 */

-  temp = temp2 = block[0] - last_dc_val;
+  temp = block[0] - last_dc_val;

  /* This is a well-known technique for obtaining the absolute value without a
   * branch.  It is derived from an assembly language technique presented in
   * "How to Optimize for the Pentium Processors", Copyright (c) 1996, 1997 by
-   * Agner Fog.
+   * Agner Fog.  This code assumes we are on a two's complement machine.
   */
-  temp3 = temp >> (CHAR_BIT * sizeof(int) - 1);
-  temp ^= temp3;
-  temp -= temp3;
-
-  /* For a negative input, want temp2 = bitwise complement of abs(input) */
-  /* This code assumes we are on a two's complement machine */
-  temp2 += temp3;
+  nbits = temp >> (CHAR_BIT * sizeof(int) - 1);
+  temp += nbits;
+  nbits ^= temp;

  /* Find the number of bits needed for the magnitude of the coefficient */
-  nbits = JPEG_NBITS(temp);
+  nbits = JPEG_NBITS(nbits);

-  /* Emit the Huffman-coded symbol for the number of bits */
-  code = dctbl->ehufco[nbits];
-  size = dctbl->ehufsi[nbits];
-  EMIT_BITS(code, size)
-
-  /* Mask off any extra bits in code */
-  temp2 &= (((JLONG)1) << nbits) - 1;
-
-  /* Emit that number of bits of the value, if positive, */
-  /* or the complement of its magnitude, if negative. */
-  EMIT_BITS(temp2, nbits)
+  /* Emit the Huffman-coded symbol for the number of bits.
+   * Emit that number of bits of the value, if positive,
+   * or the complement of its magnitude, if negative.
+   */
+  PUT_CODE(dctbl->ehufco[nbits], dctbl->ehufsi[nbits])

  /* Encode the AC coefficients per section F.1.2.2 */

-  r = 0;                        /* r = run length of zeros */
+  {
+    int r = 0;                  /* r = run length of zeros */

 /* Manually unroll the k loop to eliminate the counter variable.  This
 * improves performance greatly on systems with a limited number of
@@ -563,51 +607,46 @@ encode_one_block(working_state *state, JCOEFPTR block, int last_dc_val,
 */
 #define kloop(jpeg_natural_order_of_k) { \
  if ((temp = block[jpeg_natural_order_of_k]) == 0) { \
-    r++; \
+    r += 16; \
  } else { \
-    temp2 = temp; \
    /* Branch-less absolute value, bitwise complement, etc., same as above */ \
-    temp3 = temp >> (CHAR_BIT * sizeof(int) - 1); \
-    temp ^= temp3; \
-    temp -= temp3; \
-    temp2 += temp3; \
-    nbits = JPEG_NBITS_NONZERO(temp); \
+    nbits = temp >> (CHAR_BIT * sizeof(int) - 1); \
+    temp += nbits; \
+    nbits ^= temp; \
+    nbits = JPEG_NBITS_NONZERO(nbits); \
    /* if run length > 15, must emit special run-length-16 codes (0xF0) */ \
-    while (r > 15) { \
-      EMIT_BITS(code_0xf0, size_0xf0) \
-      r -= 16; \
+    while (r >= 16 * 16) { \
+      r -= 16 * 16; \
+      PUT_BITS(actbl->ehufco[0xf0], actbl->ehufsi[0xf0]) \
    } \
    /* Emit Huffman symbol for run length / number of bits */ \
-    temp3 = (r << 4) + nbits; \
-    code = actbl->ehufco[temp3]; \
-    size = actbl->ehufsi[temp3]; \
-    EMIT_CODE(code, size) \
+    r += nbits; \
+    PUT_CODE(actbl->ehufco[r], actbl->ehufsi[r]) \
    r = 0; \
  } \
 }

-  /* One iteration for each value in jpeg_natural_order[] */
-  kloop(1);   kloop(8);   kloop(16);  kloop(9);   kloop(2);   kloop(3);
-  kloop(10);  kloop(17);  kloop(24);  kloop(32);  kloop(25);  kloop(18);
-  kloop(11);  kloop(4);   kloop(5);   kloop(12);  kloop(19);  kloop(26);
-  kloop(33);  kloop(40);  kloop(48);  kloop(41);  kloop(34);  kloop(27);
-  kloop(20);  kloop(13);  kloop(6);   kloop(7);   kloop(14);  kloop(21);
-  kloop(28);  kloop(35);  kloop(42);  kloop(49);  kloop(56);  kloop(57);
-  kloop(50);  kloop(43);  kloop(36);  kloop(29);  kloop(22);  kloop(15);
-  kloop(23);  kloop(30);  kloop(37);  kloop(44);  kloop(51);  kloop(58);
-  kloop(59);  kloop(52);  kloop(45);  kloop(38);  kloop(31);  kloop(39);
-  kloop(46);  kloop(53);  kloop(60);  kloop(61);  kloop(54);  kloop(47);
-  kloop(55);  kloop(62);  kloop(63);
+    /* One iteration for each value in jpeg_natural_order[] */
+    kloop(1);   kloop(8);   kloop(16);  kloop(9);   kloop(2);   kloop(3);
+    kloop(10);  kloop(17);  kloop(24);  kloop(32);  kloop(25);  kloop(18);
+    kloop(11);  kloop(4);   kloop(5);   kloop(12);  kloop(19);  kloop(26);
+    kloop(33);  kloop(40);  kloop(48);  kloop(41);  kloop(34);  kloop(27);
+    kloop(20);  kloop(13);  kloop(6);   kloop(7);   kloop(14);  kloop(21);
+    kloop(28);  kloop(35);  kloop(42);  kloop(49);  kloop(56);  kloop(57);
+    kloop(50);  kloop(43);  kloop(36);  kloop(29);  kloop(22);  kloop(15);
+    kloop(23);  kloop(30);  kloop(37);  kloop(44);  kloop(51);  kloop(58);
+    kloop(59);  kloop(52);  kloop(45);  kloop(38);  kloop(31);  kloop(39);
+    kloop(46);  kloop(53);  kloop(60);  kloop(61);  kloop(54);  kloop(47);
+    kloop(55);  kloop(62);  kloop(63);

-  /* If the last coef(s) were zero, emit an end-of-block code */
-  if (r > 0) {
-    code = actbl->ehufco[0];
-    size = actbl->ehufsi[0];
-    EMIT_BITS(code, size)
+    /* If the last coef(s) were zero, emit an end-of-block code */
+    if (r > 0) {
+      PUT_BITS(actbl->ehufco[0], actbl->ehufsi[0])
+    }
  }

-  state->cur.put_buffer = put_buffer;
-  state->cur.put_bits = put_bits;
+  state->cur.put_buffer.c = put_buffer;
+  state->cur.free_bits = free_bits;
  STORE_BUFFER()

  return TRUE;
@@ -654,8 +693,9 @@ encode_mcu_huff(j_compress_ptr cinfo, JBLOCKROW *MCU_data)
  /* Load up working state */
  state.next_output_byte = cinfo->dest->next_output_byte;
  state.free_in_buffer = cinfo->dest->free_in_buffer;
-  ASSIGN_STATE(state.cur, entropy->saved);
+  state.cur = entropy->saved;
  state.cinfo = cinfo;
+  state.simd = entropy->simd;

  /* Emit restart marker if needed */
  if (cinfo->restart_interval) {
@@ -694,7 +734,7 @@ encode_mcu_huff(j_compress_ptr cinfo, JBLOCKROW *MCU_data)
  /* Completed MCU, so update state */
  cinfo->dest->next_output_byte = state.next_output_byte;
  cinfo->dest->free_in_buffer = state.free_in_buffer;
-  ASSIGN_STATE(entropy->saved, state.cur);
+  entropy->saved = state.cur;

  /* Update restart-interval state too */
  if (cinfo->restart_interval) {
@@ -723,8 +763,9 @@ finish_pass_huff(j_compress_ptr cinfo)
  /* Load up working state ... flush_bits needs it */
  state.next_output_byte = cinfo->dest->next_output_byte;
  state.free_in_buffer = cinfo->dest->free_in_buffer;
-  ASSIGN_STATE(state.cur, entropy->saved);
+  state.cur = entropy->saved;
  state.cinfo = cinfo;
+  state.simd = entropy->simd;

  /* Flush out the last data */
  if (!flush_bits(&state))
@@ -733,7 +774,7 @@ finish_pass_huff(j_compress_ptr cinfo)
  /* Update state */
  cinfo->dest->next_output_byte = state.next_output_byte;
  cinfo->dest->free_in_buffer = state.free_in_buffer;
-  ASSIGN_STATE(entropy->saved, state.cur);
+  entropy->saved = state.cur;
 }


@@ -493,7 +493,7 @@ prepare_for_pass(j_compress_ptr cinfo)
    master->pass_type = output_pass;
    master->pass_number++;
 #endif
-    /*FALLTHROUGH*/
+    FALLTHROUGH                 /*FALLTHROUGH*/
  case output_pass:
    /* Do a data-output pass. */
    /* We need not repeat per-scan setup if prior optimization pass did it. */
@@ -4,8 +4,10 @@
 * This file was part of the Independent JPEG Group's software:
 * Copyright (C) 1995-1997, Thomas G. Lane.
 * libjpeg-turbo Modifications:
- * Copyright (C) 2011, 2015, 2018, D. R. Commander.
+ * Copyright (C) 2011, 2015, 2018, 2021, D. R. Commander.
 * Copyright (C) 2016, 2018, Matthieu Darbois.
+ * Copyright (C) 2020, Arm Limited.
+ * Copyright (C) 2021, Alex Richardson.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 *
@@ -52,14 +54,19 @@
 */

 /* NOTE: Both GCC and Clang define __GNUC__ */
-#if defined(__GNUC__) && (defined(__arm__) || defined(__aarch64__))
+#if (defined(__GNUC__) && (defined(__arm__) || defined(__aarch64__))) || \
+    defined(_M_ARM) || defined(_M_ARM64)
 #if !defined(__thumb__) || defined(__thumb2__)
 #define USE_CLZ_INTRINSIC
 #endif
 #endif

 #ifdef USE_CLZ_INTRINSIC
+#if defined(_MSC_VER) && !defined(__clang__)
+#define JPEG_NBITS_NONZERO(x)  (32 - _CountLeadingZeros(x))
+#else
 #define JPEG_NBITS_NONZERO(x)  (32 - __builtin_clz(x))
+#endif
 #define JPEG_NBITS(x)          (x ? JPEG_NBITS_NONZERO(x) : 0)
 #else
 #include "jpeg_nbits_table.h"
@@ -169,24 +176,26 @@ INLINE
 METHODDEF(int)
 count_zeroes(size_t *x)
 {
-  int result;
 #if defined(HAVE_BUILTIN_CTZL)
+  int result;
  result = __builtin_ctzl(*x);
  *x >>= result;
 #elif defined(HAVE_BITSCANFORWARD64)
+  unsigned long result;
  _BitScanForward64(&result, *x);
  *x >>= result;
 #elif defined(HAVE_BITSCANFORWARD)
+  unsigned long result;
  _BitScanForward(&result, *x);
  *x >>= result;
 #else
-  result = 0;
+  int result = 0;
  while ((*x & 1) == 0) {
    ++result;
    *x >>= 1;
  }
 #endif
-  return result;
+  return (int)result;
 }


@@ -672,7 +681,7 @@ encode_mcu_AC_first(j_compress_ptr cinfo, JBLOCKROW *MCU_data)
      emit_restart(entropy, entropy->next_restart_num);

 #ifdef WITH_SIMD
-  cvalue = values = (JCOEF *)PAD((size_t)values_unaligned, 16);
+  cvalue = values = (JCOEF *)PAD((JUINTPTR)values_unaligned, 16);
 #else
  /* Not using SIMD, so alignment is not needed */
  cvalue = values = values_unaligned;
@@ -860,7 +869,7 @@ encode_mcu_AC_refine_prepare(const JCOEF *block,

 #define ENCODE_COEFS_AC_REFINE(label) { \
  while (zerobits) { \
-    int idx = count_zeroes(&zerobits); \
+    idx = count_zeroes(&zerobits); \
    r += idx; \
    cabsvalue += idx; \
    signbits >>= idx; \
@@ -917,7 +926,7 @@ METHODDEF(boolean)
 encode_mcu_AC_refine(j_compress_ptr cinfo, JBLOCKROW *MCU_data)
 {
  phuff_entropy_ptr entropy = (phuff_entropy_ptr)cinfo->entropy;
-  register int temp, r;
+  register int temp, r, idx;
  char *BR_buffer;
  unsigned int BR;
  int Sl = cinfo->Se - cinfo->Ss + 1;
@@ -937,7 +946,7 @@ encode_mcu_AC_refine(j_compress_ptr cinfo, JBLOCKROW *MCU_data)
      emit_restart(entropy, entropy->next_restart_num);

 #ifdef WITH_SIMD
-  cabsvalue = absvalues = (JCOEF *)PAD((size_t)absvalues_unaligned, 16);
+  cabsvalue = absvalues = (JCOEF *)PAD((JUINTPTR)absvalues_unaligned, 16);
 #else
  /* Not using SIMD, so alignment is not needed */
  cabsvalue = absvalues = absvalues_unaligned;
@@ -968,7 +977,7 @@ encode_mcu_AC_refine(j_compress_ptr cinfo, JBLOCKROW *MCU_data)

  if (zerobits) {
    int diff = ((absvalues + DCTSIZE2 / 2) - cabsvalue);
-    int idx = count_zeroes(&zerobits);
+    idx = count_zeroes(&zerobits);
    signbits >>= idx;
    idx += diff;
    r += idx;
@@ -6,7 +6,7 @@
 * libjpeg-turbo Modifications:
 * Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
 * Copyright (C) 2014, MIPS Technologies, Inc., California.
- * Copyright (C) 2015, D. R. Commander.
+ * Copyright (C) 2015, 2019, D. R. Commander.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 *
@@ -103,7 +103,7 @@ expand_right_edge(JSAMPARRAY image_data, int num_rows, JDIMENSION input_cols,
  if (numcols > 0) {
    for (row = 0; row < num_rows; row++) {
      ptr = image_data[row] + input_cols;
-      pixval = ptr[-1];         /* don't need GETJSAMPLE() here */
+      pixval = ptr[-1];
      for (count = numcols; count > 0; count--)
        *ptr++ = pixval;
    }
@@ -174,7 +174,7 @@ int_downsample(j_compress_ptr cinfo, jpeg_component_info *compptr,
      for (v = 0; v < v_expand; v++) {
        inptr = input_data[inrow + v] + outcol_h;
        for (h = 0; h < h_expand; h++) {
-          outvalue += (JLONG)GETJSAMPLE(*inptr++);
+          outvalue += (JLONG)(*inptr++);
        }
      }
      *outptr++ = (JSAMPLE)((outvalue + numpix2) / numpix);
@@ -237,8 +237,7 @@ h2v1_downsample(j_compress_ptr cinfo, jpeg_component_info *compptr,
    inptr = input_data[outrow];
    bias = 0;                   /* bias = 0,1,0,1,... for successive samples */
    for (outcol = 0; outcol < output_cols; outcol++) {
-      *outptr++ =
-        (JSAMPLE)((GETJSAMPLE(*inptr) + GETJSAMPLE(inptr[1]) + bias) >> 1);
+      *outptr++ = (JSAMPLE)((inptr[0] + inptr[1] + bias) >> 1);
      bias ^= 1;                /* 0=>1, 1=>0 */
      inptr += 2;
    }
@@ -277,8 +276,7 @@ h2v2_downsample(j_compress_ptr cinfo, jpeg_component_info *compptr,
    bias = 1;                   /* bias = 1,2,1,2,... for successive samples */
    for (outcol = 0; outcol < output_cols; outcol++) {
      *outptr++ =
-        (JSAMPLE)((GETJSAMPLE(*inptr0) + GETJSAMPLE(inptr0[1]) +
-                   GETJSAMPLE(*inptr1) + GETJSAMPLE(inptr1[1]) + bias) >> 2);
+        (JSAMPLE)((inptr0[0] + inptr0[1] + inptr1[0] + inptr1[1] + bias) >> 2);
      bias ^= 3;                /* 1=>2, 2=>1 */
      inptr0 += 2;  inptr1 += 2;
    }
@@ -337,33 +335,25 @@ h2v2_smooth_downsample(j_compress_ptr cinfo, jpeg_component_info *compptr,
    below_ptr = input_data[inrow + 2];

    /* Special case for first column: pretend column -1 is same as column 0 */
-    membersum = GETJSAMPLE(*inptr0) + GETJSAMPLE(inptr0[1]) +
-                GETJSAMPLE(*inptr1) + GETJSAMPLE(inptr1[1]);
-    neighsum = GETJSAMPLE(*above_ptr) + GETJSAMPLE(above_ptr[1]) +
-               GETJSAMPLE(*below_ptr) + GETJSAMPLE(below_ptr[1]) +
-               GETJSAMPLE(*inptr0) + GETJSAMPLE(inptr0[2]) +
-               GETJSAMPLE(*inptr1) + GETJSAMPLE(inptr1[2]);
+    membersum = inptr0[0] + inptr0[1] + inptr1[0] + inptr1[1];
+    neighsum = above_ptr[0] + above_ptr[1] + below_ptr[0] + below_ptr[1] +
+               inptr0[0] + inptr0[2] + inptr1[0] + inptr1[2];
    neighsum += neighsum;
-    neighsum += GETJSAMPLE(*above_ptr) + GETJSAMPLE(above_ptr[2]) +
-                GETJSAMPLE(*below_ptr) + GETJSAMPLE(below_ptr[2]);
+    neighsum += above_ptr[0] + above_ptr[2] + below_ptr[0] + below_ptr[2];
    membersum = membersum * memberscale + neighsum * neighscale;
    *outptr++ = (JSAMPLE)((membersum + 32768) >> 16);
    inptr0 += 2;  inptr1 += 2;  above_ptr += 2;  below_ptr += 2;

    for (colctr = output_cols - 2; colctr > 0; colctr--) {
      /* sum of pixels directly mapped to this output element */
-      membersum = GETJSAMPLE(*inptr0) + GETJSAMPLE(inptr0[1]) +
-                  GETJSAMPLE(*inptr1) + GETJSAMPLE(inptr1[1]);
+      membersum = inptr0[0] + inptr0[1] + inptr1[0] + inptr1[1];
      /* sum of edge-neighbor pixels */
-      neighsum = GETJSAMPLE(*above_ptr) + GETJSAMPLE(above_ptr[1]) +
-                 GETJSAMPLE(*below_ptr) + GETJSAMPLE(below_ptr[1]) +
-                 GETJSAMPLE(inptr0[-1]) + GETJSAMPLE(inptr0[2]) +
-                 GETJSAMPLE(inptr1[-1]) + GETJSAMPLE(inptr1[2]);
+      neighsum = above_ptr[0] + above_ptr[1] + below_ptr[0] + below_ptr[1] +
+                 inptr0[-1] + inptr0[2] + inptr1[-1] + inptr1[2];
      /* The edge-neighbors count twice as much as corner-neighbors */
      neighsum += neighsum;
      /* Add in the corner-neighbors */
-      neighsum += GETJSAMPLE(above_ptr[-1]) + GETJSAMPLE(above_ptr[2]) +
-                  GETJSAMPLE(below_ptr[-1]) + GETJSAMPLE(below_ptr[2]);
+      neighsum += above_ptr[-1] + above_ptr[2] + below_ptr[-1] + below_ptr[2];
      /* form final output scaled up by 2^16 */
      membersum = membersum * memberscale + neighsum * neighscale;
      /* round, descale and output it */
@@ -372,15 +362,11 @@ h2v2_smooth_downsample(j_compress_ptr cinfo, jpeg_component_info *compptr,
    }

    /* Special case for last column */
-    membersum = GETJSAMPLE(*inptr0) + GETJSAMPLE(inptr0[1]) +
-                GETJSAMPLE(*inptr1) + GETJSAMPLE(inptr1[1]);
-    neighsum = GETJSAMPLE(*above_ptr) + GETJSAMPLE(above_ptr[1]) +
-               GETJSAMPLE(*below_ptr) + GETJSAMPLE(below_ptr[1]) +
-               GETJSAMPLE(inptr0[-1]) + GETJSAMPLE(inptr0[1]) +
-               GETJSAMPLE(inptr1[-1]) + GETJSAMPLE(inptr1[1]);
+    membersum = inptr0[0] + inptr0[1] + inptr1[0] + inptr1[1];
+    neighsum = above_ptr[0] + above_ptr[1] + below_ptr[0] + below_ptr[1] +
+               inptr0[-1] + inptr0[1] + inptr1[-1] + inptr1[1];
    neighsum += neighsum;
-    neighsum += GETJSAMPLE(above_ptr[-1]) + GETJSAMPLE(above_ptr[1]) +
-                GETJSAMPLE(below_ptr[-1]) + GETJSAMPLE(below_ptr[1]);
+    neighsum += above_ptr[-1] + above_ptr[1] + below_ptr[-1] + below_ptr[1];
    membersum = membersum * memberscale + neighsum * neighscale;
    *outptr = (JSAMPLE)((membersum + 32768) >> 16);

@@ -429,21 +415,18 @@ fullsize_smooth_downsample(j_compress_ptr cinfo, jpeg_component_info *compptr,
    below_ptr = input_data[outrow + 1];

    /* Special case for first column */
-    colsum = GETJSAMPLE(*above_ptr++) + GETJSAMPLE(*below_ptr++) +
-             GETJSAMPLE(*inptr);
-    membersum = GETJSAMPLE(*inptr++);
-    nextcolsum = GETJSAMPLE(*above_ptr) + GETJSAMPLE(*below_ptr) +
-                 GETJSAMPLE(*inptr);
+    colsum = (*above_ptr++) + (*below_ptr++) + inptr[0];
+    membersum = *inptr++;
+    nextcolsum = above_ptr[0] + below_ptr[0] + inptr[0];
    neighsum = colsum + (colsum - membersum) + nextcolsum;
    membersum = membersum * memberscale + neighsum * neighscale;
    *outptr++ = (JSAMPLE)((membersum + 32768) >> 16);
    lastcolsum = colsum;  colsum = nextcolsum;

    for (colctr = output_cols - 2; colctr > 0; colctr--) {
-      membersum = GETJSAMPLE(*inptr++);
+      membersum = *inptr++;
      above_ptr++;  below_ptr++;
-      nextcolsum = GETJSAMPLE(*above_ptr) + GETJSAMPLE(*below_ptr) +
-                   GETJSAMPLE(*inptr);
+      nextcolsum = above_ptr[0] + below_ptr[0] + inptr[0];
      neighsum = lastcolsum + (colsum - membersum) + nextcolsum;
      membersum = membersum * memberscale + neighsum * neighscale;
      *outptr++ = (JSAMPLE)((membersum + 32768) >> 16);
@@ -451,7 +434,7 @@ fullsize_smooth_downsample(j_compress_ptr cinfo, jpeg_component_info *compptr,
    }

    /* Special case for last column */
-    membersum = GETJSAMPLE(*inptr);
+    membersum = *inptr;
    neighsum = lastcolsum + (colsum - membersum) + colsum;
    membersum = membersum * memberscale + neighsum * neighscale;
    *outptr = (JSAMPLE)((membersum + 32768) >> 16);
@@ -23,6 +23,7 @@
 #include "jinclude.h"
 #include "jpeglib.h"
 #include "jdmaster.h"
+#include "jconfigint.h"


 /*
@@ -308,7 +309,7 @@ jpeg_consume_input(j_decompress_ptr cinfo)
    /* Initialize application's data source module */
    (*cinfo->src->init_source) (cinfo);
    cinfo->global_state = DSTATE_INHEADER;
-    /*FALLTHROUGH*/
+    FALLTHROUGH                 /*FALLTHROUGH*/
  case DSTATE_INHEADER:
    retcode = (*cinfo->inputctl->consume_input) (cinfo);
    if (retcode == JPEG_REACHED_SOS) { /* Found SOS, prepare to decompress */
@@ -4,7 +4,7 @@
 * This file was part of the Independent JPEG Group's software:
 * Copyright (C) 1994-1996, Thomas G. Lane.
 * libjpeg-turbo Modifications:
- * Copyright (C) 2010, 2015-2018, 2020, D. R. Commander.
+ * Copyright (C) 2010, 2015-2020, D. R. Commander.
 * Copyright (C) 2015, Google, Inc.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
@@ -319,6 +319,8 @@ read_and_discard_scanlines(j_decompress_ptr cinfo, JDIMENSION num_lines)
 {
  JDIMENSION n;
  my_master_ptr master = (my_master_ptr)cinfo->master;
+  JSAMPLE dummy_sample[1] = { 0 };
+  JSAMPROW dummy_row = dummy_sample;
  JSAMPARRAY scanlines = NULL;
  void (*color_convert) (j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
                         JDIMENSION input_row, JSAMPARRAY output_buf,
@@ -329,6 +331,10 @@ read_and_discard_scanlines(j_decompress_ptr cinfo, JDIMENSION num_lines)
  if (cinfo->cconvert && cinfo->cconvert->color_convert) {
    color_convert = cinfo->cconvert->color_convert;
    cinfo->cconvert->color_convert = noop_convert;
+    /* This just prevents UBSan from complaining about adding 0 to a NULL
+     * pointer.  The pointer isn't actually used.
+     */
+    scanlines = &dummy_row;
  }

  if (cinfo->cquantize && cinfo->cquantize->color_quantize) {
@@ -532,6 +538,8 @@ jpeg_skip_scanlines(j_decompress_ptr cinfo, JDIMENSION num_lines)
         * decoded coefficients.  This is ~5% faster for large subsets, but
         * it's tough to tell a difference for smaller images.
         */
+        if (!cinfo->entropy->insufficient_data)
+          cinfo->master->last_good_iMCU_row = cinfo->input_iMCU_row;
        (*cinfo->entropy->decode_mcu) (cinfo, NULL);
      }
    }
@@ -4,7 +4,7 @@
 * This file was part of the Independent JPEG Group's software:
 * Developed 1997-2015 by Guido Vollbeding.
 * libjpeg-turbo Modifications:
- * Copyright (C) 2015-2018, D. R. Commander.
+ * Copyright (C) 2015-2020, D. R. Commander.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 *
@@ -80,7 +80,7 @@ get_byte(j_decompress_ptr cinfo)
    if (!(*src->fill_input_buffer) (cinfo))
      ERREXIT(cinfo, JERR_CANT_SUSPEND);
  src->bytes_in_buffer--;
-  return GETJOCTET(*src->next_input_byte++);
+  return *src->next_input_byte++;
 }


@@ -665,8 +665,16 @@ bad:
    for (ci = 0; ci < cinfo->comps_in_scan; ci++) {
      int coefi, cindex = cinfo->cur_comp_info[ci]->component_index;
      int *coef_bit_ptr = &cinfo->coef_bits[cindex][0];
+      int *prev_coef_bit_ptr =
+        &cinfo->coef_bits[cindex + cinfo->num_components][0];
      if (cinfo->Ss && coef_bit_ptr[0] < 0) /* AC without prior DC scan */
        WARNMS2(cinfo, JWRN_BOGUS_PROGRESSION, cindex, 0);
+      for (coefi = MIN(cinfo->Ss, 1); coefi <= MAX(cinfo->Se, 9); coefi++) {
+        if (cinfo->input_scan_number > 1)
+          prev_coef_bit_ptr[coefi] = coef_bit_ptr[coefi];
+        else
+          prev_coef_bit_ptr[coefi] = 0;
+      }
      for (coefi = cinfo->Ss; coefi <= cinfo->Se; coefi++) {
        int expected = (coef_bit_ptr[coefi] < 0) ? 0 : coef_bit_ptr[coefi];
        if (cinfo->Ah != expected)
@@ -727,6 +735,7 @@ bad:
  entropy->c = 0;
  entropy->a = 0;
  entropy->ct = -16;    /* force reading 2 initial bytes to fill C */
+  entropy->pub.insufficient_data = FALSE;

  /* Initialize restart counter */
  entropy->restarts_to_go = cinfo->restart_interval;
@@ -763,7 +772,7 @@ jinit_arith_decoder(j_decompress_ptr cinfo)
    int *coef_bit_ptr, ci;
    cinfo->coef_bits = (int (*)[DCTSIZE2])
      (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,
-                                  cinfo->num_components * DCTSIZE2 *
+                                  cinfo->num_components * 2 * DCTSIZE2 *
                                  sizeof(int));
    coef_bit_ptr = &cinfo->coef_bits[0][0];
    for (ci = 0; ci < cinfo->num_components; ci++)
@@ -5,7 +5,7 @@
 * Copyright (C) 1994-1997, Thomas G. Lane.
 * libjpeg-turbo Modifications:
 * Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
- * Copyright (C) 2010, 2015-2016, D. R. Commander.
+ * Copyright (C) 2010, 2015-2016, 2019-2020, D. R. Commander.
 * Copyright (C) 2015, 2020, Google, Inc.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
@@ -102,6 +102,8 @@ decompress_onepass(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
      /* Try to fetch an MCU.  Entropy decoder expects buffer to be zeroed. */
      jzero_far((void *)coef->MCU_buffer[0],
                (size_t)(cinfo->blocks_in_MCU * sizeof(JBLOCK)));
+      if (!cinfo->entropy->insufficient_data)
+        cinfo->master->last_good_iMCU_row = cinfo->input_iMCU_row;
      if (!(*cinfo->entropy->decode_mcu) (cinfo, coef->MCU_buffer)) {
        /* Suspension forced; update state counters and exit */
        coef->MCU_vert_offset = yoffset;
@@ -227,6 +229,8 @@ consume_data(j_decompress_ptr cinfo)
          }
        }
      }
+      if (!cinfo->entropy->insufficient_data)
+        cinfo->master->last_good_iMCU_row = cinfo->input_iMCU_row;
      /* Try to fetch the MCU. */
      if (!(*cinfo->entropy->decode_mcu) (cinfo, coef->MCU_buffer)) {
        /* Suspension forced; update state counters and exit */
@@ -326,19 +330,22 @@ decompress_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
 #ifdef BLOCK_SMOOTHING_SUPPORTED

 /*
- * This code applies interblock smoothing as described by section K.8
- * of the JPEG standard: the first 5 AC coefficients are estimated from
- * the DC values of a DCT block and its 8 neighboring blocks.
+ * This code applies interblock smoothing; the first 9 AC coefficients are
+ * estimated from the DC values of a DCT block and its 24 neighboring blocks.
 * We apply smoothing only for progressive JPEG decoding, and only if
 * the coefficients it can estimate are not yet known to full precision.
 */

-/* Natural-order array positions of the first 5 zigzag-order coefficients */
+/* Natural-order array positions of the first 9 zigzag-order coefficients */
 #define Q01_POS  1
 #define Q10_POS  8
 #define Q20_POS  16
 #define Q11_POS  9
 #define Q02_POS  2
+#define Q03_POS  3
+#define Q12_POS  10
+#define Q21_POS  17
+#define Q30_POS  24

 /*
 * Determine whether block smoothing is applicable and safe.
@@ -356,8 +363,8 @@ smoothing_ok(j_decompress_ptr cinfo)
  int ci, coefi;
  jpeg_component_info *compptr;
  JQUANT_TBL *qtable;
-  int *coef_bits;
-  int *coef_bits_latch;
+  int *coef_bits, *prev_coef_bits;
+  int *coef_bits_latch, *prev_coef_bits_latch;

  if (!cinfo->progressive_mode || cinfo->coef_bits == NULL)
    return FALSE;
@@ -366,34 +373,47 @@ smoothing_ok(j_decompress_ptr cinfo)
  if (coef->coef_bits_latch == NULL)
    coef->coef_bits_latch = (int *)
      (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,
-                                  cinfo->num_components *
+                                  cinfo->num_components * 2 *
                                  (SAVED_COEFS * sizeof(int)));
  coef_bits_latch = coef->coef_bits_latch;
+  prev_coef_bits_latch =
+    &coef->coef_bits_latch[cinfo->num_components * SAVED_COEFS];

  for (ci = 0, compptr = cinfo->comp_info; ci < cinfo->num_components;
       ci++, compptr++) {
    /* All components' quantization values must already be latched. */
    if ((qtable = compptr->quant_table) == NULL)
      return FALSE;
-    /* Verify DC & first 5 AC quantizers are nonzero to avoid zero-divide. */
+    /* Verify DC & first 9 AC quantizers are nonzero to avoid zero-divide. */
    if (qtable->quantval[0] == 0 ||
        qtable->quantval[Q01_POS] == 0 ||
        qtable->quantval[Q10_POS] == 0 ||
        qtable->quantval[Q20_POS] == 0 ||
        qtable->quantval[Q11_POS] == 0 ||
-        qtable->quantval[Q02_POS] == 0)
+        qtable->quantval[Q02_POS] == 0 ||
+        qtable->quantval[Q03_POS] == 0 ||
+        qtable->quantval[Q12_POS] == 0 ||
+        qtable->quantval[Q21_POS] == 0 ||
+        qtable->quantval[Q30_POS] == 0)
      return FALSE;
    /* DC values must be at least partly known for all components. */
    coef_bits = cinfo->coef_bits[ci];
+    prev_coef_bits = cinfo->coef_bits[ci + cinfo->num_components];
    if (coef_bits[0] < 0)
      return FALSE;
+    coef_bits_latch[0] = coef_bits[0];
    /* Block smoothing is helpful if some AC coefficients remain inaccurate. */
-    for (coefi = 1; coefi <= 5; coefi++) {
+    for (coefi = 1; coefi < SAVED_COEFS; coefi++) {
+      if (cinfo->input_scan_number > 1)
+        prev_coef_bits_latch[coefi] = prev_coef_bits[coefi];
+      else
+        prev_coef_bits_latch[coefi] = -1;
      coef_bits_latch[coefi] = coef_bits[coefi];
      if (coef_bits[coefi] != 0)
        smoothing_useful = TRUE;
    }
    coef_bits_latch += SAVED_COEFS;
+    prev_coef_bits_latch += SAVED_COEFS;
  }

  return smoothing_useful;
@@ -412,17 +432,20 @@ decompress_smooth_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
  JDIMENSION block_num, last_block_column;
  int ci, block_row, block_rows, access_rows;
  JBLOCKARRAY buffer;
-  JBLOCKROW buffer_ptr, prev_block_row, next_block_row;
+  JBLOCKROW buffer_ptr, prev_prev_block_row, prev_block_row;
+  JBLOCKROW next_block_row, next_next_block_row;
  JSAMPARRAY output_ptr;
  JDIMENSION output_col;
  jpeg_component_info *compptr;
  inverse_DCT_method_ptr inverse_DCT;
-  boolean first_row, last_row;
+  boolean change_dc;
  JCOEF *workspace;
  int *coef_bits;
  JQUANT_TBL *quanttbl;
-  JLONG Q00, Q01, Q02, Q10, Q11, Q20, num;
-  int DC1, DC2, DC3, DC4, DC5, DC6, DC7, DC8, DC9;
+  JLONG Q00, Q01, Q02, Q03 = 0, Q10, Q11, Q12 = 0, Q20, Q21 = 0, Q30 = 0, num;
+  int DC01, DC02, DC03, DC04, DC05, DC06, DC07, DC08, DC09, DC10, DC11, DC12,
+      DC13, DC14, DC15, DC16, DC17, DC18, DC19, DC20, DC21, DC22, DC23, DC24,
+      DC25;
  int Al, pred;

  /* Keep a local variable to avoid looking it up more than once */
@@ -434,10 +457,10 @@ decompress_smooth_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
    if (cinfo->input_scan_number == cinfo->output_scan_number) {
      /* If input is working on current scan, we ordinarily want it to
       * have completed the current row.  But if input scan is DC,
-       * we want it to keep one row ahead so that next block row's DC
+       * we want it to keep two rows ahead so that next two block rows' DC
       * values are up to date.
       */
-      JDIMENSION delta = (cinfo->Ss == 0) ? 1 : 0;
+      JDIMENSION delta = (cinfo->Ss == 0) ? 2 : 0;
      if (cinfo->input_iMCU_row > cinfo->output_iMCU_row + delta)
        break;
    }
@@ -452,34 +475,53 @@ decompress_smooth_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
    if (!compptr->component_needed)
      continue;
    /* Count non-dummy DCT block rows in this iMCU row. */
-    if (cinfo->output_iMCU_row < last_iMCU_row) {
+    if (cinfo->output_iMCU_row < last_iMCU_row - 1) {
+      block_rows = compptr->v_samp_factor;
+      access_rows = block_rows * 3; /* this and next two iMCU rows */
+    } else if (cinfo->output_iMCU_row < last_iMCU_row) {
      block_rows = compptr->v_samp_factor;
      access_rows = block_rows * 2; /* this and next iMCU row */
-      last_row = FALSE;
    } else {
      /* NB: can't use last_row_height here; it is input-side-dependent! */
      block_rows = (int)(compptr->height_in_blocks % compptr->v_samp_factor);
      if (block_rows == 0) block_rows = compptr->v_samp_factor;
      access_rows = block_rows; /* this iMCU row only */
-      last_row = TRUE;
    }
    /* Align the virtual buffer for this component. */
-    if (cinfo->output_iMCU_row > 0) {
-      access_rows += compptr->v_samp_factor; /* prior iMCU row too */
+    if (cinfo->output_iMCU_row > 1) {
+      access_rows += 2 * compptr->v_samp_factor; /* prior two iMCU rows too */
+      buffer = (*cinfo->mem->access_virt_barray)
+        ((j_common_ptr)cinfo, coef->whole_image[ci],
+         (cinfo->output_iMCU_row - 2) * compptr->v_samp_factor,
+         (JDIMENSION)access_rows, FALSE);
+      buffer += 2 * compptr->v_samp_factor; /* point to current iMCU row */
+    } else if (cinfo->output_iMCU_row > 0) {
      buffer = (*cinfo->mem->access_virt_barray)
        ((j_common_ptr)cinfo, coef->whole_image[ci],
         (cinfo->output_iMCU_row - 1) * compptr->v_samp_factor,
         (JDIMENSION)access_rows, FALSE);
      buffer += compptr->v_samp_factor; /* point to current iMCU row */
-      first_row = FALSE;
    } else {
      buffer = (*cinfo->mem->access_virt_barray)
        ((j_common_ptr)cinfo, coef->whole_image[ci],
         (JDIMENSION)0, (JDIMENSION)access_rows, FALSE);
-      first_row = TRUE;
    }
-    /* Fetch component-dependent info */
-    coef_bits = coef->coef_bits_latch + (ci * SAVED_COEFS);
+    /* Fetch component-dependent info.
+     * If the current scan is incomplete, then we use the component-dependent
+     * info from the previous scan.
+     */
+    if (cinfo->output_iMCU_row > cinfo->master->last_good_iMCU_row)
+      coef_bits =
+        coef->coef_bits_latch + ((ci + cinfo->num_components) * SAVED_COEFS);
+    else
+      coef_bits = coef->coef_bits_latch + (ci * SAVED_COEFS);
+
+    /* We only do DC interpolation if no AC coefficient data is available. */
+    change_dc =
+      coef_bits[1] == -1 && coef_bits[2] == -1 && coef_bits[3] == -1 &&
+      coef_bits[4] == -1 && coef_bits[5] == -1 && coef_bits[6] == -1 &&
+      coef_bits[7] == -1 && coef_bits[8] == -1 && coef_bits[9] == -1;
+
    quanttbl = compptr->quant_table;
    Q00 = quanttbl->quantval[0];
    Q01 = quanttbl->quantval[Q01_POS];
@@ -487,27 +529,51 @@ decompress_smooth_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
    Q20 = quanttbl->quantval[Q20_POS];
    Q11 = quanttbl->quantval[Q11_POS];
    Q02 = quanttbl->quantval[Q02_POS];
+    if (change_dc) {
+      Q03 = quanttbl->quantval[Q03_POS];
+      Q12 = quanttbl->quantval[Q12_POS];
+      Q21 = quanttbl->quantval[Q21_POS];
+      Q30 = quanttbl->quantval[Q30_POS];
+    }
    inverse_DCT = cinfo->idct->inverse_DCT[ci];
    output_ptr = output_buf[ci];
    /* Loop over all DCT blocks to be processed. */
    for (block_row = 0; block_row < block_rows; block_row++) {
      buffer_ptr = buffer[block_row] + cinfo->master->first_MCU_col[ci];
-      if (first_row && block_row == 0)
+
+      if (block_row > 0 || cinfo->output_iMCU_row > 0)
+        prev_block_row =
+          buffer[block_row - 1] + cinfo->master->first_MCU_col[ci];
+      else
        prev_block_row = buffer_ptr;
+
+      if (block_row > 1 || cinfo->output_iMCU_row > 1)
+        prev_prev_block_row =
+          buffer[block_row - 2] + cinfo->master->first_MCU_col[ci];
+      else
+        prev_prev_block_row = prev_block_row;
+
+      if (block_row < block_rows - 1 || cinfo->output_iMCU_row < last_iMCU_row)
+        next_block_row =
+          buffer[block_row + 1] + cinfo->master->first_MCU_col[ci];
      else
-        prev_block_row = buffer[block_row - 1] +
-                         cinfo->master->first_MCU_col[ci];
-      if (last_row && block_row == block_rows - 1)
        next_block_row = buffer_ptr;
+
+      if (block_row < block_rows - 2 ||
+          cinfo->output_iMCU_row < last_iMCU_row - 1)
+        next_next_block_row =
+          buffer[block_row + 2] + cinfo->master->first_MCU_col[ci];
      else
-        next_block_row = buffer[block_row + 1] +
-                         cinfo->master->first_MCU_col[ci];
+        next_next_block_row = next_block_row;
+
      /* We fetch the surrounding DC values using a sliding-register approach.
-       * Initialize all nine here so as to do the right thing on narrow pics.
+       * Initialize all 25 here so as to do the right thing on narrow pics.
       */
-      DC1 = DC2 = DC3 = (int)prev_block_row[0][0];
-      DC4 = DC5 = DC6 = (int)buffer_ptr[0][0];
-      DC7 = DC8 = DC9 = (int)next_block_row[0][0];
+      DC01 = DC02 = DC03 = DC04 = DC05 = (int)prev_prev_block_row[0][0];
+      DC06 = DC07 = DC08 = DC09 = DC10 = (int)prev_block_row[0][0];
+      DC11 = DC12 = DC13 = DC14 = DC15 = (int)buffer_ptr[0][0];
+      DC16 = DC17 = DC18 = DC19 = DC20 = (int)next_block_row[0][0];
+      DC21 = DC22 = DC23 = DC24 = DC25 = (int)next_next_block_row[0][0];
      output_col = 0;
      last_block_column = compptr->width_in_blocks - 1;
      for (block_num = cinfo->master->first_MCU_col[ci];
@@ -515,18 +581,39 @@ decompress_smooth_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
        /* Fetch current DCT block into workspace so we can modify it. */
        jcopy_block_row(buffer_ptr, (JBLOCKROW)workspace, (JDIMENSION)1);
        /* Update DC values */
-        if (block_num < last_block_column) {
-          DC3 = (int)prev_block_row[1][0];
-          DC6 = (int)buffer_ptr[1][0];
-          DC9 = (int)next_block_row[1][0];
+        if (block_num == cinfo->master->first_MCU_col[ci] &&
+            block_num < last_block_column) {
+          DC04 = (int)prev_prev_block_row[1][0];
+          DC09 = (int)prev_block_row[1][0];
+          DC14 = (int)buffer_ptr[1][0];
+          DC19 = (int)next_block_row[1][0];
+          DC24 = (int)next_next_block_row[1][0];
        }
-        /* Compute coefficient estimates per K.8.
-         * An estimate is applied only if coefficient is still zero,
-         * and is not known to be fully accurate.
+        if (block_num + 1 < last_block_column) {
+          DC05 = (int)prev_prev_block_row[2][0];
+          DC10 = (int)prev_block_row[2][0];
+          DC15 = (int)buffer_ptr[2][0];
+          DC20 = (int)next_block_row[2][0];
+          DC25 = (int)next_next_block_row[2][0];
+        }
+        /* If DC interpolation is enabled, compute coefficient estimates using
+         * a Gaussian-like kernel, keeping the averages of the DC values.
+         *
+         * If DC interpolation is disabled, compute coefficient estimates using
+         * an algorithm similar to the one described in Section K.8 of the JPEG
+         * standard, except applied to a 5x5 window rather than a 3x3 window.
+         *
+         * An estimate is applied only if the coefficient is still zero and is
+         * not known to be fully accurate.
         */
        /* AC01 */
        if ((Al = coef_bits[1]) != 0 && workspace[1] == 0) {
-          num = 36 * Q00 * (DC4 - DC6);
+          num = Q00 * (change_dc ?
+                (-DC01 - DC02 + DC04 + DC05 - 3 * DC06 + 13 * DC07 -
+                 13 * DC09 + 3 * DC10 - 3 * DC11 + 38 * DC12 - 38 * DC14 +
+                 3 * DC15 - 3 * DC16 + 13 * DC17 - 13 * DC19 + 3 * DC20 -
+                 DC21 - DC22 + DC24 + DC25) :
+                (-7 * DC11 + 50 * DC12 - 50 * DC14 + 7 * DC15));
          if (num >= 0) {
            pred = (int)(((Q01 << 7) + num) / (Q01 << 8));
            if (Al > 0 && pred >= (1 << Al))
@@ -541,7 +628,12 @@ decompress_smooth_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
        }
        /* AC10 */
        if ((Al = coef_bits[2]) != 0 && workspace[8] == 0) {
-          num = 36 * Q00 * (DC2 - DC8);
+          num = Q00 * (change_dc ?
+                (-DC01 - 3 * DC02 - 3 * DC03 - 3 * DC04 - DC05 - DC06 +
+                 13 * DC07 + 38 * DC08 + 13 * DC09 - DC10 + DC16 -
+                 13 * DC17 - 38 * DC18 - 13 * DC19 + DC20 + DC21 +
+                 3 * DC22 + 3 * DC23 + 3 * DC24 + DC25) :
+                (-7 * DC03 + 50 * DC08 - 50 * DC18 + 7 * DC23));
          if (num >= 0) {
            pred = (int)(((Q10 << 7) + num) / (Q10 << 8));
            if (Al > 0 && pred >= (1 << Al))
@@ -556,7 +648,10 @@ decompress_smooth_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
        }
        /* AC20 */
        if ((Al = coef_bits[3]) != 0 && workspace[16] == 0) {
-          num = 9 * Q00 * (DC2 + DC8 - 2 * DC5);
+          num = Q00 * (change_dc ?
+                (DC03 + 2 * DC07 + 7 * DC08 + 2 * DC09 - 5 * DC12 - 14 * DC13 -
+                 5 * DC14 + 2 * DC17 + 7 * DC18 + 2 * DC19 + DC23) :
+                (-DC03 + 13 * DC08 - 24 * DC13 + 13 * DC18 - DC23));
          if (num >= 0) {
            pred = (int)(((Q20 << 7) + num) / (Q20 << 8));
            if (Al > 0 && pred >= (1 << Al))
@@ -571,7 +666,11 @@ decompress_smooth_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
        }
        /* AC11 */
        if ((Al = coef_bits[4]) != 0 && workspace[9] == 0) {
-          num = 5 * Q00 * (DC1 - DC3 - DC7 + DC9);
+          num = Q00 * (change_dc ?
+                (-DC01 + DC05 + 9 * DC07 - 9 * DC09 - 9 * DC17 +
+                 9 * DC19 + DC21 - DC25) :
+                (DC10 + DC16 - 10 * DC17 + 10 * DC19 - DC02 - DC20 + DC22 -
+                 DC24 + DC04 - DC06 + 10 * DC07 - 10 * DC09));
          if (num >= 0) {
            pred = (int)(((Q11 << 7) + num) / (Q11 << 8));
            if (Al > 0 && pred >= (1 << Al))
@@ -586,7 +685,10 @@ decompress_smooth_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
        }
        /* AC02 */
        if ((Al = coef_bits[5]) != 0 && workspace[2] == 0) {
-          num = 9 * Q00 * (DC4 + DC6 - 2 * DC5);
+          num = Q00 * (change_dc ?
+                (2 * DC07 - 5 * DC08 + 2 * DC09 + DC11 + 7 * DC12 - 14 * DC13 +
+                 7 * DC14 + DC15 + 2 * DC17 - 5 * DC18 + 2 * DC19) :
+                (-DC11 + 13 * DC12 - 24 * DC13 + 13 * DC14 - DC15));
          if (num >= 0) {
            pred = (int)(((Q02 << 7) + num) / (Q02 << 8));
            if (Al > 0 && pred >= (1 << Al))
@@ -599,14 +701,96 @@ decompress_smooth_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf)
          }
          workspace[2] = (JCOEF)pred;
        }
+        if (change_dc) {
+          /* AC03 */
+          if ((Al = coef_bits[6]) != 0 && workspace[3] == 0) {
+            num = Q00 * (DC07 - DC09 + 2 * DC12 - 2 * DC14 + DC17 - DC19);
+            if (num >= 0) {
+              pred = (int)(((Q03 << 7) + num) / (Q03 << 8));
+              if (Al > 0 && pred >= (1 << Al))
+                pred = (1 << Al) - 1;
+            } else {
+              pred = (int)(((Q03 << 7) - num) / (Q03 << 8));
+              if (Al > 0 && pred >= (1 << Al))
+                pred = (1 << Al) - 1;
+              pred = -pred;
+            }
+            workspace[3] = (JCOEF)pred;
+          }
+          /* AC12 */
+          if ((Al = coef_bits[7]) != 0 && workspace[10] == 0) {
+            num = Q00 * (DC07 - 3 * DC08 + DC09 - DC17 + 3 * DC18 - DC19);
+            if (num >= 0) {
+              pred = (int)(((Q12 << 7) + num) / (Q12 << 8));
+              if (Al > 0 && pred >= (1 << Al))
+                pred = (1 << Al) - 1;
+            } else {
+              pred = (int)(((Q12 << 7) - num) / (Q12 << 8));
+              if (Al > 0 && pred >= (1 << Al))
+                pred = (1 << Al) - 1;
+              pred = -pred;
+            }
+            workspace[10] = (JCOEF)pred;
+          }
+          /* AC21 */
+          if ((Al = coef_bits[8]) != 0 && workspace[17] == 0) {
+            num = Q00 * (DC07 - DC09 - 3 * DC12 + 3 * DC14 + DC17 - DC19);
+            if (num >= 0) {
+              pred = (int)(((Q21 << 7) + num) / (Q21 << 8));
+              if (Al > 0 && pred >= (1 << Al))
+                pred = (1 << Al) - 1;
+            } else {
+              pred = (int)(((Q21 << 7) - num) / (Q21 << 8));
+              if (Al > 0 && pred >= (1 << Al))
+                pred = (1 << Al) - 1;
+              pred = -pred;
+            }
+            workspace[17] = (JCOEF)pred;
+          }
+          /* AC30 */
+          if ((Al = coef_bits[9]) != 0 && workspace[24] == 0) {
+            num = Q00 * (DC07 + 2 * DC08 + DC09 - DC17 - 2 * DC18 - DC19);
+            if (num >= 0) {
+              pred = (int)(((Q30 << 7) + num) / (Q30 << 8));
+              if (Al > 0 && pred >= (1 << Al))
+                pred = (1 << Al) - 1;
+            } else {
+              pred = (int)(((Q30 << 7) - num) / (Q30 << 8));
+              if (Al > 0 && pred >= (1 << Al))
+                pred = (1 << Al) - 1;
+              pred = -pred;
+            }
+            workspace[24] = (JCOEF)pred;
+          }
+          /* coef_bits[0] is non-negative.  Otherwise this function would not
+           * be called.
+           */
+          num = Q00 *
+                (-2 * DC01 - 6 * DC02 - 8 * DC03 - 6 * DC04 - 2 * DC05 -
+                 6 * DC06 + 6 * DC07 + 42 * DC08 + 6 * DC09 - 6 * DC10 -
+                 8 * DC11 + 42 * DC12 + 152 * DC13 + 42 * DC14 - 8 * DC15 -
+                 6 * DC16 + 6 * DC17 + 42 * DC18 + 6 * DC19 - 6 * DC20 -
+                 2 * DC21 - 6 * DC22 - 8 * DC23 - 6 * DC24 - 2 * DC25);
+          if (num >= 0) {
+            pred = (int)(((Q00 << 7) + num) / (Q00 << 8));
+          } else {
+            pred = (int)(((Q00 << 7) - num) / (Q00 << 8));
+            pred = -pred;
+          }
+          workspace[0] = (JCOEF)pred;
+        }  /* change_dc */
+
        /* OK, do the IDCT */
        (*inverse_DCT) (cinfo, compptr, (JCOEFPTR)workspace, output_ptr,
                        output_col);
        /* Advance for next column */
-        DC1 = DC2;  DC2 = DC3;
-        DC4 = DC5;  DC5 = DC6;
-        DC7 = DC8;  DC8 = DC9;
-        buffer_ptr++, prev_block_row++, next_block_row++;
+        DC01 = DC02;  DC02 = DC03;  DC03 = DC04;  DC04 = DC05;
+        DC06 = DC07;  DC07 = DC08;  DC08 = DC09;  DC09 = DC10;
+        DC11 = DC12;  DC12 = DC13;  DC13 = DC14;  DC14 = DC15;
+        DC16 = DC17;  DC17 = DC18;  DC18 = DC19;  DC19 = DC20;
+        DC21 = DC22;  DC22 = DC23;  DC23 = DC24;  DC24 = DC25;
+        buffer_ptr++, prev_block_row++, next_block_row++,
+          prev_prev_block_row++, next_next_block_row++;
        output_col += compptr->_DCT_scaled_size;
      }
      output_ptr += compptr->_DCT_scaled_size;
@@ -655,7 +839,7 @@ jinit_d_coef_controller(j_decompress_ptr cinfo, boolean need_full_buffer)
 #ifdef BLOCK_SMOOTHING_SUPPORTED
      /* If block smoothing could be used, need a bigger window */
      if (cinfo->progressive_mode)
-        access_rows *= 3;
+        access_rows *= 5;
 #endif
      coef->whole_image[ci] = (*cinfo->mem->request_virt_barray)
        ((j_common_ptr)cinfo, JPOOL_IMAGE, TRUE,
@@ -5,6 +5,7 @@
 * Copyright (C) 1994-1997, Thomas G. Lane.
 * libjpeg-turbo Modifications:
 * Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
+ * Copyright (C) 2020, Google, Inc.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 */
@@ -51,7 +52,7 @@ typedef struct {
 #ifdef BLOCK_SMOOTHING_SUPPORTED
  /* When doing block smoothing, we latch coefficient Al values here */
  int *coef_bits_latch;
-#define SAVED_COEFS  6          /* we save coef_bits[0..5] */
+#define SAVED_COEFS  10         /* we save coef_bits[0..9] */
 #endif
 } my_coef_controller;

@@ -45,9 +45,9 @@ ycc_rgb565_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    outptr = *output_buf++;

    if (PACK_NEED_ALIGNMENT(outptr)) {
-      y  = GETJSAMPLE(*inptr0++);
-      cb = GETJSAMPLE(*inptr1++);
-      cr = GETJSAMPLE(*inptr2++);
+      y  = *inptr0++;
+      cb = *inptr1++;
+      cr = *inptr2++;
      r = range_limit[y + Crrtab[cr]];
      g = range_limit[y + ((int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr],
                                            SCALEBITS))];
@@ -58,18 +58,18 @@ ycc_rgb565_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
      num_cols--;
    }
    for (col = 0; col < (num_cols >> 1); col++) {
-      y  = GETJSAMPLE(*inptr0++);
-      cb = GETJSAMPLE(*inptr1++);
-      cr = GETJSAMPLE(*inptr2++);
+      y  = *inptr0++;
+      cb = *inptr1++;
+      cr = *inptr2++;
      r = range_limit[y + Crrtab[cr]];
      g = range_limit[y + ((int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr],
                                            SCALEBITS))];
      b = range_limit[y + Cbbtab[cb]];
      rgb = PACK_SHORT_565(r, g, b);

-      y  = GETJSAMPLE(*inptr0++);
-      cb = GETJSAMPLE(*inptr1++);
-      cr = GETJSAMPLE(*inptr2++);
+      y  = *inptr0++;
+      cb = *inptr1++;
+      cr = *inptr2++;
      r = range_limit[y + Crrtab[cr]];
      g = range_limit[y + ((int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr],
                                            SCALEBITS))];
@@ -80,9 +80,9 @@ ycc_rgb565_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
      outptr += 4;
    }
    if (num_cols & 1) {
-      y  = GETJSAMPLE(*inptr0);
-      cb = GETJSAMPLE(*inptr1);
-      cr = GETJSAMPLE(*inptr2);
+      y  = *inptr0;
+      cb = *inptr1;
+      cr = *inptr2;
      r = range_limit[y + Crrtab[cr]];
      g = range_limit[y + ((int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr],
                                            SCALEBITS))];
@@ -125,9 +125,9 @@ ycc_rgb565D_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    input_row++;
    outptr = *output_buf++;
    if (PACK_NEED_ALIGNMENT(outptr)) {
-      y  = GETJSAMPLE(*inptr0++);
-      cb = GETJSAMPLE(*inptr1++);
-      cr = GETJSAMPLE(*inptr2++);
+      y  = *inptr0++;
+      cb = *inptr1++;
+      cr = *inptr2++;
      r = range_limit[DITHER_565_R(y + Crrtab[cr], d0)];
      g = range_limit[DITHER_565_G(y +
                                   ((int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr],
@@ -139,9 +139,9 @@ ycc_rgb565D_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
      num_cols--;
    }
    for (col = 0; col < (num_cols >> 1); col++) {
-      y  = GETJSAMPLE(*inptr0++);
-      cb = GETJSAMPLE(*inptr1++);
-      cr = GETJSAMPLE(*inptr2++);
+      y  = *inptr0++;
+      cb = *inptr1++;
+      cr = *inptr2++;
      r = range_limit[DITHER_565_R(y + Crrtab[cr], d0)];
      g = range_limit[DITHER_565_G(y +
                                   ((int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr],
@@ -150,9 +150,9 @@ ycc_rgb565D_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
      d0 = DITHER_ROTATE(d0);
      rgb = PACK_SHORT_565(r, g, b);

-      y  = GETJSAMPLE(*inptr0++);
-      cb = GETJSAMPLE(*inptr1++);
-      cr = GETJSAMPLE(*inptr2++);
+      y  = *inptr0++;
+      cb = *inptr1++;
+      cr = *inptr2++;
      r = range_limit[DITHER_565_R(y + Crrtab[cr], d0)];
      g = range_limit[DITHER_565_G(y +
                                   ((int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr],
@@ -165,9 +165,9 @@ ycc_rgb565D_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
      outptr += 4;
    }
    if (num_cols & 1) {
-      y  = GETJSAMPLE(*inptr0);
-      cb = GETJSAMPLE(*inptr1);
-      cr = GETJSAMPLE(*inptr2);
+      y  = *inptr0;
+      cb = *inptr1;
+      cr = *inptr2;
      r = range_limit[DITHER_565_R(y + Crrtab[cr], d0)];
      g = range_limit[DITHER_565_G(y +
                                   ((int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr],
@@ -202,32 +202,32 @@ rgb_rgb565_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    input_row++;
    outptr = *output_buf++;
    if (PACK_NEED_ALIGNMENT(outptr)) {
-      r = GETJSAMPLE(*inptr0++);
-      g = GETJSAMPLE(*inptr1++);
-      b = GETJSAMPLE(*inptr2++);
+      r = *inptr0++;
+      g = *inptr1++;
+      b = *inptr2++;
      rgb = PACK_SHORT_565(r, g, b);
      *(INT16 *)outptr = (INT16)rgb;
      outptr += 2;
      num_cols--;
    }
    for (col = 0; col < (num_cols >> 1); col++) {
-      r = GETJSAMPLE(*inptr0++);
-      g = GETJSAMPLE(*inptr1++);
-      b = GETJSAMPLE(*inptr2++);
+      r = *inptr0++;
+      g = *inptr1++;
+      b = *inptr2++;
      rgb = PACK_SHORT_565(r, g, b);

-      r = GETJSAMPLE(*inptr0++);
-      g = GETJSAMPLE(*inptr1++);
-      b = GETJSAMPLE(*inptr2++);
+      r = *inptr0++;
+      g = *inptr1++;
+      b = *inptr2++;
      rgb = PACK_TWO_PIXELS(rgb, PACK_SHORT_565(r, g, b));

      WRITE_TWO_ALIGNED_PIXELS(outptr, rgb);
      outptr += 4;
    }
    if (num_cols & 1) {
-      r = GETJSAMPLE(*inptr0);
-      g = GETJSAMPLE(*inptr1);
-      b = GETJSAMPLE(*inptr2);
+      r = *inptr0;
+      g = *inptr1;
+      b = *inptr2;
      rgb = PACK_SHORT_565(r, g, b);
      *(INT16 *)outptr = (INT16)rgb;
    }
@@ -259,24 +259,24 @@ rgb_rgb565D_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    input_row++;
    outptr = *output_buf++;
    if (PACK_NEED_ALIGNMENT(outptr)) {
-      r = range_limit[DITHER_565_R(GETJSAMPLE(*inptr0++), d0)];
-      g = range_limit[DITHER_565_G(GETJSAMPLE(*inptr1++), d0)];
-      b = range_limit[DITHER_565_B(GETJSAMPLE(*inptr2++), d0)];
+      r = range_limit[DITHER_565_R(*inptr0++, d0)];
+      g = range_limit[DITHER_565_G(*inptr1++, d0)];
+      b = range_limit[DITHER_565_B(*inptr2++, d0)];
      rgb = PACK_SHORT_565(r, g, b);
      *(INT16 *)outptr = (INT16)rgb;
      outptr += 2;
      num_cols--;
    }
    for (col = 0; col < (num_cols >> 1); col++) {
-      r = range_limit[DITHER_565_R(GETJSAMPLE(*inptr0++), d0)];
-      g = range_limit[DITHER_565_G(GETJSAMPLE(*inptr1++), d0)];
-      b = range_limit[DITHER_565_B(GETJSAMPLE(*inptr2++), d0)];
+      r = range_limit[DITHER_565_R(*inptr0++, d0)];
+      g = range_limit[DITHER_565_G(*inptr1++, d0)];
+      b = range_limit[DITHER_565_B(*inptr2++, d0)];
      d0 = DITHER_ROTATE(d0);
      rgb = PACK_SHORT_565(r, g, b);

-      r = range_limit[DITHER_565_R(GETJSAMPLE(*inptr0++), d0)];
-      g = range_limit[DITHER_565_G(GETJSAMPLE(*inptr1++), d0)];
-      b = range_limit[DITHER_565_B(GETJSAMPLE(*inptr2++), d0)];
+      r = range_limit[DITHER_565_R(*inptr0++, d0)];
+      g = range_limit[DITHER_565_G(*inptr1++, d0)];
+      b = range_limit[DITHER_565_B(*inptr2++, d0)];
      d0 = DITHER_ROTATE(d0);
      rgb = PACK_TWO_PIXELS(rgb, PACK_SHORT_565(r, g, b));

@@ -284,9 +284,9 @@ rgb_rgb565D_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
      outptr += 4;
    }
    if (num_cols & 1) {
-      r = range_limit[DITHER_565_R(GETJSAMPLE(*inptr0), d0)];
-      g = range_limit[DITHER_565_G(GETJSAMPLE(*inptr1), d0)];
-      b = range_limit[DITHER_565_B(GETJSAMPLE(*inptr2), d0)];
+      r = range_limit[DITHER_565_R(*inptr0, d0)];
+      g = range_limit[DITHER_565_G(*inptr1, d0)];
+      b = range_limit[DITHER_565_B(*inptr2, d0)];
      rgb = PACK_SHORT_565(r, g, b);
      *(INT16 *)outptr = (INT16)rgb;
    }
@@ -53,9 +53,9 @@ ycc_rgb_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    input_row++;
    outptr = *output_buf++;
    for (col = 0; col < num_cols; col++) {
-      y  = GETJSAMPLE(inptr0[col]);
-      cb = GETJSAMPLE(inptr1[col]);
-      cr = GETJSAMPLE(inptr2[col]);
+      y  = inptr0[col];
+      cb = inptr1[col];
+      cr = inptr2[col];
      /* Range-limiting is essential due to noise introduced by DCT losses. */
      outptr[RGB_RED] =   range_limit[y + Crrtab[cr]];
      outptr[RGB_GREEN] = range_limit[y +
@@ -93,7 +93,6 @@ gray_rgb_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    inptr = input_buf[0][input_row++];
    outptr = *output_buf++;
    for (col = 0; col < num_cols; col++) {
-      /* We can dispense with GETJSAMPLE() here */
      outptr[RGB_RED] = outptr[RGB_GREEN] = outptr[RGB_BLUE] = inptr[col];
      /* Set unused byte to 0xFF so it can be interpreted as an opaque */
      /* alpha channel value */
@@ -128,7 +127,6 @@ rgb_rgb_convert_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    input_row++;
    outptr = *output_buf++;
    for (col = 0; col < num_cols; col++) {
-      /* We can dispense with GETJSAMPLE() here */
      outptr[RGB_RED] = inptr0[col];
      outptr[RGB_GREEN] = inptr1[col];
      outptr[RGB_BLUE] = inptr2[col];
@@ -341,9 +341,9 @@ rgb_gray_convert(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    input_row++;
    outptr = *output_buf++;
    for (col = 0; col < num_cols; col++) {
-      r = GETJSAMPLE(inptr0[col]);
-      g = GETJSAMPLE(inptr1[col]);
-      b = GETJSAMPLE(inptr2[col]);
+      r = inptr0[col];
+      g = inptr1[col];
+      b = inptr2[col];
      /* Y */
      outptr[col] = (JSAMPLE)((ctab[r + R_Y_OFF] + ctab[g + G_Y_OFF] +
                               ctab[b + B_Y_OFF]) >> SCALEBITS);
@@ -550,9 +550,9 @@ ycck_cmyk_convert(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    input_row++;
    outptr = *output_buf++;
    for (col = 0; col < num_cols; col++) {
-      y  = GETJSAMPLE(inptr0[col]);
-      cb = GETJSAMPLE(inptr1[col]);
-      cr = GETJSAMPLE(inptr2[col]);
+      y  = inptr0[col];
+      cb = inptr1[col];
+      cr = inptr2[col];
      /* Range-limiting is essential due to noise introduced by DCT losses. */
      outptr[0] = range_limit[MAXJSAMPLE - (y + Crrtab[cr])];   /* red */
      outptr[1] = range_limit[MAXJSAMPLE - (y +                 /* green */
@@ -560,7 +560,7 @@ ycck_cmyk_convert(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
                                                 SCALEBITS)))];
      outptr[2] = range_limit[MAXJSAMPLE - (y + Cbbtab[cb])];   /* blue */
      /* K passes through unchanged */
-      outptr[3] = inptr3[col];  /* don't need GETJSAMPLE here */
+      outptr[3] = inptr3[col];
      outptr += 4;
    }
  }
@@ -5,6 +5,7 @@
 * Copyright (C) 1991-1997, Thomas G. Lane.
 * libjpeg-turbo Modifications:
 * Copyright (C) 2009-2011, 2016, 2018-2019, D. R. Commander.
+ * Copyright (C) 2018, Matthias Räncker.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 *
@@ -39,24 +40,6 @@ typedef struct {
  int last_dc_val[MAX_COMPS_IN_SCAN]; /* last DC coef for each component */
 } savable_state;

-/* This macro is to work around compilers with missing or broken
- * structure assignment.  You'll need to fix this code if you have
- * such a compiler and you change MAX_COMPS_IN_SCAN.
- */
-
-#ifndef NO_STRUCT_ASSIGN
-#define ASSIGN_STATE(dest, src)  ((dest) = (src))
-#else
-#if MAX_COMPS_IN_SCAN == 4
-#define ASSIGN_STATE(dest, src) \
-  ((dest).last_dc_val[0] = (src).last_dc_val[0], \
-   (dest).last_dc_val[1] = (src).last_dc_val[1], \
-   (dest).last_dc_val[2] = (src).last_dc_val[2], \
-   (dest).last_dc_val[3] = (src).last_dc_val[3])
-#endif
-#endif
-
-
 typedef struct {
  struct jpeg_entropy_decoder pub; /* public fields */

@@ -325,7 +308,7 @@ jpeg_fill_bit_buffer(bitread_working_state *state,
        bytes_in_buffer = cinfo->src->bytes_in_buffer;
      }
      bytes_in_buffer--;
-      c = GETJOCTET(*next_input_byte++);
+      c = *next_input_byte++;

      /* If it's 0xFF, check and discard stuffed zero byte */
      if (c == 0xFF) {
@@ -342,7 +325,7 @@ jpeg_fill_bit_buffer(bitread_working_state *state,
            bytes_in_buffer = cinfo->src->bytes_in_buffer;
          }
          bytes_in_buffer--;
-          c = GETJOCTET(*next_input_byte++);
+          c = *next_input_byte++;
        } while (c == 0xFF);

        if (c == 0) {
@@ -405,8 +388,8 @@ no_more_bytes:

 #define GET_BYTE { \
  register int c0, c1; \
-  c0 = GETJOCTET(*buffer++); \
-  c1 = GETJOCTET(*buffer); \
+  c0 = *buffer++; \
+  c1 = *buffer; \
  /* Pre-execute most common case */ \
  get_buffer = (get_buffer << 8) | c0; \
  bits_left += 8; \
@@ -423,7 +406,7 @@ no_more_bytes:
  } \
 }

-#if SIZEOF_SIZE_T == 8 || defined(_WIN64)
+#if SIZEOF_SIZE_T == 8 || defined(_WIN64) || (defined(__x86_64__) && defined(__ILP32__))

 /* Pre-fetch 48 bytes, because the holding register is 64-bit */
 #define FILL_BIT_BUFFER_FAST \
@@ -557,6 +540,12 @@ process_restart(j_decompress_ptr cinfo)
 }


+#if defined(__has_feature)
+#if __has_feature(undefined_behavior_sanitizer)
+__attribute__((no_sanitize("signed-integer-overflow"),
+               no_sanitize("unsigned-integer-overflow")))
+#endif
+#endif
 LOCAL(boolean)
 decode_mcu_slow(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)
 {
@@ -568,7 +557,7 @@ decode_mcu_slow(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)

  /* Load up working state */
  BITREAD_LOAD_STATE(cinfo, entropy->bitstate);
-  ASSIGN_STATE(state, entropy->saved);
+  state = entropy->saved;

  for (blkn = 0; blkn < cinfo->blocks_in_MCU; blkn++) {
    JBLOCKROW block = MCU_data ? MCU_data[blkn] : NULL;
@@ -589,11 +578,15 @@ decode_mcu_slow(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)
    if (entropy->dc_needed[blkn]) {
      /* Convert DC difference to actual value, update last_dc_val */
      int ci = cinfo->MCU_membership[blkn];
-      /* This is really just
-       *   s += state.last_dc_val[ci];
-       * It is written this way in order to shut up UBSan.
+      /* Certain malformed JPEG images produce repeated DC coefficient
+       * differences of 2047 or -2047, which causes state.last_dc_val[ci] to
+       * grow until it overflows or underflows a 32-bit signed integer.  This
+       * behavior is, to the best of our understanding, innocuous, and it is
+       * unclear how to work around it without potentially affecting
+       * performance.  Thus, we (hopefully temporarily) suppress UBSan integer
+       * overflow errors for this function and decode_mcu_fast().
       */
-      s = (int)((unsigned int)s + (unsigned int)state.last_dc_val[ci]);
+      s += state.last_dc_val[ci];
      state.last_dc_val[ci] = s;
      if (block) {
        /* Output the DC coefficient (assumes jpeg_natural_order[0] = 0) */
@@ -653,11 +646,17 @@ decode_mcu_slow(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)

  /* Completed MCU, so update state */
  BITREAD_SAVE_STATE(cinfo, entropy->bitstate);
-  ASSIGN_STATE(entropy->saved, state);
+  entropy->saved = state;
  return TRUE;
 }


+#if defined(__has_feature)
+#if __has_feature(undefined_behavior_sanitizer)
+__attribute__((no_sanitize("signed-integer-overflow"),
+               no_sanitize("unsigned-integer-overflow")))
+#endif
+#endif
 LOCAL(boolean)
 decode_mcu_fast(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)
 {
@@ -671,7 +670,7 @@ decode_mcu_fast(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)
  /* Load up working state */
  BITREAD_LOAD_STATE(cinfo, entropy->bitstate);
  buffer = (JOCTET *)br_state.next_input_byte;
-  ASSIGN_STATE(state, entropy->saved);
+  state = entropy->saved;

  for (blkn = 0; blkn < cinfo->blocks_in_MCU; blkn++) {
    JBLOCKROW block = MCU_data ? MCU_data[blkn] : NULL;
@@ -688,7 +687,10 @@ decode_mcu_fast(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)

    if (entropy->dc_needed[blkn]) {
      int ci = cinfo->MCU_membership[blkn];
-      s = (int)((unsigned int)s + (unsigned int)state.last_dc_val[ci]);
+      /* Refer to the comment in decode_mcu_slow() regarding the supression of
+       * a UBSan integer overflow error in this line of code.
+       */
+      s += state.last_dc_val[ci];
      state.last_dc_val[ci] = s;
      if (block)
        (*block)[0] = (JCOEF)s;
@@ -740,7 +742,7 @@ decode_mcu_fast(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)
  br_state.bytes_in_buffer -= (buffer - br_state.next_input_byte);
  br_state.next_input_byte = buffer;
  BITREAD_SAVE_STATE(cinfo, entropy->bitstate);
-  ASSIGN_STATE(entropy->saved, state);
+  entropy->saved = state;
  return TRUE;
 }

@@ -795,7 +797,8 @@ use_slow:
  }

  /* Account for restart interval (no-op if not using restarts) */
-  entropy->restarts_to_go--;
+  if (cinfo->restart_interval)
+    entropy->restarts_to_go--;

  return TRUE;
 }
@@ -4,7 +4,8 @@
 * This file was part of the Independent JPEG Group's software:
 * Copyright (C) 1991-1997, Thomas G. Lane.
 * libjpeg-turbo Modifications:
- * Copyright (C) 2010-2011, 2015-2016, D. R. Commander.
+ * Copyright (C) 2010-2011, 2015-2016, 2021, D. R. Commander.
+ * Copyright (C) 2018, Matthias Räncker.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 *
@@ -78,6 +79,11 @@ EXTERN(void) jpeg_make_d_derived_tbl(j_decompress_ptr cinfo, boolean isDC,
 typedef size_t bit_buf_type;            /* type of bit-extraction buffer */
 #define BIT_BUF_SIZE  64                /* size of buffer in bits */

+#elif defined(__x86_64__) && defined(__ILP32__)
+
+typedef unsigned long long bit_buf_type; /* type of bit-extraction buffer */
+#define BIT_BUF_SIZE  64                 /* size of buffer in bits */
+
 #else

 typedef unsigned long bit_buf_type;     /* type of bit-extraction buffer */
@@ -228,7 +234,10 @@ slowlabel: \
      s |= GET_BITS(1); \
      nb++; \
    } \
-    s = htbl->pub->huffval[(int)(s + htbl->valoffset[nb]) & 0xFF]; \
+    if (nb > 16) \
+      s = 0; \
+    else \
+      s = htbl->pub->huffval[(int)(s + htbl->valoffset[nb]) & 0xFF]; \
  }

 /* Out-of-line case for Huffman code fetching */
@@ -38,18 +38,18 @@ marker_is_icc(jpeg_saved_marker_ptr marker)
    marker->marker == ICC_MARKER &&
    marker->data_length >= ICC_OVERHEAD_LEN &&
    /* verify the identifying string */
-    GETJOCTET(marker->data[0]) == 0x49 &&
-    GETJOCTET(marker->data[1]) == 0x43 &&
-    GETJOCTET(marker->data[2]) == 0x43 &&
-    GETJOCTET(marker->data[3]) == 0x5F &&
-    GETJOCTET(marker->data[4]) == 0x50 &&
-    GETJOCTET(marker->data[5]) == 0x52 &&
-    GETJOCTET(marker->data[6]) == 0x4F &&
-    GETJOCTET(marker->data[7]) == 0x46 &&
-    GETJOCTET(marker->data[8]) == 0x49 &&
-    GETJOCTET(marker->data[9]) == 0x4C &&
-    GETJOCTET(marker->data[10]) == 0x45 &&
-    GETJOCTET(marker->data[11]) == 0x0;
+    marker->data[0] == 0x49 &&
+    marker->data[1] == 0x43 &&
+    marker->data[2] == 0x43 &&
+    marker->data[3] == 0x5F &&
+    marker->data[4] == 0x50 &&
+    marker->data[5] == 0x52 &&
+    marker->data[6] == 0x4F &&
+    marker->data[7] == 0x46 &&
+    marker->data[8] == 0x49 &&
+    marker->data[9] == 0x4C &&
+    marker->data[10] == 0x45 &&
+    marker->data[11] == 0x0;
 }


@@ -102,12 +102,12 @@ jpeg_read_icc_profile(j_decompress_ptr cinfo, JOCTET **icc_data_ptr,
  for (marker = cinfo->marker_list; marker != NULL; marker = marker->next) {
    if (marker_is_icc(marker)) {
      if (num_markers == 0)
-        num_markers = GETJOCTET(marker->data[13]);
-      else if (num_markers != GETJOCTET(marker->data[13])) {
+        num_markers = marker->data[13];
+      else if (num_markers != marker->data[13]) {
        WARNMS(cinfo, JWRN_BOGUS_ICC);  /* inconsistent num_markers fields */
        return FALSE;
      }
-      seq_no = GETJOCTET(marker->data[12]);
+      seq_no = marker->data[12];
      if (seq_no <= 0 || seq_no > num_markers) {
        WARNMS(cinfo, JWRN_BOGUS_ICC);  /* bogus sequence number */
        return FALSE;
@@ -154,7 +154,7 @@ jpeg_read_icc_profile(j_decompress_ptr cinfo, JOCTET **icc_data_ptr,
      JOCTET FAR *src_ptr;
      JOCTET *dst_ptr;
      unsigned int length;
-      seq_no = GETJOCTET(marker->data[12]);
+      seq_no = marker->data[12];
      dst_ptr = icc_data + data_offset[seq_no];
      src_ptr = marker->data + ICC_OVERHEAD_LEN;
      length = data_length[seq_no];
@@ -18,6 +18,7 @@

 #include "jinclude.h"
 #include "jdmainct.h"
+#include "jconfigint.h"


 /*
@@ -360,7 +361,7 @@ process_data_context_main(j_decompress_ptr cinfo, JSAMPARRAY output_buf,
    main_ptr->context_state = CTX_PREPARE_FOR_IMCU;
    if (*out_row_ctr >= out_rows_avail)
      return;                   /* Postprocessor exactly filled output buf */
-    /*FALLTHROUGH*/
+    FALLTHROUGH                 /*FALLTHROUGH*/
  case CTX_PREPARE_FOR_IMCU:
    /* Prepare to process first M-1 row groups of this iMCU row */
    main_ptr->rowgroup_ctr = 0;
@@ -371,7 +372,7 @@ process_data_context_main(j_decompress_ptr cinfo, JSAMPARRAY output_buf,
    if (main_ptr->iMCU_row_ctr == cinfo->total_iMCU_rows)
      set_bottom_pointers(cinfo);
    main_ptr->context_state = CTX_PROCESS_IMCU;
-    /*FALLTHROUGH*/
+    FALLTHROUGH                 /*FALLTHROUGH*/
  case CTX_PROCESS_IMCU:
    /* Call postprocessor using previously set pointers */
    (*cinfo->post->post_process_data) (cinfo,
@@ -151,7 +151,7 @@ typedef my_marker_reader *my_marker_ptr;
 #define INPUT_BYTE(cinfo, V, action) \
  MAKESTMT( MAKE_BYTE_AVAIL(cinfo, action); \
            bytes_in_buffer--; \
-            V = GETJOCTET(*next_input_byte++); )
+            V = *next_input_byte++; )

 /* As above, but read two bytes interpreted as an unsigned 16-bit integer.
 * V should be declared unsigned int or perhaps JLONG.
@@ -159,10 +159,10 @@ typedef my_marker_reader *my_marker_ptr;
 #define INPUT_2BYTES(cinfo, V, action) \
  MAKESTMT( MAKE_BYTE_AVAIL(cinfo, action); \
            bytes_in_buffer--; \
-            V = ((unsigned int)GETJOCTET(*next_input_byte++)) << 8; \
+            V = ((unsigned int)(*next_input_byte++)) << 8; \
            MAKE_BYTE_AVAIL(cinfo, action); \
            bytes_in_buffer--; \
-            V += GETJOCTET(*next_input_byte++); )
+            V += *next_input_byte++; )


 /*
@@ -608,18 +608,18 @@ examine_app0(j_decompress_ptr cinfo, JOCTET *data, unsigned int datalen,
  JLONG totallen = (JLONG)datalen + remaining;

  if (datalen >= APP0_DATA_LEN &&
-      GETJOCTET(data[0]) == 0x4A &&
-      GETJOCTET(data[1]) == 0x46 &&
-      GETJOCTET(data[2]) == 0x49 &&
-      GETJOCTET(data[3]) == 0x46 &&
-      GETJOCTET(data[4]) == 0) {
+      data[0] == 0x4A &&
+      data[1] == 0x46 &&
+      data[2] == 0x49 &&
+      data[3] == 0x46 &&
+      data[4] == 0) {
    /* Found JFIF APP0 marker: save info */
    cinfo->saw_JFIF_marker = TRUE;
-    cinfo->JFIF_major_version = GETJOCTET(data[5]);
-    cinfo->JFIF_minor_version = GETJOCTET(data[6]);
-    cinfo->density_unit = GETJOCTET(data[7]);
-    cinfo->X_density = (GETJOCTET(data[8]) << 8) + GETJOCTET(data[9]);
-    cinfo->Y_density = (GETJOCTET(data[10]) << 8) + GETJOCTET(data[11]);
+    cinfo->JFIF_major_version = data[5];
+    cinfo->JFIF_minor_version = data[6];
+    cinfo->density_unit = data[7];
+    cinfo->X_density = (data[8] << 8) + data[9];
+    cinfo->Y_density = (data[10] << 8) + data[11];
    /* Check version.
     * Major version must be 1, anything else signals an incompatible change.
     * (We used to treat this as an error, but now it's a nonfatal warning,
@@ -634,24 +634,22 @@ examine_app0(j_decompress_ptr cinfo, JOCTET *data, unsigned int datalen,
             cinfo->JFIF_major_version, cinfo->JFIF_minor_version,
             cinfo->X_density, cinfo->Y_density, cinfo->density_unit);
    /* Validate thumbnail dimensions and issue appropriate messages */
-    if (GETJOCTET(data[12]) | GETJOCTET(data[13]))
-      TRACEMS2(cinfo, 1, JTRC_JFIF_THUMBNAIL,
-               GETJOCTET(data[12]), GETJOCTET(data[13]));
+    if (data[12] | data[13])
+      TRACEMS2(cinfo, 1, JTRC_JFIF_THUMBNAIL, data[12], data[13]);
    totallen -= APP0_DATA_LEN;
-    if (totallen !=
-        ((JLONG)GETJOCTET(data[12]) * (JLONG)GETJOCTET(data[13]) * (JLONG)3))
+    if (totallen != ((JLONG)data[12] * (JLONG)data[13] * (JLONG)3))
      TRACEMS1(cinfo, 1, JTRC_JFIF_BADTHUMBNAILSIZE, (int)totallen);
  } else if (datalen >= 6 &&
-             GETJOCTET(data[0]) == 0x4A &&
-             GETJOCTET(data[1]) == 0x46 &&
-             GETJOCTET(data[2]) == 0x58 &&
-             GETJOCTET(data[3]) == 0x58 &&
-             GETJOCTET(data[4]) == 0) {
+             data[0] == 0x4A &&
+             data[1] == 0x46 &&
+             data[2] == 0x58 &&
+             data[3] == 0x58 &&
+             data[4] == 0) {
    /* Found JFIF "JFXX" extension APP0 marker */
    /* The library doesn't actually do anything with these,
     * but we try to produce a helpful trace message.
     */
-    switch (GETJOCTET(data[5])) {
+    switch (data[5]) {
    case 0x10:
      TRACEMS1(cinfo, 1, JTRC_THUMB_JPEG, (int)totallen);
      break;
@@ -662,8 +660,7 @@ examine_app0(j_decompress_ptr cinfo, JOCTET *data, unsigned int datalen,
      TRACEMS1(cinfo, 1, JTRC_THUMB_RGB, (int)totallen);
      break;
    default:
-      TRACEMS2(cinfo, 1, JTRC_JFIF_EXTENSION,
-               GETJOCTET(data[5]), (int)totallen);
+      TRACEMS2(cinfo, 1, JTRC_JFIF_EXTENSION, data[5], (int)totallen);
      break;
    }
  } else {
@@ -684,16 +681,16 @@ examine_app14(j_decompress_ptr cinfo, JOCTET *data, unsigned int datalen,
  unsigned int version, flags0, flags1, transform;

  if (datalen >= APP14_DATA_LEN &&
-      GETJOCTET(data[0]) == 0x41 &&
-      GETJOCTET(data[1]) == 0x64 &&
-      GETJOCTET(data[2]) == 0x6F &&
-      GETJOCTET(data[3]) == 0x62 &&
-      GETJOCTET(data[4]) == 0x65) {
+      data[0] == 0x41 &&
+      data[1] == 0x64 &&
+      data[2] == 0x6F &&
+      data[3] == 0x62 &&
+      data[4] == 0x65) {
    /* Found Adobe APP14 marker */
-    version = (GETJOCTET(data[5]) << 8) + GETJOCTET(data[6]);
-    flags0 = (GETJOCTET(data[7]) << 8) + GETJOCTET(data[8]);
-    flags1 = (GETJOCTET(data[9]) << 8) + GETJOCTET(data[10]);
-    transform = GETJOCTET(data[11]);
+    version = (data[5] << 8) + data[6];
+    flags0 = (data[7] << 8) + data[8];
+    flags1 = (data[9] << 8) + data[10];
+    transform = data[11];
    TRACEMS4(cinfo, 1, JTRC_ADOBE, version, flags0, flags1, transform);
    cinfo->saw_Adobe_marker = TRUE;
    cinfo->Adobe_transform = (UINT8)transform;
@@ -5,7 +5,7 @@
 * Copyright (C) 1991-1997, Thomas G. Lane.
 * Modified 2002-2009 by Guido Vollbeding.
 * libjpeg-turbo Modifications:
- * Copyright (C) 2009-2011, 2016, D. R. Commander.
+ * Copyright (C) 2009-2011, 2016, 2019, D. R. Commander.
 * Copyright (C) 2013, Linaro Limited.
 * Copyright (C) 2015, Google, Inc.
 * For conditions of distribution and use, see the accompanying README.ijg
@@ -22,7 +22,6 @@
 #include "jpeglib.h"
 #include "jpegcomp.h"
 #include "jdmaster.h"
-#include "jsimd.h"


 /*
@@ -70,17 +69,6 @@ use_merged_upsample(j_decompress_ptr cinfo)
      cinfo->comp_info[1]._DCT_scaled_size != cinfo->_min_DCT_scaled_size ||
      cinfo->comp_info[2]._DCT_scaled_size != cinfo->_min_DCT_scaled_size)
    return FALSE;
-#ifdef WITH_SIMD
-  /* If YCbCr-to-RGB color conversion is SIMD-accelerated but merged upsampling
-     isn't, then disabling merged upsampling is likely to be faster when
-     decompressing YCbCr JPEG images. */
-  if (!jsimd_can_h2v2_merged_upsample() && !jsimd_can_h2v1_merged_upsample() &&
-      jsimd_can_ycc_rgb() && cinfo->jpeg_color_space == JCS_YCbCr &&
-      (cinfo->out_color_space == JCS_RGB ||
-       (cinfo->out_color_space >= JCS_EXT_RGB &&
-        cinfo->out_color_space <= JCS_EXT_ARGB)))
-    return FALSE;
-#endif
  /* ??? also need to test for upsample-time rescaling, when & if supported */
  return TRUE;                  /* by golly, it'll work... */
 #else
@@ -580,6 +568,7 @@ master_selection(j_decompress_ptr cinfo)
   */
  cinfo->master->first_iMCU_col = 0;
  cinfo->master->last_iMCU_col = cinfo->MCUs_per_row - 1;
+  cinfo->master->last_good_iMCU_row = 0;

 #ifdef D_MULTISCAN_FILES_SUPPORTED
  /* If jpeg_start_decompress will read the whole file, initialize
@@ -43,20 +43,20 @@ h2v1_merged_upsample_565_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
  /* Loop for each pair of output pixels */
  for (col = cinfo->output_width >> 1; col > 0; col--) {
    /* Do the chroma part of the calculation */
-    cb = GETJSAMPLE(*inptr1++);
-    cr = GETJSAMPLE(*inptr2++);
+    cb = *inptr1++;
+    cr = *inptr2++;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];

    /* Fetch 2 Y values and emit 2 pixels */
-    y  = GETJSAMPLE(*inptr0++);
+    y  = *inptr0++;
    r = range_limit[y + cred];
    g = range_limit[y + cgreen];
    b = range_limit[y + cblue];
    rgb = PACK_SHORT_565(r, g, b);

-    y  = GETJSAMPLE(*inptr0++);
+    y  = *inptr0++;
    r = range_limit[y + cred];
    g = range_limit[y + cgreen];
    b = range_limit[y + cblue];
@@ -68,12 +68,12 @@ h2v1_merged_upsample_565_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,

  /* If image width is odd, do the last output column separately */
  if (cinfo->output_width & 1) {
-    cb = GETJSAMPLE(*inptr1);
-    cr = GETJSAMPLE(*inptr2);
+    cb = *inptr1;
+    cr = *inptr2;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];
-    y  = GETJSAMPLE(*inptr0);
+    y  = *inptr0;
    r = range_limit[y + cred];
    g = range_limit[y + cgreen];
    b = range_limit[y + cblue];
@@ -115,21 +115,21 @@ h2v1_merged_upsample_565D_internal(j_decompress_ptr cinfo,
  /* Loop for each pair of output pixels */
  for (col = cinfo->output_width >> 1; col > 0; col--) {
    /* Do the chroma part of the calculation */
-    cb = GETJSAMPLE(*inptr1++);
-    cr = GETJSAMPLE(*inptr2++);
+    cb = *inptr1++;
+    cr = *inptr2++;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];

    /* Fetch 2 Y values and emit 2 pixels */
-    y  = GETJSAMPLE(*inptr0++);
+    y  = *inptr0++;
    r = range_limit[DITHER_565_R(y + cred, d0)];
    g = range_limit[DITHER_565_G(y + cgreen, d0)];
    b = range_limit[DITHER_565_B(y + cblue, d0)];
    d0 = DITHER_ROTATE(d0);
    rgb = PACK_SHORT_565(r, g, b);

-    y  = GETJSAMPLE(*inptr0++);
+    y  = *inptr0++;
    r = range_limit[DITHER_565_R(y + cred, d0)];
    g = range_limit[DITHER_565_G(y + cgreen, d0)];
    b = range_limit[DITHER_565_B(y + cblue, d0)];
@@ -142,12 +142,12 @@ h2v1_merged_upsample_565D_internal(j_decompress_ptr cinfo,

  /* If image width is odd, do the last output column separately */
  if (cinfo->output_width & 1) {
-    cb = GETJSAMPLE(*inptr1);
-    cr = GETJSAMPLE(*inptr2);
+    cb = *inptr1;
+    cr = *inptr2;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];
-    y  = GETJSAMPLE(*inptr0);
+    y  = *inptr0;
    r = range_limit[DITHER_565_R(y + cred, d0)];
    g = range_limit[DITHER_565_G(y + cgreen, d0)];
    b = range_limit[DITHER_565_B(y + cblue, d0)];
@@ -189,20 +189,20 @@ h2v2_merged_upsample_565_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
  /* Loop for each group of output pixels */
  for (col = cinfo->output_width >> 1; col > 0; col--) {
    /* Do the chroma part of the calculation */
-    cb = GETJSAMPLE(*inptr1++);
-    cr = GETJSAMPLE(*inptr2++);
+    cb = *inptr1++;
+    cr = *inptr2++;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];

    /* Fetch 4 Y values and emit 4 pixels */
-    y  = GETJSAMPLE(*inptr00++);
+    y  = *inptr00++;
    r = range_limit[y + cred];
    g = range_limit[y + cgreen];
    b = range_limit[y + cblue];
    rgb = PACK_SHORT_565(r, g, b);

-    y  = GETJSAMPLE(*inptr00++);
+    y  = *inptr00++;
    r = range_limit[y + cred];
    g = range_limit[y + cgreen];
    b = range_limit[y + cblue];
@@ -211,13 +211,13 @@ h2v2_merged_upsample_565_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    WRITE_TWO_PIXELS(outptr0, rgb);
    outptr0 += 4;

-    y  = GETJSAMPLE(*inptr01++);
+    y  = *inptr01++;
    r = range_limit[y + cred];
    g = range_limit[y + cgreen];
    b = range_limit[y + cblue];
    rgb = PACK_SHORT_565(r, g, b);

-    y  = GETJSAMPLE(*inptr01++);
+    y  = *inptr01++;
    r = range_limit[y + cred];
    g = range_limit[y + cgreen];
    b = range_limit[y + cblue];
@@ -229,20 +229,20 @@ h2v2_merged_upsample_565_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,

  /* If image width is odd, do the last output column separately */
  if (cinfo->output_width & 1) {
-    cb = GETJSAMPLE(*inptr1);
-    cr = GETJSAMPLE(*inptr2);
+    cb = *inptr1;
+    cr = *inptr2;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];

-    y  = GETJSAMPLE(*inptr00);
+    y  = *inptr00;
    r = range_limit[y + cred];
    g = range_limit[y + cgreen];
    b = range_limit[y + cblue];
    rgb = PACK_SHORT_565(r, g, b);
    *(INT16 *)outptr0 = (INT16)rgb;

-    y  = GETJSAMPLE(*inptr01);
+    y  = *inptr01;
    r = range_limit[y + cred];
    g = range_limit[y + cgreen];
    b = range_limit[y + cblue];
@@ -287,21 +287,21 @@ h2v2_merged_upsample_565D_internal(j_decompress_ptr cinfo,
  /* Loop for each group of output pixels */
  for (col = cinfo->output_width >> 1; col > 0; col--) {
    /* Do the chroma part of the calculation */
-    cb = GETJSAMPLE(*inptr1++);
-    cr = GETJSAMPLE(*inptr2++);
+    cb = *inptr1++;
+    cr = *inptr2++;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];

    /* Fetch 4 Y values and emit 4 pixels */
-    y  = GETJSAMPLE(*inptr00++);
+    y  = *inptr00++;
    r = range_limit[DITHER_565_R(y + cred, d0)];
    g = range_limit[DITHER_565_G(y + cgreen, d0)];
    b = range_limit[DITHER_565_B(y + cblue, d0)];
    d0 = DITHER_ROTATE(d0);
    rgb = PACK_SHORT_565(r, g, b);

-    y  = GETJSAMPLE(*inptr00++);
+    y  = *inptr00++;
    r = range_limit[DITHER_565_R(y + cred, d0)];
    g = range_limit[DITHER_565_G(y + cgreen, d0)];
    b = range_limit[DITHER_565_B(y + cblue, d0)];
@@ -311,14 +311,14 @@ h2v2_merged_upsample_565D_internal(j_decompress_ptr cinfo,
    WRITE_TWO_PIXELS(outptr0, rgb);
    outptr0 += 4;

-    y  = GETJSAMPLE(*inptr01++);
+    y  = *inptr01++;
    r = range_limit[DITHER_565_R(y + cred, d1)];
    g = range_limit[DITHER_565_G(y + cgreen, d1)];
    b = range_limit[DITHER_565_B(y + cblue, d1)];
    d1 = DITHER_ROTATE(d1);
    rgb = PACK_SHORT_565(r, g, b);

-    y  = GETJSAMPLE(*inptr01++);
+    y  = *inptr01++;
    r = range_limit[DITHER_565_R(y + cred, d1)];
    g = range_limit[DITHER_565_G(y + cgreen, d1)];
    b = range_limit[DITHER_565_B(y + cblue, d1)];
@@ -331,20 +331,20 @@ h2v2_merged_upsample_565D_internal(j_decompress_ptr cinfo,

  /* If image width is odd, do the last output column separately */
  if (cinfo->output_width & 1) {
-    cb = GETJSAMPLE(*inptr1);
-    cr = GETJSAMPLE(*inptr2);
+    cb = *inptr1;
+    cr = *inptr2;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];

-    y  = GETJSAMPLE(*inptr00);
+    y  = *inptr00;
    r = range_limit[DITHER_565_R(y + cred, d0)];
    g = range_limit[DITHER_565_G(y + cgreen, d0)];
    b = range_limit[DITHER_565_B(y + cblue, d0)];
    rgb = PACK_SHORT_565(r, g, b);
    *(INT16 *)outptr0 = (INT16)rgb;

-    y  = GETJSAMPLE(*inptr01);
+    y  = *inptr01;
    r = range_limit[DITHER_565_R(y + cred, d1)];
    g = range_limit[DITHER_565_G(y + cgreen, d1)];
    b = range_limit[DITHER_565_B(y + cblue, d1)];
@@ -46,13 +46,13 @@ h2v1_merged_upsample_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
  /* Loop for each pair of output pixels */
  for (col = cinfo->output_width >> 1; col > 0; col--) {
    /* Do the chroma part of the calculation */
-    cb = GETJSAMPLE(*inptr1++);
-    cr = GETJSAMPLE(*inptr2++);
+    cb = *inptr1++;
+    cr = *inptr2++;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];
    /* Fetch 2 Y values and emit 2 pixels */
-    y  = GETJSAMPLE(*inptr0++);
+    y  = *inptr0++;
    outptr[RGB_RED] =   range_limit[y + cred];
    outptr[RGB_GREEN] = range_limit[y + cgreen];
    outptr[RGB_BLUE] =  range_limit[y + cblue];
@@ -60,7 +60,7 @@ h2v1_merged_upsample_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    outptr[RGB_ALPHA] = 0xFF;
 #endif
    outptr += RGB_PIXELSIZE;
-    y  = GETJSAMPLE(*inptr0++);
+    y  = *inptr0++;
    outptr[RGB_RED] =   range_limit[y + cred];
    outptr[RGB_GREEN] = range_limit[y + cgreen];
    outptr[RGB_BLUE] =  range_limit[y + cblue];
@@ -71,12 +71,12 @@ h2v1_merged_upsample_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
  }
  /* If image width is odd, do the last output column separately */
  if (cinfo->output_width & 1) {
-    cb = GETJSAMPLE(*inptr1);
-    cr = GETJSAMPLE(*inptr2);
+    cb = *inptr1;
+    cr = *inptr2;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];
-    y  = GETJSAMPLE(*inptr0);
+    y  = *inptr0;
    outptr[RGB_RED] =   range_limit[y + cred];
    outptr[RGB_GREEN] = range_limit[y + cgreen];
    outptr[RGB_BLUE] =  range_limit[y + cblue];
@@ -120,13 +120,13 @@ h2v2_merged_upsample_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
  /* Loop for each group of output pixels */
  for (col = cinfo->output_width >> 1; col > 0; col--) {
    /* Do the chroma part of the calculation */
-    cb = GETJSAMPLE(*inptr1++);
-    cr = GETJSAMPLE(*inptr2++);
+    cb = *inptr1++;
+    cr = *inptr2++;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];
    /* Fetch 4 Y values and emit 4 pixels */
-    y  = GETJSAMPLE(*inptr00++);
+    y  = *inptr00++;
    outptr0[RGB_RED] =   range_limit[y + cred];
    outptr0[RGB_GREEN] = range_limit[y + cgreen];
    outptr0[RGB_BLUE] =  range_limit[y + cblue];
@@ -134,7 +134,7 @@ h2v2_merged_upsample_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    outptr0[RGB_ALPHA] = 0xFF;
 #endif
    outptr0 += RGB_PIXELSIZE;
-    y  = GETJSAMPLE(*inptr00++);
+    y  = *inptr00++;
    outptr0[RGB_RED] =   range_limit[y + cred];
    outptr0[RGB_GREEN] = range_limit[y + cgreen];
    outptr0[RGB_BLUE] =  range_limit[y + cblue];
@@ -142,7 +142,7 @@ h2v2_merged_upsample_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    outptr0[RGB_ALPHA] = 0xFF;
 #endif
    outptr0 += RGB_PIXELSIZE;
-    y  = GETJSAMPLE(*inptr01++);
+    y  = *inptr01++;
    outptr1[RGB_RED] =   range_limit[y + cred];
    outptr1[RGB_GREEN] = range_limit[y + cgreen];
    outptr1[RGB_BLUE] =  range_limit[y + cblue];
@@ -150,7 +150,7 @@ h2v2_merged_upsample_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
    outptr1[RGB_ALPHA] = 0xFF;
 #endif
    outptr1 += RGB_PIXELSIZE;
-    y  = GETJSAMPLE(*inptr01++);
+    y  = *inptr01++;
    outptr1[RGB_RED] =   range_limit[y + cred];
    outptr1[RGB_GREEN] = range_limit[y + cgreen];
    outptr1[RGB_BLUE] =  range_limit[y + cblue];
@@ -161,19 +161,19 @@ h2v2_merged_upsample_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
  }
  /* If image width is odd, do the last output column separately */
  if (cinfo->output_width & 1) {
-    cb = GETJSAMPLE(*inptr1);
-    cr = GETJSAMPLE(*inptr2);
+    cb = *inptr1;
+    cr = *inptr2;
    cred = Crrtab[cr];
    cgreen = (int)RIGHT_SHIFT(Cbgtab[cb] + Crgtab[cr], SCALEBITS);
    cblue = Cbbtab[cb];
-    y  = GETJSAMPLE(*inptr00);
+    y  = *inptr00;
    outptr0[RGB_RED] =   range_limit[y + cred];
    outptr0[RGB_GREEN] = range_limit[y + cgreen];
    outptr0[RGB_BLUE] =  range_limit[y + cblue];
 #ifdef RGB_ALPHA
    outptr0[RGB_ALPHA] = 0xFF;
 #endif
-    y  = GETJSAMPLE(*inptr01);
+    y  = *inptr01;
    outptr1[RGB_RED] =   range_limit[y + cred];
    outptr1[RGB_GREEN] = range_limit[y + cgreen];
    outptr1[RGB_BLUE] =  range_limit[y + cblue];
@@ -4,7 +4,7 @@
 * This file was part of the Independent JPEG Group's software:
 * Copyright (C) 1995-1997, Thomas G. Lane.
 * libjpeg-turbo Modifications:
- * Copyright (C) 2015-2016, 2018, D. R. Commander.
+ * Copyright (C) 2015-2016, 2018-2021, D. R. Commander.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 *
@@ -41,25 +41,6 @@ typedef struct {
  int last_dc_val[MAX_COMPS_IN_SCAN];   /* last DC coef for each component */
 } savable_state;

-/* This macro is to work around compilers with missing or broken
- * structure assignment.  You'll need to fix this code if you have
- * such a compiler and you change MAX_COMPS_IN_SCAN.
- */
-
-#ifndef NO_STRUCT_ASSIGN
-#define ASSIGN_STATE(dest, src)  ((dest) = (src))
-#else
-#if MAX_COMPS_IN_SCAN == 4
-#define ASSIGN_STATE(dest, src) \
-  ((dest).EOBRUN = (src).EOBRUN, \
-   (dest).last_dc_val[0] = (src).last_dc_val[0], \
-   (dest).last_dc_val[1] = (src).last_dc_val[1], \
-   (dest).last_dc_val[2] = (src).last_dc_val[2], \
-   (dest).last_dc_val[3] = (src).last_dc_val[3])
-#endif
-#endif
-
-
 typedef struct {
  struct jpeg_entropy_decoder pub; /* public fields */

@@ -102,7 +83,7 @@ start_pass_phuff_decoder(j_decompress_ptr cinfo)
  boolean is_DC_band, bad;
  int ci, coefi, tbl;
  d_derived_tbl **pdtbl;
-  int *coef_bit_ptr;
+  int *coef_bit_ptr, *prev_coef_bit_ptr;
  jpeg_component_info *compptr;

  is_DC_band = (cinfo->Ss == 0);
@@ -143,8 +124,15 @@ start_pass_phuff_decoder(j_decompress_ptr cinfo)
  for (ci = 0; ci < cinfo->comps_in_scan; ci++) {
    int cindex = cinfo->cur_comp_info[ci]->component_index;
    coef_bit_ptr = &cinfo->coef_bits[cindex][0];
+    prev_coef_bit_ptr = &cinfo->coef_bits[cindex + cinfo->num_components][0];
    if (!is_DC_band && coef_bit_ptr[0] < 0) /* AC without prior DC scan */
      WARNMS2(cinfo, JWRN_BOGUS_PROGRESSION, cindex, 0);
+    for (coefi = MIN(cinfo->Ss, 1); coefi <= MAX(cinfo->Se, 9); coefi++) {
+      if (cinfo->input_scan_number > 1)
+        prev_coef_bit_ptr[coefi] = coef_bit_ptr[coefi];
+      else
+        prev_coef_bit_ptr[coefi] = 0;
+    }
    for (coefi = cinfo->Ss; coefi <= cinfo->Se; coefi++) {
      int expected = (coef_bit_ptr[coefi] < 0) ? 0 : coef_bit_ptr[coefi];
      if (cinfo->Ah != expected)
@@ -323,7 +311,7 @@ decode_mcu_DC_first(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)

    /* Load up working state */
    BITREAD_LOAD_STATE(cinfo, entropy->bitstate);
-    ASSIGN_STATE(state, entropy->saved);
+    state = entropy->saved;

    /* Outer loop handles each block in the MCU */

@@ -356,11 +344,12 @@ decode_mcu_DC_first(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)

    /* Completed MCU, so update state */
    BITREAD_SAVE_STATE(cinfo, entropy->bitstate);
-    ASSIGN_STATE(entropy->saved, state);
+    entropy->saved = state;
  }

  /* Account for restart interval (no-op if not using restarts) */
-  entropy->restarts_to_go--;
+  if (cinfo->restart_interval)
+    entropy->restarts_to_go--;

  return TRUE;
 }
@@ -444,7 +433,8 @@ decode_mcu_AC_first(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)
  }

  /* Account for restart interval (no-op if not using restarts) */
-  entropy->restarts_to_go--;
+  if (cinfo->restart_interval)
+    entropy->restarts_to_go--;

  return TRUE;
 }
@@ -495,7 +485,8 @@ decode_mcu_DC_refine(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)
  BITREAD_SAVE_STATE(cinfo, entropy->bitstate);

  /* Account for restart interval (no-op if not using restarts) */
-  entropy->restarts_to_go--;
+  if (cinfo->restart_interval)
+    entropy->restarts_to_go--;

  return TRUE;
 }
@@ -638,7 +629,8 @@ decode_mcu_AC_refine(j_decompress_ptr cinfo, JBLOCKROW *MCU_data)
  }

  /* Account for restart interval (no-op if not using restarts) */
-  entropy->restarts_to_go--;
+  if (cinfo->restart_interval)
+    entropy->restarts_to_go--;

  return TRUE;

@@ -676,7 +668,7 @@ jinit_phuff_decoder(j_decompress_ptr cinfo)
  /* Create progression status table */
  cinfo->coef_bits = (int (*)[DCTSIZE2])
    (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,
-                                cinfo->num_components * DCTSIZE2 *
+                                cinfo->num_components * 2 * DCTSIZE2 *
                                sizeof(int));
  coef_bit_ptr = &cinfo->coef_bits[0][0];
  for (ci = 0; ci < cinfo->num_components; ci++)
@@ -8,7 +8,7 @@
 * Copyright (C) 2010, 2015-2016, D. R. Commander.
 * Copyright (C) 2014, MIPS Technologies, Inc., California.
 * Copyright (C) 2015, Google, Inc.
- * Copyright (C) 2019, Arm Limited.
+ * Copyright (C) 2019-2020, Arm Limited.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 *
@@ -177,7 +177,7 @@ int_upsample(j_decompress_ptr cinfo, jpeg_component_info *compptr,
    outptr = output_data[outrow];
    outend = outptr + cinfo->output_width;
    while (outptr < outend) {
-      invalue = *inptr++;       /* don't need GETJSAMPLE() here */
+      invalue = *inptr++;
      for (h = h_expand; h > 0; h--) {
        *outptr++ = invalue;
      }
@@ -213,7 +213,7 @@ h2v1_upsample(j_decompress_ptr cinfo, jpeg_component_info *compptr,
    outptr = output_data[inrow];
    outend = outptr + cinfo->output_width;
    while (outptr < outend) {
-      invalue = *inptr++;       /* don't need GETJSAMPLE() here */
+      invalue = *inptr++;
      *outptr++ = invalue;
      *outptr++ = invalue;
    }
@@ -242,7 +242,7 @@ h2v2_upsample(j_decompress_ptr cinfo, jpeg_component_info *compptr,
    outptr = output_data[outrow];
    outend = outptr + cinfo->output_width;
    while (outptr < outend) {
-      invalue = *inptr++;       /* don't need GETJSAMPLE() here */
+      invalue = *inptr++;
      *outptr++ = invalue;
      *outptr++ = invalue;
    }
@@ -283,20 +283,20 @@ h2v1_fancy_upsample(j_decompress_ptr cinfo, jpeg_component_info *compptr,
    inptr = input_data[inrow];
    outptr = output_data[inrow];
    /* Special case for first column */
-    invalue = GETJSAMPLE(*inptr++);
+    invalue = *inptr++;
    *outptr++ = (JSAMPLE)invalue;
-    *outptr++ = (JSAMPLE)((invalue * 3 + GETJSAMPLE(*inptr) + 2) >> 2);
+    *outptr++ = (JSAMPLE)((invalue * 3 + inptr[0] + 2) >> 2);

    for (colctr = compptr->downsampled_width - 2; colctr > 0; colctr--) {
      /* General case: 3/4 * nearer pixel + 1/4 * further pixel */
-      invalue = GETJSAMPLE(*inptr++) * 3;
-      *outptr++ = (JSAMPLE)((invalue + GETJSAMPLE(inptr[-2]) + 1) >> 2);
-      *outptr++ = (JSAMPLE)((invalue + GETJSAMPLE(*inptr) + 2) >> 2);
+      invalue = (*inptr++) * 3;
+      *outptr++ = (JSAMPLE)((invalue + inptr[-2] + 1) >> 2);
+      *outptr++ = (JSAMPLE)((invalue + inptr[0] + 2) >> 2);
    }

    /* Special case for last column */
-    invalue = GETJSAMPLE(*inptr);
-    *outptr++ = (JSAMPLE)((invalue * 3 + GETJSAMPLE(inptr[-1]) + 1) >> 2);
+    invalue = *inptr;
+    *outptr++ = (JSAMPLE)((invalue * 3 + inptr[-1] + 1) >> 2);
    *outptr++ = (JSAMPLE)invalue;
  }
 }
@@ -338,7 +338,7 @@ h1v2_fancy_upsample(j_decompress_ptr cinfo, jpeg_component_info *compptr,
      outptr = output_data[outrow++];

      for (colctr = 0; colctr < compptr->downsampled_width; colctr++) {
-        thiscolsum = GETJSAMPLE(*inptr0++) * 3 + GETJSAMPLE(*inptr1++);
+        thiscolsum = (*inptr0++) * 3 + (*inptr1++);
        *outptr++ = (JSAMPLE)((thiscolsum + bias) >> 2);
      }
    }
@@ -381,8 +381,8 @@ h2v2_fancy_upsample(j_decompress_ptr cinfo, jpeg_component_info *compptr,
      outptr = output_data[outrow++];

      /* Special case for first column */
-      thiscolsum = GETJSAMPLE(*inptr0++) * 3 + GETJSAMPLE(*inptr1++);
-      nextcolsum = GETJSAMPLE(*inptr0++) * 3 + GETJSAMPLE(*inptr1++);
+      thiscolsum = (*inptr0++) * 3 + (*inptr1++);
+      nextcolsum = (*inptr0++) * 3 + (*inptr1++);
      *outptr++ = (JSAMPLE)((thiscolsum * 4 + 8) >> 4);
      *outptr++ = (JSAMPLE)((thiscolsum * 3 + nextcolsum + 7) >> 4);
      lastcolsum = thiscolsum;  thiscolsum = nextcolsum;
@@ -390,7 +390,7 @@ h2v2_fancy_upsample(j_decompress_ptr cinfo, jpeg_component_info *compptr,
      for (colctr = compptr->downsampled_width - 2; colctr > 0; colctr--) {
        /* General case: 3/4 * nearer pixel + 1/4 * further pixel in each */
        /* dimension, thus 9/16, 3/16, 3/16, 1/16 overall */
-        nextcolsum = GETJSAMPLE(*inptr0++) * 3 + GETJSAMPLE(*inptr1++);
+        nextcolsum = (*inptr0++) * 3 + (*inptr1++);
        *outptr++ = (JSAMPLE)((thiscolsum * 3 + lastcolsum + 8) >> 4);
        *outptr++ = (JSAMPLE)((thiscolsum * 3 + nextcolsum + 7) >> 4);
        lastcolsum = thiscolsum;  thiscolsum = nextcolsum;
@@ -477,7 +477,13 @@ jinit_upsampler(j_decompress_ptr cinfo)
    } else if (h_in_group == h_out_group &&
               v_in_group * 2 == v_out_group && do_fancy) {
      /* Non-fancy upsampling is handled by the generic method */
-      upsample->methods[ci] = h1v2_fancy_upsample;
+#if defined(__arm__) || defined(__aarch64__) || \
+    defined(_M_ARM) || defined(_M_ARM64)
+      if (jsimd_can_h1v2_fancy_upsample())
+        upsample->methods[ci] = jsimd_h1v2_fancy_upsample;
+      else
+#endif
+        upsample->methods[ci] = h1v2_fancy_upsample;
      upsample->pub.need_context_rows = TRUE;
    } else if (h_in_group * 2 == h_out_group &&
               v_in_group * 2 == v_out_group) {
@@ -207,6 +207,10 @@ JMESSAGE(JWRN_ARITH_BAD_CODE, "Corrupt JPEG data: bad arithmetic code")
 #endif
 #endif
 JMESSAGE(JWRN_BOGUS_ICC, "Corrupt JPEG data: bad ICC marker")
+#if JPEG_LIB_VERSION < 70
+JMESSAGE(JERR_BAD_DROP_SAMPLING,
+         "Component index %d: mismatching sampling ratio %d:%d, %d:%d, %c")
+#endif

 #ifdef JMAKE_ENUM_LIST

@@ -252,6 +256,15 @@ JMESSAGE(JWRN_BOGUS_ICC, "Corrupt JPEG data: bad ICC marker")
   (cinfo)->err->msg_parm.i[2] = (p3), \
   (cinfo)->err->msg_parm.i[3] = (p4), \
   (*(cinfo)->err->error_exit) ((j_common_ptr)(cinfo)))
+#define ERREXIT6(cinfo, code, p1, p2, p3, p4, p5, p6) \
+  ((cinfo)->err->msg_code = (code), \
+   (cinfo)->err->msg_parm.i[0] = (p1), \
+   (cinfo)->err->msg_parm.i[1] = (p2), \
+   (cinfo)->err->msg_parm.i[2] = (p3), \
+   (cinfo)->err->msg_parm.i[3] = (p4), \
+   (cinfo)->err->msg_parm.i[4] = (p5), \
+   (cinfo)->err->msg_parm.i[5] = (p6), \
+   (*(cinfo)->err->error_exit) ((j_common_ptr)(cinfo)))
 #define ERREXITS(cinfo, code, str) \
  ((cinfo)->err->msg_code = (code), \
   strncpy((cinfo)->err->msg_parm.s, (str), JMSG_STR_PARM_MAX), \
@@ -3,7 +3,7 @@
 *
 * This file was part of the Independent JPEG Group's software:
 * Copyright (C) 1991-1998, Thomas G. Lane.
- * Modification developed 2002-2009 by Guido Vollbeding.
+ * Modification developed 2002-2018 by Guido Vollbeding.
 * libjpeg-turbo Modifications:
 * Copyright (C) 2015, 2020, D. R. Commander.
 * For conditions of distribution and use, see the accompanying README.ijg
@@ -417,7 +417,7 @@ jpeg_idct_islow(j_decompress_ptr cinfo, jpeg_component_info *compptr,

 /*
 * Perform dequantization and inverse DCT on one block of coefficients,
- * producing a 7x7 output block.
+ * producing a reduced-size 7x7 output block.
 *
 * Optimized algorithm with 12 multiplications in the 1-D kernel.
 * cK represents sqrt(2) * cos(K*pi/14).
@@ -1258,7 +1258,7 @@ jpeg_idct_10x10(j_decompress_ptr cinfo, jpeg_component_info *compptr,

 /*
 * Perform dequantization and inverse DCT on one block of coefficients,
- * producing a 11x11 output block.
+ * producing an 11x11 output block.
 *
 * Optimized algorithm with 24 multiplications in the 1-D kernel.
 * cK represents sqrt(2) * cos(K*pi/22).
@@ -2398,7 +2398,7 @@ jpeg_idct_16x16(j_decompress_ptr cinfo, jpeg_component_info *compptr,
    tmp0 = DEQUANTIZE(inptr[DCTSIZE * 0], quantptr[DCTSIZE * 0]);
    tmp0 = LEFT_SHIFT(tmp0, CONST_BITS);
    /* Add fudge factor here for final descale. */
-    tmp0 += 1 << (CONST_BITS - PASS1_BITS - 1);
+    tmp0 += ONE << (CONST_BITS - PASS1_BITS - 1);

    z1 = DEQUANTIZE(inptr[DCTSIZE * 4], quantptr[DCTSIZE * 4]);
    tmp1 = MULTIPLY(z1, FIX(1.306562965));      /* c4[16] = c2[8] */
@@ -4,7 +4,7 @@
 * This file was part of the Independent JPEG Group's software:
 * Copyright (C) 1991-1997, Thomas G. Lane.
 * libjpeg-turbo Modifications:
- * Copyright (C) 2016, D. R. Commander.
+ * Copyright (C) 2016, 2021, D. R. Commander.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 *
@@ -1032,7 +1032,7 @@ free_pool(j_common_ptr cinfo, int pool_id)
    large_pool_ptr next_lhdr_ptr = lhdr_ptr->next;
    space_freed = lhdr_ptr->bytes_used +
                  lhdr_ptr->bytes_left +
-                  sizeof(large_pool_hdr);
+                  sizeof(large_pool_hdr) + ALIGN_SIZE - 1;
    jpeg_free_large(cinfo, (void *)lhdr_ptr, space_freed);
    mem->total_space_allocated -= space_freed;
    lhdr_ptr = next_lhdr_ptr;
@@ -1045,7 +1045,7 @@ free_pool(j_common_ptr cinfo, int pool_id)
  while (shdr_ptr != NULL) {
    small_pool_ptr next_shdr_ptr = shdr_ptr->next;
    space_freed = shdr_ptr->bytes_used + shdr_ptr->bytes_left +
-                  sizeof(small_pool_hdr);
+                  sizeof(small_pool_hdr) + ALIGN_SIZE - 1;
    jpeg_free_small(cinfo, (void *)shdr_ptr, space_freed);
    mem->total_space_allocated -= space_freed;
    shdr_ptr = next_shdr_ptr;
@@ -43,25 +43,11 @@

 #if BITS_IN_JSAMPLE == 8
 /* JSAMPLE should be the smallest type that will hold the values 0..255.
- * You can use a signed char by having GETJSAMPLE mask it with 0xFF.
 */

-#ifdef HAVE_UNSIGNED_CHAR
-
 typedef unsigned char JSAMPLE;
 #define GETJSAMPLE(value)  ((int)(value))

-#else /* not HAVE_UNSIGNED_CHAR */
-
-typedef char JSAMPLE;
-#ifdef __CHAR_UNSIGNED__
-#define GETJSAMPLE(value)  ((int)(value))
-#else
-#define GETJSAMPLE(value)  ((int)(value) & 0xFF)
-#endif /* __CHAR_UNSIGNED__ */
-
-#endif /* HAVE_UNSIGNED_CHAR */
-
 #define MAXJSAMPLE      255
 #define CENTERJSAMPLE   128

@@ -97,22 +83,9 @@ typedef short JCOEF;
 * managers, this is also the data type passed to fread/fwrite.
 */

-#ifdef HAVE_UNSIGNED_CHAR
-
 typedef unsigned char JOCTET;
 #define GETJOCTET(value)  (value)

-#else /* not HAVE_UNSIGNED_CHAR */
-
-typedef char JOCTET;
-#ifdef __CHAR_UNSIGNED__
-#define GETJOCTET(value)  (value)
-#else
-#define GETJOCTET(value)  ((value) & 0xFF)
-#endif /* __CHAR_UNSIGNED__ */
-
-#endif /* HAVE_UNSIGNED_CHAR */
-

 /* These typedefs are used for various table entries and so forth.
 * They must be at least as wide as specified; but making them too big
@@ -123,15 +96,7 @@ typedef char JOCTET;

 /* UINT8 must hold at least the values 0..255. */

-#ifdef HAVE_UNSIGNED_CHAR
 typedef unsigned char UINT8;
-#else /* not HAVE_UNSIGNED_CHAR */
-#ifdef __CHAR_UNSIGNED__
-typedef char UINT8;
-#else /* not __CHAR_UNSIGNED__ */
-typedef short UINT8;
-#endif /* __CHAR_UNSIGNED__ */
-#endif /* HAVE_UNSIGNED_CHAR */

 /* UINT16 must hold at least the values 0..65535. */

@@ -5,8 +5,9 @@
 * Copyright (C) 1991-1997, Thomas G. Lane.
 * Modified 1997-2009 by Guido Vollbeding.
 * libjpeg-turbo Modifications:
- * Copyright (C) 2015-2016, D. R. Commander.
+ * Copyright (C) 2015-2016, 2019, 2021, D. R. Commander.
 * Copyright (C) 2015, Google, Inc.
+ * Copyright (C) 2021, Alex Richardson.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 *
@@ -47,6 +48,18 @@ typedef enum {            /* Operating modes for buffer controllers */
 /* JLONG must hold at least signed 32-bit values. */
 typedef long JLONG;

+/* JUINTPTR must hold pointer values. */
+#ifdef __UINTPTR_TYPE__
+/*
+ * __UINTPTR_TYPE__ is GNU-specific and available in GCC 4.6+ and Clang 3.0+.
+ * Fortunately, that is sufficient to support the few architectures for which
+ * sizeof(void *) != sizeof(size_t).  The only other options would require C99
+ * or Clang-specific builtins.
+ */
+typedef __UINTPTR_TYPE__ JUINTPTR;
+#else
+typedef size_t JUINTPTR;
+#endif

 /*
 * Left shift macro that handles a negative operand without causing any
@@ -158,6 +171,9 @@ struct jpeg_decomp_master {
  JDIMENSION first_MCU_col[MAX_COMPONENTS];
  JDIMENSION last_MCU_col[MAX_COMPONENTS];
  boolean jinit_upsampler_no_alloc;
+
+  /* Last iMCU row that was successfully decoded */
+  JDIMENSION last_good_iMCU_row;
 };

 /* Input control module */
@@ -479,7 +479,7 @@ color_quantize(j_decompress_ptr cinfo, JSAMPARRAY input_buf,
    for (col = width; col > 0; col--) {
      pixcode = 0;
      for (ci = 0; ci < nc; ci++) {
-        pixcode += GETJSAMPLE(colorindex[ci][GETJSAMPLE(*ptrin++)]);
+        pixcode += colorindex[ci][*ptrin++];
      }
      *ptrout++ = (JSAMPLE)pixcode;
    }
@@ -506,9 +506,9 @@ color_quantize3(j_decompress_ptr cinfo, JSAMPARRAY input_buf,
    ptrin = input_buf[row];
    ptrout = output_buf[row];
    for (col = width; col > 0; col--) {
-      pixcode  = GETJSAMPLE(colorindex0[GETJSAMPLE(*ptrin++)]);
-      pixcode += GETJSAMPLE(colorindex1[GETJSAMPLE(*ptrin++)]);
-      pixcode += GETJSAMPLE(colorindex2[GETJSAMPLE(*ptrin++)]);
+      pixcode  = colorindex0[*ptrin++];
+      pixcode += colorindex1[*ptrin++];
+      pixcode += colorindex2[*ptrin++];
      *ptrout++ = (JSAMPLE)pixcode;
    }
  }
@@ -552,7 +552,7 @@ quantize_ord_dither(j_decompress_ptr cinfo, JSAMPARRAY input_buf,
         * required amount of padding.
         */
        *output_ptr +=
-          colorindex_ci[GETJSAMPLE(*input_ptr) + dither[col_index]];
+          colorindex_ci[*input_ptr + dither[col_index]];
        input_ptr += nc;
        output_ptr++;
        col_index = (col_index + 1) & ODITHER_MASK;
@@ -595,12 +595,9 @@ quantize3_ord_dither(j_decompress_ptr cinfo, JSAMPARRAY input_buf,
    col_index = 0;

    for (col = width; col > 0; col--) {
-      pixcode  =
-        GETJSAMPLE(colorindex0[GETJSAMPLE(*input_ptr++) + dither0[col_index]]);
-      pixcode +=
-        GETJSAMPLE(colorindex1[GETJSAMPLE(*input_ptr++) + dither1[col_index]]);
-      pixcode +=
-        GETJSAMPLE(colorindex2[GETJSAMPLE(*input_ptr++) + dither2[col_index]]);
+      pixcode  = colorindex0[(*input_ptr++) + dither0[col_index]];
+      pixcode += colorindex1[(*input_ptr++) + dither1[col_index]];
+      pixcode += colorindex2[(*input_ptr++) + dither2[col_index]];
      *output_ptr++ = (JSAMPLE)pixcode;
      col_index = (col_index + 1) & ODITHER_MASK;
    }
@@ -677,15 +674,15 @@ quantize_fs_dither(j_decompress_ptr cinfo, JSAMPARRAY input_buf,
         * The maximum error is +- MAXJSAMPLE; this sets the required size
         * of the range_limit array.
         */
-        cur += GETJSAMPLE(*input_ptr);
-        cur = GETJSAMPLE(range_limit[cur]);
+        cur += *input_ptr;
+        cur = range_limit[cur];
        /* Select output value, accumulate into output code for this pixel */
-        pixcode = GETJSAMPLE(colorindex_ci[cur]);
+        pixcode = colorindex_ci[cur];
        *output_ptr += (JSAMPLE)pixcode;
        /* Compute actual representation error at this pixel */
        /* Note: we can do this even though we don't have the final */
        /* pixel code, because the colormap is orthogonal. */
-        cur -= GETJSAMPLE(colormap_ci[pixcode]);
+        cur -= colormap_ci[pixcode];
        /* Compute error fractions to be propagated to adjacent pixels.
         * Add these into the running sums, and simultaneously shift the
         * next-line error sums left by 1 column.
@@ -215,9 +215,9 @@ prescan_quantize(j_decompress_ptr cinfo, JSAMPARRAY input_buf,
    ptr = input_buf[row];
    for (col = width; col > 0; col--) {
      /* get pixel value and index into the histogram */
-      histp = &histogram[GETJSAMPLE(ptr[0]) >> C0_SHIFT]
-                        [GETJSAMPLE(ptr[1]) >> C1_SHIFT]
-                        [GETJSAMPLE(ptr[2]) >> C2_SHIFT];
+      histp = &histogram[ptr[0] >> C0_SHIFT]
+                        [ptr[1] >> C1_SHIFT]
+                        [ptr[2] >> C2_SHIFT];
      /* increment, check for overflow and undo increment if so. */
      if (++(*histp) <= 0)
        (*histp)--;
@@ -665,7 +665,7 @@ find_nearby_colors(j_decompress_ptr cinfo, int minc0, int minc1, int minc2,

  for (i = 0; i < numcolors; i++) {
    /* We compute the squared-c0-distance term, then add in the other two. */
-    x = GETJSAMPLE(cinfo->colormap[0][i]);
+    x = cinfo->colormap[0][i];
    if (x < minc0) {
      tdist = (x - minc0) * C0_SCALE;
      min_dist = tdist * tdist;
@@ -688,7 +688,7 @@ find_nearby_colors(j_decompress_ptr cinfo, int minc0, int minc1, int minc2,
      }
    }

-    x = GETJSAMPLE(cinfo->colormap[1][i]);
+    x = cinfo->colormap[1][i];
    if (x < minc1) {
      tdist = (x - minc1) * C1_SCALE;
      min_dist += tdist * tdist;
@@ -710,7 +710,7 @@ find_nearby_colors(j_decompress_ptr cinfo, int minc0, int minc1, int minc2,
      }
    }

-    x = GETJSAMPLE(cinfo->colormap[2][i]);
+    x = cinfo->colormap[2][i];
    if (x < minc2) {
      tdist = (x - minc2) * C2_SCALE;
      min_dist += tdist * tdist;
@@ -788,13 +788,13 @@ find_best_colors(j_decompress_ptr cinfo, int minc0, int minc1, int minc2,
 #define STEP_C2  ((1 << C2_SHIFT) * C2_SCALE)

  for (i = 0; i < numcolors; i++) {
-    icolor = GETJSAMPLE(colorlist[i]);
+    icolor = colorlist[i];
    /* Compute (square of) distance from minc0/c1/c2 to this color */
-    inc0 = (minc0 - GETJSAMPLE(cinfo->colormap[0][icolor])) * C0_SCALE;
+    inc0 = (minc0 - cinfo->colormap[0][icolor]) * C0_SCALE;
    dist0 = inc0 * inc0;
-    inc1 = (minc1 - GETJSAMPLE(cinfo->colormap[1][icolor])) * C1_SCALE;
+    inc1 = (minc1 - cinfo->colormap[1][icolor]) * C1_SCALE;
    dist0 += inc1 * inc1;
-    inc2 = (minc2 - GETJSAMPLE(cinfo->colormap[2][icolor])) * C2_SCALE;
+    inc2 = (minc2 - cinfo->colormap[2][icolor]) * C2_SCALE;
    dist0 += inc2 * inc2;
    /* Form the initial difference increments */
    inc0 = inc0 * (2 * STEP_C0) + STEP_C0 * STEP_C0;
@@ -879,7 +879,7 @@ fill_inverse_cmap(j_decompress_ptr cinfo, int c0, int c1, int c2)
    for (ic1 = 0; ic1 < BOX_C1_ELEMS; ic1++) {
      cachep = &histogram[c0 + ic0][c1 + ic1][c2];
      for (ic2 = 0; ic2 < BOX_C2_ELEMS; ic2++) {
-        *cachep++ = (histcell)(GETJSAMPLE(*cptr++) + 1);
+        *cachep++ = (histcell)((*cptr++) + 1);
      }
    }
  }
@@ -909,9 +909,9 @@ pass2_no_dither(j_decompress_ptr cinfo, JSAMPARRAY input_buf,
    outptr = output_buf[row];
    for (col = width; col > 0; col--) {
      /* get pixel value and index into the cache */
-      c0 = GETJSAMPLE(*inptr++) >> C0_SHIFT;
-      c1 = GETJSAMPLE(*inptr++) >> C1_SHIFT;
-      c2 = GETJSAMPLE(*inptr++) >> C2_SHIFT;
+      c0 = (*inptr++) >> C0_SHIFT;
+      c1 = (*inptr++) >> C1_SHIFT;
+      c2 = (*inptr++) >> C2_SHIFT;
      cachep = &histogram[c0][c1][c2];
      /* If we have not seen this color before, find nearest colormap entry */
      /* and update the cache */
@@ -996,12 +996,12 @@ pass2_fs_dither(j_decompress_ptr cinfo, JSAMPARRAY input_buf,
       * The maximum error is +- MAXJSAMPLE (or less with error limiting);
       * this sets the required size of the range_limit array.
       */
-      cur0 += GETJSAMPLE(inptr[0]);
-      cur1 += GETJSAMPLE(inptr[1]);
-      cur2 += GETJSAMPLE(inptr[2]);
-      cur0 = GETJSAMPLE(range_limit[cur0]);
-      cur1 = GETJSAMPLE(range_limit[cur1]);
-      cur2 = GETJSAMPLE(range_limit[cur2]);
+      cur0 += inptr[0];
+      cur1 += inptr[1];
+      cur2 += inptr[2];
+      cur0 = range_limit[cur0];
+      cur1 = range_limit[cur1];
+      cur2 = range_limit[cur2];
      /* Index into the cache with adjusted pixel value */
      cachep =
        &histogram[cur0 >> C0_SHIFT][cur1 >> C1_SHIFT][cur2 >> C2_SHIFT];
@@ -1015,9 +1015,9 @@ pass2_fs_dither(j_decompress_ptr cinfo, JSAMPARRAY input_buf,
        register int pixcode = *cachep - 1;
        *outptr = (JSAMPLE)pixcode;
        /* Compute representation error for this pixel */
-        cur0 -= GETJSAMPLE(colormap0[pixcode]);
-        cur1 -= GETJSAMPLE(colormap1[pixcode]);
-        cur2 -= GETJSAMPLE(colormap2[pixcode]);
+        cur0 -= colormap0[pixcode];
+        cur1 -= colormap1[pixcode];
+        cur2 -= colormap2[pixcode];
      }
      /* Compute error fractions to be propagated to adjacent pixels.
       * Add these into the running sums, and simultaneously shift the
@@ -4,6 +4,7 @@
 * Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
 * Copyright (C) 2011, 2014, D. R. Commander.
 * Copyright (C) 2015-2016, 2018, Matthieu Darbois.
+ * Copyright (C) 2020, Arm Limited.
 *
 * Based on the x86 SIMD extension for IJG JPEG library,
 * Copyright (C) 1999-2006, MIYASAKA Masaru.
@@ -75,6 +76,7 @@ EXTERN(void) jsimd_int_upsample(j_decompress_ptr cinfo,

 EXTERN(int) jsimd_can_h2v2_fancy_upsample(void);
 EXTERN(int) jsimd_can_h2v1_fancy_upsample(void);
+EXTERN(int) jsimd_can_h1v2_fancy_upsample(void);

 EXTERN(void) jsimd_h2v2_fancy_upsample(j_decompress_ptr cinfo,
                                       jpeg_component_info *compptr,
@@ -84,6 +86,10 @@ EXTERN(void) jsimd_h2v1_fancy_upsample(j_decompress_ptr cinfo,
                                       jpeg_component_info *compptr,
                                       JSAMPARRAY input_data,
                                       JSAMPARRAY *output_data_ptr);
+EXTERN(void) jsimd_h1v2_fancy_upsample(j_decompress_ptr cinfo,
+                                       jpeg_component_info *compptr,
+                                       JSAMPARRAY input_data,
+                                       JSAMPARRAY *output_data_ptr);

 EXTERN(int) jsimd_can_h2v2_merged_upsample(void);
 EXTERN(int) jsimd_can_h2v1_merged_upsample(void);
@@ -4,6 +4,7 @@
 * Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
 * Copyright (C) 2009-2011, 2014, D. R. Commander.
 * Copyright (C) 2015-2016, 2018, Matthieu Darbois.
+ * Copyright (C) 2020, Arm Limited.
 *
 * Based on the x86 SIMD extension for IJG JPEG library,
 * Copyright (C) 1999-2006, MIYASAKA Masaru.
@@ -169,6 +170,12 @@ jsimd_can_h2v1_fancy_upsample(void)
  return 0;
 }

+GLOBAL(int)
+jsimd_can_h1v2_fancy_upsample(void)
+{
+  return 0;
+}
+
 GLOBAL(void)
 jsimd_h2v2_fancy_upsample(j_decompress_ptr cinfo, jpeg_component_info *compptr,
                          JSAMPARRAY input_data, JSAMPARRAY *output_data_ptr)
@@ -181,6 +188,12 @@ jsimd_h2v1_fancy_upsample(j_decompress_ptr cinfo, jpeg_component_info *compptr,
 {
 }

+GLOBAL(void)
+jsimd_h1v2_fancy_upsample(j_decompress_ptr cinfo, jpeg_component_info *compptr,
+                          JSAMPARRAY input_data, JSAMPARRAY *output_data_ptr)
+{
+}
+
 GLOBAL(int)
 jsimd_can_h2v2_merged_upsample(void)
 {
@@ -2,9 +2,9 @@
 * jversion.h
 *
 * This file was part of the Independent JPEG Group's software:
- * Copyright (C) 1991-2012, Thomas G. Lane, Guido Vollbeding.
+ * Copyright (C) 1991-2020, Thomas G. Lane, Guido Vollbeding.
 * libjpeg-turbo Modifications:
- * Copyright (C) 2010, 2012-2020, D. R. Commander.
+ * Copyright (C) 2010, 2012-2021, D. R. Commander.
 * For conditions of distribution and use, see the accompanying README.ijg
 * file.
 *
@@ -37,9 +37,9 @@
 */

 #define JCOPYRIGHT \
-  "Copyright (C) 2009-2020 D. R. Commander\n" \
+  "Copyright (C) 2009-2021 D. R. Commander\n" \
  "Copyright (C) 2015, 2020 Google, Inc.\n" \
-  "Copyright (C) 2019 Arm Limited\n" \
+  "Copyright (C) 2019-2020 Arm Limited\n" \
  "Copyright (C) 2015-2016, 2018 Matthieu Darbois\n" \
  "Copyright (C) 2011-2016 Siarhei Siamashka\n" \
  "Copyright (C) 2015 Intel Corporation\n" \
@@ -48,7 +48,7 @@
  "Copyright (C) 2009, 2012 Pierre Ossman for Cendio AB\n" \
  "Copyright (C) 2009-2011 Nokia Corporation and/or its subsidiary(-ies)\n" \
  "Copyright (C) 1999-2006 MIYASAKA Masaru\n" \
-  "Copyright (C) 1991-2017 Thomas G. Lane, Guido Vollbeding"
+  "Copyright (C) 1991-2020 Thomas G. Lane, Guido Vollbeding"

 #define JCOPYRIGHT_SHORT \
-  "Copyright (C) 1991-2020 The libjpeg-turbo Project and many others"
+  "Copyright (C) 1991-2021 The libjpeg-turbo Project and many others"
@@ -77,7 +77,11 @@ endif(MSVC)
 add_library(${PNG_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_srcs} ${lib_hdrs})
 target_link_libraries(${PNG_LIBRARY} ${ZLIB_LIBRARIES})

-ocv_warnings_disable(CMAKE_C_FLAGS -Wundef -Wcast-align -Wimplicit-fallthrough -Wunused-parameter -Wsign-compare)
+ocv_warnings_disable(CMAKE_C_FLAGS -Wundef -Wcast-align -Wimplicit-fallthrough -Wunused-parameter -Wsign-compare
+    -Wmaybe-uninitialized
+    -Wnull-pointer-subtraction                   # clang15
+    -Wunused-but-set-variable                    # clang15
+)

 set_target_properties(${PNG_LIBRARY}
  PROPERTIES OUTPUT_NAME ${PNG_LIBRARY}
@@ -452,8 +452,10 @@ ocv_warnings_disable(CMAKE_C_FLAGS -Wno-unused-but-set-variable -Wmissing-protot
                                   -Wcast-align -Wshadow -Wno-maybe-uninitialized -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast
                                   -Wmisleading-indentation
                                   -Wimplicit-fallthrough
+                                   -Wunused-parameter  # clang
+                                   -Warray-parameter
+                                   -Wstrict-prototypes  # clang15
 )
-ocv_warnings_disable(CMAKE_C_FLAGS -Wunused-parameter) # clang
 ocv_warnings_disable(CMAKE_CXX_FLAGS -Wmissing-declarations -Wunused-parameter -Wmissing-prototypes
    -Wundef  # tiffiop.h: #if __clang_major__ >= 4
 )
@@ -0,0 +1,73 @@
+set(TIMVX_COMMIT_HASH "1d9c7ab941b3d8d9c4d28d80058402725731e3d6")
+set(OCV_TIMVX_DIR "${OpenCV_BINARY_DIR}/3rdparty/libtim-vx")
+set(OCV_TIMVX_SOURCE_PATH "${OCV_TIMVX_DIR}/TIM-VX-${TIMVX_COMMIT_HASH}")
+
+# Download TIM-VX source code
+if(EXISTS "${OCV_TIMVX_SOURCE_PATH}")
+    message(STATUS "TIM-VX: Use cache of TIM-VX source code at ${OCV_TIMVX_SOURCE_PATH}")
+    set(TIMVX_FOUND ON)
+else()
+    set(OCV_TIMVX_FILENAME "${TIMVX_COMMIT_HASH}.zip")
+    set(OCV_TIMVX_URL "https://github.com/VeriSilicon/TIM-VX/archive/")
+    set(timvx_zip_md5sum 92619cc4498014ac7a09834d5e33ebd5)
+
+    ocv_download(FILENAME ${OCV_TIMVX_FILENAME}
+                 HASH ${timvx_zip_md5sum}
+                 URL "${OCV_TIMVX_URL}"
+                 DESTINATION_DIR "${OCV_TIMVX_DIR}"
+                 ID "TIM-VX"
+                 STATUS res
+                 UNPACK RELATIVE_URL)
+    if(res)
+        set(TIMVX_FOUND ON)
+        message(STATUS "TIM-VX: Source code downloaded at ${OCV_TIMVX_SOURCE_PATH}.") 
+    else()
+        set(TIMVX_FOUND OFF)
+        message(STATUS "TIM-VX: Failed to download source code from github. Turning off TIMVX_FOUND")
+        return()
+    endif()
+endif()
+
+# set VIVANTE SDK especially for x86_64 which comes along with TIM-VX source code
+if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)
+    set(VIVANTE_SDK_DIR "${OCV_TIMVX_SOURCE_PATH}/prebuilt-sdk/x86_64_linux")
+    message(STATUS "TIM-VX: Build from source using prebuilt x86_64 VIVANTE SDK.")
+endif()
+
+# Verify if requested VIVANTE SDK libraries are all found
+find_vivante_sdk_libs(missing ${VIVANTE_SDK_DIR})
+if(missing)
+    message(STATUS "TIM-VX: Failed to find ${missing} in ${VIVANTE_SDK_DIR}/lib. Turning off TIMVX_VIV_FOUND")
+    set(TIMVX_VIV_FOUND OFF)
+else()
+    message(STATUS "TIM-VX: dependent VIVANTE SDK libraries are found at ${VIVANTE_SDK_DIR}/lib.")
+    set(TIMVX_VIV_FOUND ON)
+endif()
+
+if(TIMVX_VIV_FOUND)
+    # vars used by TIM-VX CMake scripts
+    set(EXTERNAL_VIV_SDK "${VIVANTE_SDK_DIR}" CACHE INTERNAL "" FORCE)
+    set(VIV_SDK_DRIVER_PREFIX "lib" CACHE INTERNAL "" FORCE)
+endif()
+
+if(TIMVX_FOUND AND TIMVX_VIV_FOUND)
+    set(BUILD_TIMVX ON)
+else()
+    return()
+endif()
+
+if(BUILD_TIMVX)
+    set(HAVE_TIMVX 1)
+
+    ocv_warnings_disable(CMAKE_C_FLAGS -Wunused-parameter -Wstrict-prototypes -Wundef -Wsign-compare -Wmissing-prototypes -Wmissing-declarations -Wstrict-aliasing -Wunused-but-set-variable -Wmaybe-uninitialized -Wshadow -Wsuggest-override -Wswitch)
+    ocv_warnings_disable(CMAKE_CXX_FLAGS -Wunused-parameter -Wstrict-prototypes -Wundef -Wsign-compare -Wunused-but-set-variable -Wshadow -Wsuggest-override -Wmissing-declarations -Wswitch)
+
+    set(TIMVX_INC_DIR "${OCV_TIMVX_SOURCE_PATH}/include" CACHE INTERNAL "TIM-VX include directory")
+    if(EXISTS "${OCV_TIMVX_SOURCE_PATH}/CMakeLists.txt")
+        add_subdirectory("${OCV_TIMVX_SOURCE_PATH}" "${OCV_TIMVX_DIR}/build")
+    else()
+        message(WARNING "TIM-VX: Missing 'CMakeLists.txt' in the source code: ${OCV_TIMVX_SOURCE_PATH}")
+    endif()
+    ocv_install_target(tim-vx EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev)
+    set(TIMVX_LIB "tim-vx")
+endif()
@@ -32,7 +32,9 @@ endif()
 #         Define the library target:
 # ----------------------------------------------------------------------------------

-add_definitions(-DWEBP_USE_THREAD)
+if(NOT OPENCV_DISABLE_THREAD_SUPPORT)
+  add_definitions(-DWEBP_USE_THREAD)
+endif()

 add_library(${WEBP_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_srcs} ${lib_hdrs})
 if(ANDROID)
@@ -43,6 +45,7 @@ ocv_warnings_disable(CMAKE_C_FLAGS -Wunused-variable -Wunused-function -Wshadow
    -Wmissing-prototypes  # clang
    -Wmissing-declarations # gcc
    -Wimplicit-fallthrough
+    -Wunused-but-set-variable # clang15
 )
 ocv_warnings_disable(CMAKE_C_FLAGS /wd4244 /wd4267) # vs2005

@@ -13,6 +13,7 @@ project(openjpeg C)

 ocv_warnings_disable(CMAKE_C_FLAGS
    -Wimplicit-const-int-float-conversion  # clang
+    -Wunused-but-set-variable # clang15
 )

 #-----------------------------------------------------------------------------
@@ -77,7 +77,7 @@ E.g. external ref-counting is implemented for 1.0 version and native OpenVX one

 Also there are some **C++ 11** features are used (e.g. rvalue ref-s) when their availability is detected at ***compile time***.

-C++ exceptions are used for errors indication instead of return codes. There are two types of exceptions are defined:  `RuntimeError` is thrown when OpenVX C call returned unsuccessful result and `WrapperError` is thrown when a problem is occured in the wrappers code. Both exception calsses are derived from `std::exception` (actually from its inheritants).
+C++ exceptions are used for errors indication instead of return codes. There are two types of exceptions are defined:  `RuntimeError` is thrown when OpenVX C call returned unsuccessful result and `WrapperError` is thrown when a problem is occurred in the wrappers code. Both exception calsses are derived from `std::exception` (actually from its inheritants).

 The so called **OpenVX objects** (e.g. `vx_image`) are represented as C++ classes in wrappers.
 All these classes use automatic ref-counting that allows development of exception-safe code.
@@ -923,6 +923,11 @@ int ovx_hal_cvtGraytoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep,
 }

 int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx)
+{
+    return ovx_hal_cvtTwoPlaneYUVtoBGREx(a, astep, a + h * astep, astep, b, bstep, w, h, bcn, swapBlue, uIdx);
+}
+
+int ovx_hal_cvtTwoPlaneYUVtoBGREx(const uchar * a, size_t astep, const uchar * b, size_t bstep, uchar * c, size_t cstep, int w, int h, int bcn, bool swapBlue, int uIdx)
 {
    if (skipSmallImages<VX_KERNEL_COLOR_CONVERT>(w, h))
        return CV_HAL_ERROR_NOT_IMPLEMENTED;
@@ -933,8 +938,7 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t

    if (w & 1 || h & 1) // It's not described in spec but sample implementation unable to convert odd sized images
        return CV_HAL_ERROR_NOT_IMPLEMENTED;
-    refineStep(w, h, uIdx ? VX_DF_IMAGE_NV21 : VX_DF_IMAGE_NV12, astep);
-    refineStep(w, h, bcn == 3 ? VX_DF_IMAGE_RGB : VX_DF_IMAGE_RGBX, bstep);
+
    try
    {
        ivx::Context ctx = getOpenVXHALContext();
@@ -943,8 +947,8 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t
        std::vector<void *> ptrs;
            addr.push_back(ivx::Image::createAddressing(w, h, 1, (vx_int32)astep));
            ptrs.push_back((void*)a);
-            addr.push_back(ivx::Image::createAddressing(w / 2, h / 2, 2, (vx_int32)astep));
-            ptrs.push_back((void*)(a + h * astep));
+            addr.push_back(ivx::Image::createAddressing(w / 2, h / 2, 2, (vx_int32)bstep));
+            ptrs.push_back((void*)b);

        vxImage
            ia = ivx::Image::createFromHandle(ctx, uIdx ? VX_DF_IMAGE_NV21 : VX_DF_IMAGE_NV12, addr, ptrs);
@@ -952,7 +956,7 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t
            return CV_HAL_ERROR_NOT_IMPLEMENTED; // OpenCV store NV12/NV21 as RANGE_RESTRICTED while OpenVX expect RANGE_FULL
        vxImage
            ib = ivx::Image::createFromHandle(ctx, bcn == 3 ? VX_DF_IMAGE_RGB : VX_DF_IMAGE_RGBX,
-                ivx::Image::createAddressing(w, h, bcn, (vx_int32)bstep), b);
+                ivx::Image::createAddressing(w, h, bcn, (vx_int32)cstep), c);
        ivx::IVX_CHECK_STATUS(vxuColorConvert(ctx, ia, ib));
    }
    catch (ivx::RuntimeError & e)
@@ -49,6 +49,7 @@ int ovx_hal_morph(cvhalFilter2D *filter_context, uchar *a, size_t astep, uchar *
 int ovx_hal_cvtBGRtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int depth, int acn, int bcn, bool swapBlue);
 int ovx_hal_cvtGraytoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int depth, int bcn);
 int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx);
+int ovx_hal_cvtTwoPlaneYUVtoBGREx(const uchar * a, size_t astep, const uchar * b, size_t bstep, uchar * c, size_t cstep, int w, int h, int bcn, bool swapBlue, int uIdx);
 int ovx_hal_cvtThreePlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx);
 int ovx_hal_cvtBGRtoThreePlaneYUV(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int acn, bool swapBlue, int uIdx);
 int ovx_hal_cvtOnePlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx, int ycn);
@@ -130,6 +131,8 @@ int ovx_hal_integral(int depth, int sdepth, int, const uchar * a, size_t astep,
 #define cv_hal_cvtGraytoBGR ovx_hal_cvtGraytoBGR
 #undef cv_hal_cvtTwoPlaneYUVtoBGR
 #define cv_hal_cvtTwoPlaneYUVtoBGR ovx_hal_cvtTwoPlaneYUVtoBGR
+#undef cv_hal_cvtTwoPlaneYUVtoBGREx
+#define cv_hal_cvtTwoPlaneYUVtoBGREx ovx_hal_cvtTwoPlaneYUVtoBGREx
 #undef cv_hal_cvtThreePlaneYUVtoBGR
 #define cv_hal_cvtThreePlaneYUVtoBGR ovx_hal_cvtThreePlaneYUVtoBGR
 #undef cv_hal_cvtBGRtoThreePlaneYUV
@@ -13,6 +13,9 @@ if(MSVC)
                                       /wd4701 /wd4703     # potentially uninitialized local/pointer variable 'value' used
                                       /wd4505             # unreferenced local function has been removed
  )
+  if(MSVC_VERSION LESS 1920)  # MSVS 2015/2017
+    ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4309)          # 'static_cast': truncation of constant value
+  endif()
 else()
  #NOTE: -Wno-invalid-offsetof was used as solution for invalid offset warning on protobuf #3450
  ocv_warnings_disable(CMAKE_CXX_FLAGS -Wno-deprecated -Wmissing-prototypes -Wmissing-declarations -Wshadow
@@ -49,98 +52,101 @@ endfunction()

 set(PROTOBUF_ROOT "${CMAKE_CURRENT_LIST_DIR}")

-if(MSVC)
-  set(ATOMICOPS_INTERNALS ${PROTOBUF_ROOT}/src/google/protobuf/stubs/atomicops_internals_x86_msvc.cc)
-else()
-  set(ATOMICOPS_INTERNALS ${PROTOBUF_ROOT}/src/google/protobuf/stubs/atomicops_internals_x86_gcc.cc)
-endif()
-

 append_if_exist(Protobuf_SRCS
 # libprotobuf-lite
+  ${PROTOBUF_ROOT}/src/google/protobuf/any_lite.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/arena.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/arenastring.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/extension_set.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven_lite.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/generated_enum_util.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven_lite.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_tctable_lite.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_util.cc
+  ${PROTOBUF_ROOT}/src/google/protobuf/implicit_weak_message.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/inlined_string_field.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/io/coded_stream.cc
+  ${PROTOBUF_ROOT}/src/google/protobuf/io/io_win32.cc
+  ${PROTOBUF_ROOT}/src/google/protobuf/io/strtod.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream.cc
+  ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream_impl.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream_impl_lite.cc
+  ${PROTOBUF_ROOT}/src/google/protobuf/map.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/message_lite.cc
+  ${PROTOBUF_ROOT}/src/google/protobuf/parse_context.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/repeated_field.cc
-  ${ATOMICOPS_INTERNALS}
+  ${PROTOBUF_ROOT}/src/google/protobuf/repeated_ptr_field.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/bytestream.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/common.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/int128.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/io_win32.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/once.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/status.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/statusor.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/statusor.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/stringpiece.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/stringprintf.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/structurally_valid.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/strutil.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/time.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/time.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/wire_format_lite.cc
+
 # libprotobuf
  ${PROTOBUF_ROOT}/src/google/protobuf/any.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/any.pb.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/api.pb.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/any.pb.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/api.pb.cc
 #  ${PROTOBUF_ROOT}/src/google/protobuf/compiler/importer.cc
 #  ${PROTOBUF_ROOT}/src/google/protobuf/compiler/parser.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/descriptor.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/descriptor.pb.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/descriptor_database.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/duration.pb.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/duration.pb.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/dynamic_message.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/empty.pb.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/empty.pb.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/extension_set_heavy.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/field_mask.pb.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/field_mask.pb.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_bases.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_reflection.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/io/gzip_stream.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/io/printer.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/io/strtod.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_tctable_full.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/io/gzip_stream.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/io/printer.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/io/tokenizer.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream_impl.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/map_field.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/message.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/reflection_ops.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/service.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/source_context.pb.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/struct.pb.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/mathlimits.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/service.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/source_context.pb.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/struct.pb.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/stubs/substitute.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/text_format.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/timestamp.pb.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/type.pb.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/timestamp.pb.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/type.pb.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/unknown_field_set.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/delimited_message_util.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/field_comparator.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/field_mask_util.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/datapiece.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/default_value_objectwriter.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/delimited_message_util.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/field_comparator.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/field_mask_util.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/datapiece.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/default_value_objectwriter.cc
 #  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/error_listener.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/field_mask_utility.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_escaping.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_objectwriter.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_stream_parser.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/object_writer.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/proto_writer.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectsource.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectwriter.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/type_info.cc
-#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/type_info_test_helper.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/utility.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/json_util.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/message_differencer.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/time_util.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/util/type_resolver_util.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/field_mask_utility.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_escaping.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_objectwriter.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_stream_parser.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/object_writer.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/proto_writer.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectsource.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectwriter.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/type_info.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/utility.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/json_util.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/message_differencer.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/time_util.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/util/type_resolver_util.cc
  ${PROTOBUF_ROOT}/src/google/protobuf/wire_format.cc
-  ${PROTOBUF_ROOT}/src/google/protobuf/wrappers.pb.cc
+#  ${PROTOBUF_ROOT}/src/google/protobuf/wrappers.pb.cc
 )

-include_directories(BEFORE "${PROTOBUF_ROOT}/src")  # ensure using if own headers: https://github.com/opencv/opencv/issues/13328
+
+include_directories(BEFORE "${PROTOBUF_ROOT}/src")  # ensure using of own headers: https://github.com/opencv/opencv/issues/13328
+
 add_library(libprotobuf STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${Protobuf_SRCS})
 target_include_directories(libprotobuf SYSTEM PUBLIC $<BUILD_INTERFACE:${PROTOBUF_ROOT}/src>)
 set_target_properties(libprotobuf
@@ -1,14 +1,4 @@
-This license applies to all parts of Protocol Buffers except the following:
-
-  - Atomicops support for generic gcc, located in
-    src/google/protobuf/stubs/atomicops_internals_generic_gcc.h.
-    This file is copyrighted by Red Hat Inc.
-
-  - Atomicops support for AIX/POWER, located in
-    src/google/protobuf/stubs/atomicops_internals_power.h.
-    This file is copyrighted by Bloomberg Finance LP.
-
-Copyright 2014, Google Inc.  All rights reserved.
+Copyright 2008 Google Inc.  All rights reserved.

 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -1,3 +1,3 @@
 Project: Protocol Buffers - Google's data interchange format
 Source code: https://github.com/protocolbuffers/protobuf
-Version: 3.5.2
+Version: 3.19.1
@@ -30,85 +30,54 @@

 #include <google/protobuf/any.h>

+#include <google/protobuf/arenastring.h>
+#include <google/protobuf/descriptor.h>
 #include <google/protobuf/generated_message_util.h>
+#include <google/protobuf/message.h>
+
+#include <google/protobuf/port_def.inc>

 namespace google {
 namespace protobuf {
 namespace internal {

-namespace {
-string GetTypeUrl(const Descriptor* message,
-                  const string& type_url_prefix) {
-  if (!type_url_prefix.empty() &&
-      type_url_prefix[type_url_prefix.size() - 1] == '/') {
-    return type_url_prefix + message->full_name();
-  } else {
-    return type_url_prefix + "/" + message->full_name();
-  }
-}
-}  // namespace
-
-const char kAnyFullTypeName[] = "google.protobuf.Any";
-const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/";
-const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/";
-
-AnyMetadata::AnyMetadata(UrlType* type_url, ValueType* value)
-    : type_url_(type_url), value_(value) {
+bool AnyMetadata::PackFrom(Arena* arena, const Message& message) {
+  return PackFrom(arena, message, kTypeGoogleApisComPrefix);
 }

-void AnyMetadata::PackFrom(const Message& message) {
-  PackFrom(message, kTypeGoogleApisComPrefix);
-}
-
-void AnyMetadata::PackFrom(const Message& message,
-                           const string& type_url_prefix) {
-  type_url_->SetNoArena(&::google::protobuf::internal::GetEmptyString(),
-                        GetTypeUrl(message.GetDescriptor(), type_url_prefix));
-  message.SerializeToString(value_->MutableNoArena(
-      &::google::protobuf::internal::GetEmptyStringAlreadyInited()));
+bool AnyMetadata::PackFrom(Arena* arena, const Message& message,
+                           StringPiece type_url_prefix) {
+  type_url_->Set(
+      &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString(),
+      GetTypeUrl(message.GetDescriptor()->full_name(), type_url_prefix), arena);
+  return message.SerializeToString(
+      value_->Mutable(ArenaStringPtr::EmptyDefault{}, arena));
 }

 bool AnyMetadata::UnpackTo(Message* message) const {
-  if (!InternalIs(message->GetDescriptor())) {
+  if (!InternalIs(message->GetDescriptor()->full_name())) {
    return false;
  }
-  return message->ParseFromString(value_->GetNoArena());
+  return message->ParseFromString(value_->Get());
 }

-bool AnyMetadata::InternalIs(const Descriptor* descriptor) const {
-  const string type_url = type_url_->GetNoArena();
-  string full_name;
-  if (!ParseAnyTypeUrl(type_url, &full_name)) {
-    return false;
-  }
-  return full_name == descriptor->full_name();
-}
-
-bool ParseAnyTypeUrl(const string& type_url, string* full_type_name) {
-  size_t pos = type_url.find_last_of("/");
-  if (pos == string::npos || pos + 1 == type_url.size()) {
-    return false;
-  }
-  *full_type_name = type_url.substr(pos + 1);
-  return true;
-}
-
-
 bool GetAnyFieldDescriptors(const Message& message,
                            const FieldDescriptor** type_url_field,
                            const FieldDescriptor** value_field) {
-    const Descriptor* descriptor = message.GetDescriptor();
-    if (descriptor->full_name() != kAnyFullTypeName) {
-      return false;
-    }
-    *type_url_field = descriptor->FindFieldByNumber(1);
-    *value_field = descriptor->FindFieldByNumber(2);
-    return (*type_url_field != NULL &&
-            (*type_url_field)->type() == FieldDescriptor::TYPE_STRING &&
-            *value_field != NULL &&
-            (*value_field)->type() == FieldDescriptor::TYPE_BYTES);
+  const Descriptor* descriptor = message.GetDescriptor();
+  if (descriptor->full_name() != kAnyFullTypeName) {
+    return false;
+  }
+  *type_url_field = descriptor->FindFieldByNumber(1);
+  *value_field = descriptor->FindFieldByNumber(2);
+  return (*type_url_field != nullptr &&
+          (*type_url_field)->type() == FieldDescriptor::TYPE_STRING &&
+          *value_field != nullptr &&
+          (*value_field)->type() == FieldDescriptor::TYPE_BYTES);
 }

 }  // namespace internal
 }  // namespace protobuf
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
@@ -34,49 +34,89 @@
 #include <string>

 #include <google/protobuf/stubs/common.h>
-#include <google/protobuf/descriptor.h>
-#include <google/protobuf/message.h>
 #include <google/protobuf/arenastring.h>
+#include <google/protobuf/message_lite.h>
+
+#include <google/protobuf/port_def.inc>

 namespace google {
 namespace protobuf {
+
+class FieldDescriptor;
+class Message;
+
 namespace internal {

+extern const char kAnyFullTypeName[];          // "google.protobuf.Any".
+extern const char kTypeGoogleApisComPrefix[];  // "type.googleapis.com/".
+extern const char kTypeGoogleProdComPrefix[];  // "type.googleprod.com/".
+
+std::string GetTypeUrl(StringPiece message_name,
+                       StringPiece type_url_prefix);
+
 // Helper class used to implement google::protobuf::Any.
-class LIBPROTOBUF_EXPORT AnyMetadata {
+class PROTOBUF_EXPORT AnyMetadata {
  typedef ArenaStringPtr UrlType;
  typedef ArenaStringPtr ValueType;
 public:
  // AnyMetadata does not take ownership of "type_url" and "value".
-  AnyMetadata(UrlType* type_url, ValueType* value);
+  constexpr AnyMetadata(UrlType* type_url, ValueType* value)
+      : type_url_(type_url), value_(value) {}

  // Packs a message using the default type URL prefix: "type.googleapis.com".
  // The resulted type URL will be "type.googleapis.com/<message_full_name>".
-  void PackFrom(const Message& message);
+  // Returns false if serializing the message failed.
+  template <typename T>
+  bool PackFrom(Arena* arena, const T& message) {
+    return InternalPackFrom(arena, message, kTypeGoogleApisComPrefix,
+                            T::FullMessageName());
+  }
+
+  bool PackFrom(Arena* arena, const Message& message);
+
  // Packs a message using the given type URL prefix. The type URL will be
  // constructed by concatenating the message type's full name to the prefix
-  // with an optional "/" separator if the prefix doesn't already end up "/".
+  // with an optional "/" separator if the prefix doesn't already end with "/".
  // For example, both PackFrom(message, "type.googleapis.com") and
  // PackFrom(message, "type.googleapis.com/") yield the same result type
  // URL: "type.googleapis.com/<message_full_name>".
-  void PackFrom(const Message& message, const string& type_url_prefix);
+  // Returns false if serializing the message failed.
+  template <typename T>
+  bool PackFrom(Arena* arena, const T& message,
+                StringPiece type_url_prefix) {
+    return InternalPackFrom(arena, message, type_url_prefix,
+                            T::FullMessageName());
+  }
+
+  bool PackFrom(Arena* arena, const Message& message,
+                StringPiece type_url_prefix);

  // Unpacks the payload into the given message. Returns false if the message's
  // type doesn't match the type specified in the type URL (i.e., the full
  // name after the last "/" of the type URL doesn't match the message's actual
  // full name) or parsing the payload has failed.
+  template <typename T>
+  bool UnpackTo(T* message) const {
+    return InternalUnpackTo(T::FullMessageName(), message);
+  }
+
  bool UnpackTo(Message* message) const;

  // Checks whether the type specified in the type URL matches the given type.
-  // A type is consdiered matching if its full name matches the full name after
+  // A type is considered matching if its full name matches the full name after
  // the last "/" in the type URL.
-  template<typename T>
+  template <typename T>
  bool Is() const {
-    return InternalIs(T::default_instance().GetDescriptor());
+    return InternalIs(T::FullMessageName());
  }

 private:
-  bool InternalIs(const Descriptor* message) const;
+  bool InternalPackFrom(Arena* arena, const MessageLite& message,
+                        StringPiece type_url_prefix,
+                        StringPiece type_name);
+  bool InternalUnpackTo(StringPiece type_name,
+                        MessageLite* message) const;
+  bool InternalIs(StringPiece type_name) const;

  UrlType* type_url_;
  ValueType* value_;
@@ -84,15 +124,22 @@ class LIBPROTOBUF_EXPORT AnyMetadata {
  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(AnyMetadata);
 };

-extern const char kAnyFullTypeName[];          // "google.protobuf.Any".
-extern const char kTypeGoogleApisComPrefix[];  // "type.googleapis.com/".
-extern const char kTypeGoogleProdComPrefix[];  // "type.googleprod.com/".
-
 // Get the proto type name from Any::type_url value. For example, passing
 // "type.googleapis.com/rpc.QueryOrigin" will return "rpc.QueryOrigin" in
 // *full_type_name. Returns false if the type_url does not have a "/"
 // in the type url separating the full type name.
-bool ParseAnyTypeUrl(const string& type_url, string* full_type_name);
+//
+// NOTE: this function is available publicly as:
+//   google::protobuf::Any()  // static method on the generated message type.
+bool ParseAnyTypeUrl(StringPiece type_url, std::string* full_type_name);
+
+// Get the proto type name and prefix from Any::type_url value. For example,
+// passing "type.googleapis.com/rpc.QueryOrigin" will return
+// "type.googleapis.com/" in *url_prefix and "rpc.QueryOrigin" in
+// *full_type_name. Returns false if the type_url does not have a "/" in the
+// type url separating the full type name.
+bool ParseAnyTypeUrl(StringPiece type_url, std::string* url_prefix,
+                     std::string* full_type_name);

 // See if message is of type google.protobuf.Any, if so, return the descriptors
 // for "type_url" and "value" fields.
@@ -102,6 +149,8 @@ bool GetAnyFieldDescriptors(const Message& message,

 }  // namespace internal
 }  // namespace protobuf
-
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
 #endif  // GOOGLE_PROTOBUF_ANY_H__
@@ -1,440 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: google/protobuf/any.proto
-
-#include <google/protobuf/any.pb.h>
-
-#include <algorithm>
-
-#include <google/protobuf/stubs/common.h>
-#include <google/protobuf/stubs/port.h>
-#include <google/protobuf/stubs/once.h>
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/wire_format_lite_inl.h>
-#include <google/protobuf/descriptor.h>
-#include <google/protobuf/generated_message_reflection.h>
-#include <google/protobuf/reflection_ops.h>
-#include <google/protobuf/wire_format.h>
-// This is a temporary google only hack
-#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-#include "third_party/protobuf/version.h"
-#endif
-// @@protoc_insertion_point(includes)
-namespace google {
-namespace protobuf {
-class AnyDefaultTypeInternal {
- public:
-  ::google::protobuf::internal::ExplicitlyConstructed<Any>
-      _instance;
-} _Any_default_instance_;
-}  // namespace protobuf
-}  // namespace google
-namespace protobuf_google_2fprotobuf_2fany_2eproto {
-void InitDefaultsAnyImpl() {
-  GOOGLE_PROTOBUF_VERIFY_VERSION;
-
-#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
-#else
-  ::google::protobuf::internal::InitProtobufDefaults();
-#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-  {
-    void* ptr = &::google::protobuf::_Any_default_instance_;
-    new (ptr) ::google::protobuf::Any();
-    ::google::protobuf::internal::OnShutdownDestroyMessage(ptr);
-  }
-  ::google::protobuf::Any::InitAsDefaultInstance();
-}
-
-void InitDefaultsAny() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsAnyImpl);
-}
-
-::google::protobuf::Metadata file_level_metadata[1];
-
-const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  ~0u,  // no _has_bits_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, _internal_metadata_),
-  ~0u,  // no _extensions_
-  ~0u,  // no _oneof_case_
-  ~0u,  // no _weak_field_map_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, type_url_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, value_),
-};
-static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  { 0, -1, sizeof(::google::protobuf::Any)},
-};
-
-static ::google::protobuf::Message const * const file_default_instances[] = {
-  reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Any_default_instance_),
-};
-
-void protobuf_AssignDescriptors() {
-  AddDescriptors();
-  ::google::protobuf::MessageFactory* factory = NULL;
-  AssignDescriptors(
-      "google/protobuf/any.proto", schemas, file_default_instances, TableStruct::offsets, factory,
-      file_level_metadata, NULL, NULL);
-}
-
-void protobuf_AssignDescriptorsOnce() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &protobuf_AssignDescriptors);
-}
-
-void protobuf_RegisterTypes(const ::std::string&) GOOGLE_PROTOBUF_ATTRIBUTE_COLD;
-void protobuf_RegisterTypes(const ::std::string&) {
-  protobuf_AssignDescriptorsOnce();
-  ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1);
-}
-
-void AddDescriptorsImpl() {
-  InitDefaults();
-  static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-      "\n\031google/protobuf/any.proto\022\017google.prot"
-      "obuf\"&\n\003Any\022\020\n\010type_url\030\001 \001(\t\022\r\n\005value\030\002"
-      " \001(\014Bo\n\023com.google.protobufB\010AnyProtoP\001Z"
-      "%github.com/golang/protobuf/ptypes/any\242\002"
-      "\003GPB\252\002\036Google.Protobuf.WellKnownTypesb\006p"
-      "roto3"
-  };
-  ::google::protobuf::DescriptorPool::InternalAddGeneratedFile(
-      descriptor, 205);
-  ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
-    "google/protobuf/any.proto", &protobuf_RegisterTypes);
-}
-
-void AddDescriptors() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &AddDescriptorsImpl);
-}
-// Force AddDescriptors() to be called at dynamic initialization time.
-struct StaticDescriptorInitializer {
-  StaticDescriptorInitializer() {
-    AddDescriptors();
-  }
-} static_descriptor_initializer;
-}  // namespace protobuf_google_2fprotobuf_2fany_2eproto
-namespace google {
-namespace protobuf {
-
-// ===================================================================
-
-void Any::InitAsDefaultInstance() {
-}
-void Any::PackFrom(const ::google::protobuf::Message& message) {
-  _any_metadata_.PackFrom(message);
-}
-
-void Any::PackFrom(const ::google::protobuf::Message& message,
-                           const ::std::string& type_url_prefix) {
-  _any_metadata_.PackFrom(message, type_url_prefix);
-}
-
-bool Any::UnpackTo(::google::protobuf::Message* message) const {
-  return _any_metadata_.UnpackTo(message);
-}
-
-#if !defined(_MSC_VER) || _MSC_VER >= 1900
-const int Any::kTypeUrlFieldNumber;
-const int Any::kValueFieldNumber;
-#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
-
-Any::Any()
-  : ::google::protobuf::Message(), _internal_metadata_(NULL), _any_metadata_(&type_url_, &value_) {
-  if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) {
-    ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAny();
-  }
-  SharedCtor();
-  // @@protoc_insertion_point(constructor:google.protobuf.Any)
-}
-Any::Any(const Any& from)
-  : ::google::protobuf::Message(),
-      _internal_metadata_(NULL),
-      _cached_size_(0),
-      _any_metadata_(&type_url_, &value_) {
-  _internal_metadata_.MergeFrom(from._internal_metadata_);
-  type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-  if (from.type_url().size() > 0) {
-    type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.type_url_);
-  }
-  value_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-  if (from.value().size() > 0) {
-    value_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.value_);
-  }
-  // @@protoc_insertion_point(copy_constructor:google.protobuf.Any)
-}
-
-void Any::SharedCtor() {
-  type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-  value_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-  _cached_size_ = 0;
-}
-
-Any::~Any() {
-  // @@protoc_insertion_point(destructor:google.protobuf.Any)
-  SharedDtor();
-}
-
-void Any::SharedDtor() {
-  type_url_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-  value_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-
-void Any::SetCachedSize(int size) const {
-  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
-  _cached_size_ = size;
-  GOOGLE_SAFE_CONCURRENT_WRITES_END();
-}
-const ::google::protobuf::Descriptor* Any::descriptor() {
-  ::protobuf_google_2fprotobuf_2fany_2eproto::protobuf_AssignDescriptorsOnce();
-  return ::protobuf_google_2fprotobuf_2fany_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
-}
-
-const Any& Any::default_instance() {
-  ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAny();
-  return *internal_default_instance();
-}
-
-Any* Any::New(::google::protobuf::Arena* arena) const {
-  Any* n = new Any;
-  if (arena != NULL) {
-    arena->Own(n);
-  }
-  return n;
-}
-
-void Any::Clear() {
-// @@protoc_insertion_point(message_clear_start:google.protobuf.Any)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  // Prevent compiler warnings about cached_has_bits being unused
-  (void) cached_has_bits;
-
-  type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-  value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-  _internal_metadata_.Clear();
-}
-
-bool Any::MergePartialFromCodedStream(
-    ::google::protobuf::io::CodedInputStream* input) {
-#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
-  ::google::protobuf::uint32 tag;
-  // @@protoc_insertion_point(parse_start:google.protobuf.Any)
-  for (;;) {
-    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u);
-    tag = p.first;
-    if (!p.second) goto handle_unusual;
-    switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
-      // string type_url = 1;
-      case 1: {
-        if (static_cast< ::google::protobuf::uint8>(tag) ==
-            static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
-                input, this->mutable_type_url()));
-          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
-            this->type_url().data(), static_cast<int>(this->type_url().length()),
-            ::google::protobuf::internal::WireFormatLite::PARSE,
-            "google.protobuf.Any.type_url"));
-        } else {
-          goto handle_unusual;
-        }
-        break;
-      }
-
-      // bytes value = 2;
-      case 2: {
-        if (static_cast< ::google::protobuf::uint8>(tag) ==
-            static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadBytes(
-                input, this->mutable_value()));
-        } else {
-          goto handle_unusual;
-        }
-        break;
-      }
-
-      default: {
-      handle_unusual:
-        if (tag == 0) {
-          goto success;
-        }
-        DO_(::google::protobuf::internal::WireFormat::SkipField(
-              input, tag, _internal_metadata_.mutable_unknown_fields()));
-        break;
-      }
-    }
-  }
-success:
-  // @@protoc_insertion_point(parse_success:google.protobuf.Any)
-  return true;
-failure:
-  // @@protoc_insertion_point(parse_failure:google.protobuf.Any)
-  return false;
-#undef DO_
-}
-
-void Any::SerializeWithCachedSizes(
-    ::google::protobuf::io::CodedOutputStream* output) const {
-  // @@protoc_insertion_point(serialize_start:google.protobuf.Any)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-  // string type_url = 1;
-  if (this->type_url().size() > 0) {
-    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
-      this->type_url().data(), static_cast<int>(this->type_url().length()),
-      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
-      "google.protobuf.Any.type_url");
-    ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased(
-      1, this->type_url(), output);
-  }
-
-  // bytes value = 2;
-  if (this->value().size() > 0) {
-    ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased(
-      2, this->value(), output);
-  }
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    ::google::protobuf::internal::WireFormat::SerializeUnknownFields(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()), output);
-  }
-  // @@protoc_insertion_point(serialize_end:google.protobuf.Any)
-}
-
-::google::protobuf::uint8* Any::InternalSerializeWithCachedSizesToArray(
-    bool deterministic, ::google::protobuf::uint8* target) const {
-  (void)deterministic; // Unused
-  // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.Any)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-  // string type_url = 1;
-  if (this->type_url().size() > 0) {
-    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
-      this->type_url().data(), static_cast<int>(this->type_url().length()),
-      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
-      "google.protobuf.Any.type_url");
-    target =
-      ::google::protobuf::internal::WireFormatLite::WriteStringToArray(
-        1, this->type_url(), target);
-  }
-
-  // bytes value = 2;
-  if (this->value().size() > 0) {
-    target =
-      ::google::protobuf::internal::WireFormatLite::WriteBytesToArray(
-        2, this->value(), target);
-  }
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()), target);
-  }
-  // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Any)
-  return target;
-}
-
-size_t Any::ByteSizeLong() const {
-// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Any)
-  size_t total_size = 0;
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    total_size +=
-      ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()));
-  }
-  // string type_url = 1;
-  if (this->type_url().size() > 0) {
-    total_size += 1 +
-      ::google::protobuf::internal::WireFormatLite::StringSize(
-        this->type_url());
-  }
-
-  // bytes value = 2;
-  if (this->value().size() > 0) {
-    total_size += 1 +
-      ::google::protobuf::internal::WireFormatLite::BytesSize(
-        this->value());
-  }
-
-  int cached_size = ::google::protobuf::internal::ToCachedSize(total_size);
-  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
-  _cached_size_ = cached_size;
-  GOOGLE_SAFE_CONCURRENT_WRITES_END();
-  return total_size;
-}
-
-void Any::MergeFrom(const ::google::protobuf::Message& from) {
-// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.Any)
-  GOOGLE_DCHECK_NE(&from, this);
-  const Any* source =
-      ::google::protobuf::internal::DynamicCastToGenerated<const Any>(
-          &from);
-  if (source == NULL) {
-  // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.Any)
-    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
-  } else {
-  // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.Any)
-    MergeFrom(*source);
-  }
-}
-
-void Any::MergeFrom(const Any& from) {
-// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.Any)
-  GOOGLE_DCHECK_NE(&from, this);
-  _internal_metadata_.MergeFrom(from._internal_metadata_);
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-  if (from.type_url().size() > 0) {
-
-    type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.type_url_);
-  }
-  if (from.value().size() > 0) {
-
-    value_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.value_);
-  }
-}
-
-void Any::CopyFrom(const ::google::protobuf::Message& from) {
-// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.Any)
-  if (&from == this) return;
-  Clear();
-  MergeFrom(from);
-}
-
-void Any::CopyFrom(const Any& from) {
-// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.Any)
-  if (&from == this) return;
-  Clear();
-  MergeFrom(from);
-}
-
-bool Any::IsInitialized() const {
-  return true;
-}
-
-void Any::Swap(Any* other) {
-  if (other == this) return;
-  InternalSwap(other);
-}
-void Any::InternalSwap(Any* other) {
-  using std::swap;
-  type_url_.Swap(&other->type_url_);
-  value_.Swap(&other->value_);
-  _internal_metadata_.Swap(&other->_internal_metadata_);
-  swap(_cached_size_, other->_cached_size_);
-}
-
-::google::protobuf::Metadata Any::GetMetadata() const {
-  protobuf_google_2fprotobuf_2fany_2eproto::protobuf_AssignDescriptorsOnce();
-  return ::protobuf_google_2fprotobuf_2fany_2eproto::file_level_metadata[kIndexInFileMessages];
-}
-
-
-// @@protoc_insertion_point(namespace_scope)
-}  // namespace protobuf
-}  // namespace google
-
-// @@protoc_insertion_point(global_scope)
@@ -1,323 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: google/protobuf/any.proto
-
-#ifndef PROTOBUF_google_2fprotobuf_2fany_2eproto__INCLUDED
-#define PROTOBUF_google_2fprotobuf_2fany_2eproto__INCLUDED
-
-#include <string>
-
-#include <google/protobuf/stubs/common.h>
-
-#if GOOGLE_PROTOBUF_VERSION < 3005000
-#error This file was generated by a newer version of protoc which is
-#error incompatible with your Protocol Buffer headers.  Please update
-#error your headers.
-#endif
-#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION
-#error This file was generated by an older version of protoc which is
-#error incompatible with your Protocol Buffer headers.  Please
-#error regenerate this file with a newer version of protoc.
-#endif
-
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/arena.h>
-#include <google/protobuf/arenastring.h>
-#include <google/protobuf/generated_message_table_driven.h>
-#include <google/protobuf/generated_message_util.h>
-#include <google/protobuf/metadata.h>
-#include <google/protobuf/message.h>
-#include <google/protobuf/repeated_field.h>  // IWYU pragma: export
-#include <google/protobuf/extension_set.h>  // IWYU pragma: export
-#include <google/protobuf/unknown_field_set.h>
-#include <google/protobuf/any.h>
-// @@protoc_insertion_point(includes)
-
-namespace protobuf_google_2fprotobuf_2fany_2eproto {
-// Internal implementation detail -- do not use these members.
-struct LIBPROTOBUF_EXPORT TableStruct {
-  static const ::google::protobuf::internal::ParseTableField entries[];
-  static const ::google::protobuf::internal::AuxillaryParseTableField aux[];
-  static const ::google::protobuf::internal::ParseTable schema[1];
-  static const ::google::protobuf::internal::FieldMetadata field_metadata[];
-  static const ::google::protobuf::internal::SerializationTable serialization_table[];
-  static const ::google::protobuf::uint32 offsets[];
-};
-void LIBPROTOBUF_EXPORT AddDescriptors();
-void LIBPROTOBUF_EXPORT InitDefaultsAnyImpl();
-void LIBPROTOBUF_EXPORT InitDefaultsAny();
-inline void LIBPROTOBUF_EXPORT InitDefaults() {
-  InitDefaultsAny();
-}
-}  // namespace protobuf_google_2fprotobuf_2fany_2eproto
-namespace google {
-namespace protobuf {
-class Any;
-class AnyDefaultTypeInternal;
-LIBPROTOBUF_EXPORT extern AnyDefaultTypeInternal _Any_default_instance_;
-}  // namespace protobuf
-}  // namespace google
-namespace google {
-namespace protobuf {
-
-// ===================================================================
-
-class LIBPROTOBUF_EXPORT Any : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Any) */ {
- public:
-  Any();
-  virtual ~Any();
-
-  Any(const Any& from);
-
-  inline Any& operator=(const Any& from) {
-    CopyFrom(from);
-    return *this;
-  }
-  #if LANG_CXX11
-  Any(Any&& from) noexcept
-    : Any() {
-    *this = ::std::move(from);
-  }
-
-  inline Any& operator=(Any&& from) noexcept {
-    if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
-      if (this != &from) InternalSwap(&from);
-    } else {
-      CopyFrom(from);
-    }
-    return *this;
-  }
-  #endif
-  static const ::google::protobuf::Descriptor* descriptor();
-  static const Any& default_instance();
-
-  static void InitAsDefaultInstance();  // FOR INTERNAL USE ONLY
-  static inline const Any* internal_default_instance() {
-    return reinterpret_cast<const Any*>(
-               &_Any_default_instance_);
-  }
-  static PROTOBUF_CONSTEXPR int const kIndexInFileMessages =
-    0;
-
-  // implements Any -----------------------------------------------
-
-  void PackFrom(const ::google::protobuf::Message& message);
-  void PackFrom(const ::google::protobuf::Message& message,
-                const ::std::string& type_url_prefix);
-  bool UnpackTo(::google::protobuf::Message* message) const;
-  template<typename T> bool Is() const {
-    return _any_metadata_.Is<T>();
-  }
-
-  void Swap(Any* other);
-  friend void swap(Any& a, Any& b) {
-    a.Swap(&b);
-  }
-
-  // implements Message ----------------------------------------------
-
-  inline Any* New() const PROTOBUF_FINAL { return New(NULL); }
-
-  Any* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL;
-  void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
-  void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
-  void CopyFrom(const Any& from);
-  void MergeFrom(const Any& from);
-  void Clear() PROTOBUF_FINAL;
-  bool IsInitialized() const PROTOBUF_FINAL;
-
-  size_t ByteSizeLong() const PROTOBUF_FINAL;
-  bool MergePartialFromCodedStream(
-      ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL;
-  void SerializeWithCachedSizes(
-      ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL;
-  ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(
-      bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL;
-  int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; }
-  private:
-  void SharedCtor();
-  void SharedDtor();
-  void SetCachedSize(int size) const PROTOBUF_FINAL;
-  void InternalSwap(Any* other);
-  private:
-  inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
-    return NULL;
-  }
-  inline void* MaybeArenaPtr() const {
-    return NULL;
-  }
-  public:
-
-  ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL;
-
-  // nested types ----------------------------------------------------
-
-  // accessors -------------------------------------------------------
-
-  // string type_url = 1;
-  void clear_type_url();
-  static const int kTypeUrlFieldNumber = 1;
-  const ::std::string& type_url() const;
-  void set_type_url(const ::std::string& value);
-  #if LANG_CXX11
-  void set_type_url(::std::string&& value);
-  #endif
-  void set_type_url(const char* value);
-  void set_type_url(const char* value, size_t size);
-  ::std::string* mutable_type_url();
-  ::std::string* release_type_url();
-  void set_allocated_type_url(::std::string* type_url);
-
-  // bytes value = 2;
-  void clear_value();
-  static const int kValueFieldNumber = 2;
-  const ::std::string& value() const;
-  void set_value(const ::std::string& value);
-  #if LANG_CXX11
-  void set_value(::std::string&& value);
-  #endif
-  void set_value(const char* value);
-  void set_value(const void* value, size_t size);
-  ::std::string* mutable_value();
-  ::std::string* release_value();
-  void set_allocated_value(::std::string* value);
-
-  // @@protoc_insertion_point(class_scope:google.protobuf.Any)
- private:
-
-  ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_;
-  ::google::protobuf::internal::ArenaStringPtr type_url_;
-  ::google::protobuf::internal::ArenaStringPtr value_;
-  mutable int _cached_size_;
-  ::google::protobuf::internal::AnyMetadata _any_metadata_;
-  friend struct ::protobuf_google_2fprotobuf_2fany_2eproto::TableStruct;
-  friend void ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAnyImpl();
-};
-// ===================================================================
-
-
-// ===================================================================
-
-#ifdef __GNUC__
-  #pragma GCC diagnostic push
-  #pragma GCC diagnostic ignored "-Wstrict-aliasing"
-#endif  // __GNUC__
-// Any
-
-// string type_url = 1;
-inline void Any::clear_type_url() {
-  type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-inline const ::std::string& Any::type_url() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Any.type_url)
-  return type_url_.GetNoArena();
-}
-inline void Any::set_type_url(const ::std::string& value) {
-  
-  type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:google.protobuf.Any.type_url)
-}
-#if LANG_CXX11
-inline void Any::set_type_url(::std::string&& value) {
-  
-  type_url_.SetNoArena(
-    &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.type_url)
-}
-#endif
-inline void Any::set_type_url(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
-  
-  type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:google.protobuf.Any.type_url)
-}
-inline void Any::set_type_url(const char* value, size_t size) {
-  
-  type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.type_url)
-}
-inline ::std::string* Any::mutable_type_url() {
-  
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Any.type_url)
-  return type_url_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-inline ::std::string* Any::release_type_url() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Any.type_url)
-  
-  return type_url_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-inline void Any::set_allocated_type_url(::std::string* type_url) {
-  if (type_url != NULL) {
-    
-  } else {
-    
-  }
-  type_url_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), type_url);
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Any.type_url)
-}
-
-// bytes value = 2;
-inline void Any::clear_value() {
-  value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-inline const ::std::string& Any::value() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Any.value)
-  return value_.GetNoArena();
-}
-inline void Any::set_value(const ::std::string& value) {
-  
-  value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:google.protobuf.Any.value)
-}
-#if LANG_CXX11
-inline void Any::set_value(::std::string&& value) {
-  
-  value_.SetNoArena(
-    &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.value)
-}
-#endif
-inline void Any::set_value(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
-  
-  value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:google.protobuf.Any.value)
-}
-inline void Any::set_value(const void* value, size_t size) {
-  
-  value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.value)
-}
-inline ::std::string* Any::mutable_value() {
-  
-  // @@protoc_insertion_point(field_mutable:google.protobuf.Any.value)
-  return value_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-inline ::std::string* Any::release_value() {
-  // @@protoc_insertion_point(field_release:google.protobuf.Any.value)
-  
-  return value_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited());
-}
-inline void Any::set_allocated_value(::std::string* value) {
-  if (value != NULL) {
-    
-  } else {
-    
-  }
-  value_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set_allocated:google.protobuf.Any.value)
-}
-
-#ifdef __GNUC__
-  #pragma GCC diagnostic pop
-#endif  // __GNUC__
-
-// @@protoc_insertion_point(namespace_scope)
-
-}  // namespace protobuf
-}  // namespace google
-
-// @@protoc_insertion_point(global_scope)
-
-#endif  // PROTOBUF_google_2fprotobuf_2fany_2eproto__INCLUDED
@@ -0,0 +1,99 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <google/protobuf/any.h>
+
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <google/protobuf/arenastring.h>
+#include <google/protobuf/generated_message_util.h>
+#include <google/protobuf/stubs/strutil.h>
+
+namespace google {
+namespace protobuf {
+namespace internal {
+
+std::string GetTypeUrl(StringPiece message_name,
+                       StringPiece type_url_prefix) {
+  if (!type_url_prefix.empty() &&
+      type_url_prefix[type_url_prefix.size() - 1] == '/') {
+    return StrCat(type_url_prefix, message_name);
+  } else {
+    return StrCat(type_url_prefix, "/", message_name);
+  }
+}
+
+const char kAnyFullTypeName[] = "google.protobuf.Any";
+const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/";
+const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/";
+
+bool AnyMetadata::InternalPackFrom(Arena* arena, const MessageLite& message,
+                                   StringPiece type_url_prefix,
+                                   StringPiece type_name) {
+  type_url_->Set(&::google::protobuf::internal::GetEmptyString(),
+                 GetTypeUrl(type_name, type_url_prefix), arena);
+  return message.SerializeToString(
+      value_->Mutable(ArenaStringPtr::EmptyDefault{}, arena));
+}
+
+bool AnyMetadata::InternalUnpackTo(StringPiece type_name,
+                                   MessageLite* message) const {
+  if (!InternalIs(type_name)) {
+    return false;
+  }
+  return message->ParseFromString(value_->Get());
+}
+
+bool AnyMetadata::InternalIs(StringPiece type_name) const {
+  StringPiece type_url = type_url_->Get();
+  return type_url.size() >= type_name.size() + 1 &&
+         type_url[type_url.size() - type_name.size() - 1] == '/' &&
+         HasSuffixString(type_url, type_name);
+}
+
+bool ParseAnyTypeUrl(StringPiece type_url, std::string* url_prefix,
+                     std::string* full_type_name) {
+  size_t pos = type_url.find_last_of('/');
+  if (pos == std::string::npos || pos + 1 == type_url.size()) {
+    return false;
+  }
+  if (url_prefix) {
+    *url_prefix = std::string(type_url.substr(0, pos + 1));
+  }
+  *full_type_name = std::string(type_url.substr(pos + 1));
+  return true;
+}
+
+bool ParseAnyTypeUrl(StringPiece type_url, std::string* full_type_name) {
+  return ParseAnyTypeUrl(type_url, nullptr, full_type_name);
+}
+
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google
@@ -31,355 +31,481 @@
 #include <google/protobuf/arena.h>

 #include <algorithm>
+#include <atomic>
+#include <cstddef>
+#include <cstdint>
 #include <limits>
+#include <typeinfo>

+#include <google/protobuf/arena_impl.h>

+#include <google/protobuf/stubs/mutex.h>
 #ifdef ADDRESS_SANITIZER
 #include <sanitizer/asan_interface.h>
 #endif  // ADDRESS_SANITIZER

-#include <google/protobuf/stubs/port.h>
+#include <google/protobuf/port_def.inc>

 namespace google {
-static const size_t kMinCleanupListElements = 8;
-static const size_t kMaxCleanupListElements = 64;  // 1kB on 64-bit.
-
 namespace protobuf {
 namespace internal {

+static SerialArena::Memory AllocateMemory(const AllocationPolicy* policy_ptr,
+                                          size_t last_size, size_t min_bytes) {
+  AllocationPolicy policy;  // default policy
+  if (policy_ptr) policy = *policy_ptr;
+  size_t size;
+  if (last_size != 0) {
+    // Double the current block size, up to a limit.
+    auto max_size = policy.max_block_size;
+    size = std::min(2 * last_size, max_size);
+  } else {
+    size = policy.start_block_size;
+  }
+  // Verify that min_bytes + kBlockHeaderSize won't overflow.
+  GOOGLE_CHECK_LE(min_bytes,
+           std::numeric_limits<size_t>::max() - SerialArena::kBlockHeaderSize);
+  size = std::max(size, SerialArena::kBlockHeaderSize + min_bytes);

-google::protobuf::internal::SequenceNumber ArenaImpl::lifecycle_id_generator_;
+  void* mem;
+  if (policy.block_alloc == nullptr) {
+    mem = ::operator new(size);
+  } else {
+    mem = policy.block_alloc(size);
+  }
+  return {mem, size};
+}
+
+class GetDeallocator {
+ public:
+  GetDeallocator(const AllocationPolicy* policy, size_t* space_allocated)
+      : dealloc_(policy ? policy->block_dealloc : nullptr),
+        space_allocated_(space_allocated) {}
+
+  void operator()(SerialArena::Memory mem) const {
+#ifdef ADDRESS_SANITIZER
+    // This memory was provided by the underlying allocator as unpoisoned,
+    // so return it in an unpoisoned state.
+    ASAN_UNPOISON_MEMORY_REGION(mem.ptr, mem.size);
+#endif  // ADDRESS_SANITIZER
+    if (dealloc_) {
+      dealloc_(mem.ptr, mem.size);
+    } else {
+#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation)
+      ::operator delete(mem.ptr, mem.size);
+#else
+      ::operator delete(mem.ptr);
+#endif
+    }
+    *space_allocated_ += mem.size;
+  }
+
+ private:
+  void (*dealloc_)(void*, size_t);
+  size_t* space_allocated_;
+};
+
+SerialArena::SerialArena(Block* b, void* owner) : space_allocated_(b->size) {
+  owner_ = owner;
+  head_ = b;
+  ptr_ = b->Pointer(kBlockHeaderSize + ThreadSafeArena::kSerialArenaSize);
+  limit_ = b->Pointer(b->size & static_cast<size_t>(-8));
+}
+
+SerialArena* SerialArena::New(Memory mem, void* owner) {
+  GOOGLE_DCHECK_LE(kBlockHeaderSize + ThreadSafeArena::kSerialArenaSize, mem.size);
+
+  auto b = new (mem.ptr) Block{nullptr, mem.size};
+  return new (b->Pointer(kBlockHeaderSize)) SerialArena(b, owner);
+}
+
+template <typename Deallocator>
+SerialArena::Memory SerialArena::Free(Deallocator deallocator) {
+  Block* b = head_;
+  Memory mem = {b, b->size};
+  while (b->next) {
+    b = b->next;  // We must first advance before deleting this block
+    deallocator(mem);
+    mem = {b, b->size};
+  }
+  return mem;
+}
+
+PROTOBUF_NOINLINE
+std::pair<void*, SerialArena::CleanupNode*>
+SerialArena::AllocateAlignedWithCleanupFallback(
+    size_t n, const AllocationPolicy* policy) {
+  AllocateNewBlock(n + kCleanupSize, policy);
+  return AllocateFromExistingWithCleanupFallback(n);
+}
+
+PROTOBUF_NOINLINE
+void* SerialArena::AllocateAlignedFallback(size_t n,
+                                           const AllocationPolicy* policy) {
+  AllocateNewBlock(n, policy);
+  return AllocateFromExisting(n);
+}
+
+void SerialArena::AllocateNewBlock(size_t n, const AllocationPolicy* policy) {
+  // Sync limit to block
+  head_->start = reinterpret_cast<CleanupNode*>(limit_);
+
+  // Record how much used in this block.
+  space_used_ += ptr_ - head_->Pointer(kBlockHeaderSize);
+
+  auto mem = AllocateMemory(policy, head_->size, n);
+  // We don't want to emit an expensive RMW instruction that requires
+  // exclusive access to a cacheline. Hence we write it in terms of a
+  // regular add.
+  auto relaxed = std::memory_order_relaxed;
+  space_allocated_.store(space_allocated_.load(relaxed) + mem.size, relaxed);
+  head_ = new (mem.ptr) Block{head_, mem.size};
+  ptr_ = head_->Pointer(kBlockHeaderSize);
+  limit_ = head_->Pointer(head_->size);
+
+#ifdef ADDRESS_SANITIZER
+  ASAN_POISON_MEMORY_REGION(ptr_, limit_ - ptr_);
+#endif  // ADDRESS_SANITIZER
+}
+
+uint64_t SerialArena::SpaceUsed() const {
+  uint64_t space_used = ptr_ - head_->Pointer(kBlockHeaderSize);
+  space_used += space_used_;
+  // Remove the overhead of the SerialArena itself.
+  space_used -= ThreadSafeArena::kSerialArenaSize;
+  return space_used;
+}
+
+void SerialArena::CleanupList() {
+  Block* b = head_;
+  b->start = reinterpret_cast<CleanupNode*>(limit_);
+  do {
+    auto* limit = reinterpret_cast<CleanupNode*>(
+        b->Pointer(b->size & static_cast<size_t>(-8)));
+    auto it = b->start;
+    auto num = limit - it;
+    if (num > 0) {
+      for (; it < limit; it++) {
+        it->cleanup(it->elem);
+      }
+    }
+    b = b->next;
+  } while (b);
+}
+
+
+ThreadSafeArena::CacheAlignedLifecycleIdGenerator
+    ThreadSafeArena::lifecycle_id_generator_;
 #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
-ArenaImpl::ThreadCache& ArenaImpl::thread_cache() {
+ThreadSafeArena::ThreadCache& ThreadSafeArena::thread_cache() {
  static internal::ThreadLocalStorage<ThreadCache>* thread_cache_ =
      new internal::ThreadLocalStorage<ThreadCache>();
  return *thread_cache_->Get();
 }
 #elif defined(PROTOBUF_USE_DLLS)
-ArenaImpl::ThreadCache& ArenaImpl::thread_cache() {
-  static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_ = { -1, NULL };
+ThreadSafeArena::ThreadCache& ThreadSafeArena::thread_cache() {
+  static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_ = {
+      0, static_cast<LifecycleIdAtomic>(-1), nullptr};
  return thread_cache_;
 }
 #else
-GOOGLE_THREAD_LOCAL ArenaImpl::ThreadCache ArenaImpl::thread_cache_ = {-1, NULL};
+PROTOBUF_THREAD_LOCAL ThreadSafeArena::ThreadCache
+    ThreadSafeArena::thread_cache_ = {0, static_cast<LifecycleIdAtomic>(-1),
+                                      nullptr};
 #endif

-void ArenaImpl::Init() {
-  lifecycle_id_ = lifecycle_id_generator_.GetNext();
-  google::protobuf::internal::NoBarrier_Store(&hint_, 0);
-  google::protobuf::internal::NoBarrier_Store(&threads_, 0);
-
-  if (initial_block_) {
-    // Thread which calls Init() owns the first block. This allows the
-    // single-threaded case to allocate on the first block without having to
-    // perform atomic operations.
-    InitBlock(initial_block_, &thread_cache(), options_.initial_block_size);
-    ThreadInfo* info = NewThreadInfo(initial_block_);
-    info->next = NULL;
-    google::protobuf::internal::NoBarrier_Store(&threads_,
-                                  reinterpret_cast<google::protobuf::internal::AtomicWord>(info));
-    google::protobuf::internal::NoBarrier_Store(&space_allocated_,
-                                  options_.initial_block_size);
-    CacheBlock(initial_block_);
-  } else {
-    google::protobuf::internal::NoBarrier_Store(&space_allocated_, 0);
-  }
-}
-
-ArenaImpl::~ArenaImpl() {
-  // Have to do this in a first pass, because some of the destructors might
-  // refer to memory in other blocks.
-  CleanupList();
-  FreeBlocks();
-}
-
-uint64 ArenaImpl::Reset() {
-  // Have to do this in a first pass, because some of the destructors might
-  // refer to memory in other blocks.
-  CleanupList();
-  uint64 space_allocated = FreeBlocks();
+void ThreadSafeArena::InitializeFrom(void* mem, size_t size) {
+  GOOGLE_DCHECK_EQ(reinterpret_cast<uintptr_t>(mem) & 7, 0u);
+  GOOGLE_DCHECK(!AllocPolicy());  // Reset should call InitializeWithPolicy instead.
  Init();

-  return space_allocated;
+  // Ignore initial block if it is too small.
+  if (mem != nullptr && size >= kBlockHeaderSize + kSerialArenaSize) {
+    alloc_policy_.set_is_user_owned_initial_block(true);
+    SetInitialBlock(mem, size);
+  }
 }

-ArenaImpl::Block* ArenaImpl::NewBlock(void* me, Block* my_last_block,
-                                      size_t min_bytes) {
-  size_t size;
-  if (my_last_block != NULL) {
-    // Double the current block size, up to a limit.
-    size = std::min(2 * my_last_block->size, options_.max_block_size);
+void ThreadSafeArena::InitializeWithPolicy(void* mem, size_t size,
+                                           AllocationPolicy policy) {
+#ifndef NDEBUG
+  const uint64_t old_alloc_policy = alloc_policy_.get_raw();
+  // If there was a policy (e.g., in Reset()), make sure flags were preserved.
+#define GOOGLE_DCHECK_POLICY_FLAGS_() \
+  if (old_alloc_policy > 3)    \
+    GOOGLE_CHECK_EQ(old_alloc_policy & 3, alloc_policy_.get_raw() & 3)
+#else
+#define GOOGLE_DCHECK_POLICY_FLAGS_()
+#endif  // NDEBUG
+
+  if (policy.IsDefault()) {
+    // Legacy code doesn't use the API above, but provides the initial block
+    // through ArenaOptions. I suspect most do not touch the allocation
+    // policy parameters.
+    InitializeFrom(mem, size);
+    GOOGLE_DCHECK_POLICY_FLAGS_();
+    return;
+  }
+  GOOGLE_DCHECK_EQ(reinterpret_cast<uintptr_t>(mem) & 7, 0u);
+  Init();
+
+  // Ignore initial block if it is too small. We include an optional
+  // AllocationPolicy in this check, so that this can be allocated on the
+  // first block.
+  constexpr size_t kAPSize = internal::AlignUpTo8(sizeof(AllocationPolicy));
+  constexpr size_t kMinimumSize = kBlockHeaderSize + kSerialArenaSize + kAPSize;
+
+  // The value for alloc_policy_ stores whether or not allocations should be
+  // recorded.
+  alloc_policy_.set_should_record_allocs(
+      policy.metrics_collector != nullptr &&
+      policy.metrics_collector->RecordAllocs());
+  // Make sure we have an initial block to store the AllocationPolicy.
+  if (mem != nullptr && size >= kMinimumSize) {
+    alloc_policy_.set_is_user_owned_initial_block(true);
  } else {
-    size = options_.start_block_size;
+    auto tmp = AllocateMemory(&policy, 0, kMinimumSize);
+    mem = tmp.ptr;
+    size = tmp.size;
  }
-  // Verify that min_bytes + kHeaderSize won't overflow.
-  GOOGLE_CHECK_LE(min_bytes, std::numeric_limits<size_t>::max() - kHeaderSize);
-  size = std::max(size, kHeaderSize + min_bytes);
+  SetInitialBlock(mem, size);

-  Block* b = reinterpret_cast<Block*>(options_.block_alloc(size));
-  InitBlock(b, me, size);
-  google::protobuf::internal::NoBarrier_AtomicIncrement(&space_allocated_, size);
-  return b;
-}
-
-void ArenaImpl::InitBlock(Block* b, void *me, size_t size) {
-  b->pos = kHeaderSize;
-  b->size = size;
-  b->owner = me;
-  b->next = NULL;
-#ifdef ADDRESS_SANITIZER
-  // Poison the rest of the block for ASAN. It was unpoisoned by the underlying
-  // malloc but it's not yet usable until we return it as part of an allocation.
-  ASAN_POISON_MEMORY_REGION(
-      reinterpret_cast<char*>(b) + b->pos, b->size - b->pos);
-#endif  // ADDRESS_SANITIZER
-}
-
-ArenaImpl::CleanupChunk* ArenaImpl::ExpandCleanupList(CleanupChunk* cleanup,
-                                                      Block* b) {
-  size_t size = cleanup ? cleanup->size * 2 : kMinCleanupListElements;
-  size = std::min(size, kMaxCleanupListElements);
-  size_t bytes = internal::AlignUpTo8(CleanupChunk::SizeOf(size));
-  if (b->avail() < bytes) {
-    b = GetBlock(bytes);
+  auto sa = threads_.load(std::memory_order_relaxed);
+  // We ensured enough space so this cannot fail.
+  void* p;
+  if (!sa || !sa->MaybeAllocateAligned(kAPSize, &p)) {
+    GOOGLE_LOG(FATAL) << "MaybeAllocateAligned cannot fail here.";
+    return;
  }
-  CleanupChunk* list =
-      reinterpret_cast<CleanupChunk*>(AllocFromBlock(b, bytes));
-  list->next = b->thread_info->cleanup;
-  list->size = size;
-  list->len = 0;
-  b->thread_info->cleanup = list;
-  return list;
+  new (p) AllocationPolicy{policy};
+  // Low bits store flags, so they mustn't be overwritten.
+  GOOGLE_DCHECK_EQ(0, reinterpret_cast<uintptr_t>(p) & 3);
+  alloc_policy_.set_policy(reinterpret_cast<AllocationPolicy*>(p));
+  GOOGLE_DCHECK_POLICY_FLAGS_();
+
+#undef GOOGLE_DCHECK_POLICY_FLAGS_
 }

-inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
-void ArenaImpl::AddCleanupInBlock(
-    Block* b, void* elem, void (*func)(void*)) {
-  CleanupChunk* cleanup = b->thread_info->cleanup;
-  if (cleanup == NULL || cleanup->len == cleanup->size) {
-    cleanup = ExpandCleanupList(cleanup, b);
+void ThreadSafeArena::Init() {
+#ifndef NDEBUG
+  const bool was_message_owned = IsMessageOwned();
+#endif  // NDEBUG
+  ThreadCache& tc = thread_cache();
+  auto id = tc.next_lifecycle_id;
+  // We increment lifecycle_id's by multiples of two so we can use bit 0 as
+  // a tag.
+  constexpr uint64_t kDelta = 2;
+  constexpr uint64_t kInc = ThreadCache::kPerThreadIds * kDelta;
+  if (PROTOBUF_PREDICT_FALSE((id & (kInc - 1)) == 0)) {
+    constexpr auto relaxed = std::memory_order_relaxed;
+    // On platforms that don't support uint64_t atomics we can certainly not
+    // afford to increment by large intervals and expect uniqueness due to
+    // wrapping, hence we only add by 1.
+    id = lifecycle_id_generator_.id.fetch_add(1, relaxed) * kInc;
+  }
+  tc.next_lifecycle_id = id + kDelta;
+  // Message ownership is stored in tag_and_id_, and is set in the constructor.
+  // This flag bit must be preserved, even across calls to Reset().
+  tag_and_id_ = id | (tag_and_id_ & kMessageOwnedArena);
+  hint_.store(nullptr, std::memory_order_relaxed);
+  threads_.store(nullptr, std::memory_order_relaxed);
+#ifndef NDEBUG
+  GOOGLE_CHECK_EQ(was_message_owned, IsMessageOwned());
+#endif  // NDEBUG
+}
+
+void ThreadSafeArena::SetInitialBlock(void* mem, size_t size) {
+  SerialArena* serial = SerialArena::New({mem, size}, &thread_cache());
+  serial->set_next(NULL);
+  threads_.store(serial, std::memory_order_relaxed);
+  CacheSerialArena(serial);
+}
+
+ThreadSafeArena::~ThreadSafeArena() {
+  // Have to do this in a first pass, because some of the destructors might
+  // refer to memory in other blocks.
+  CleanupList();
+
+  size_t space_allocated = 0;
+  auto mem = Free(&space_allocated);
+
+  // Policy is about to get deleted.
+  auto* p = alloc_policy_.get();
+  ArenaMetricsCollector* collector = p ? p->metrics_collector : nullptr;
+
+  if (alloc_policy_.is_user_owned_initial_block()) {
+    space_allocated += mem.size;
+  } else {
+    GetDeallocator(alloc_policy_.get(), &space_allocated)(mem);
  }

-  CleanupNode* node = &cleanup->nodes[cleanup->len++];
-
-  node->elem = elem;
-  node->cleanup = func;
+  if (collector) collector->OnDestroy(space_allocated);
 }

-void ArenaImpl::AddCleanup(void* elem, void (*cleanup)(void*)) {
-  return AddCleanupInBlock(GetBlock(0), elem, cleanup);
-}
-
-void* ArenaImpl::AllocateAligned(size_t n) {
-  GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n);  // Must be already aligned.
-
-  return AllocFromBlock(GetBlock(n), n);
-}
-
-void* ArenaImpl::AllocateAlignedAndAddCleanup(size_t n,
-                                              void (*cleanup)(void*)) {
-  GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n);  // Must be already aligned.
-
-  Block* b = GetBlock(n);
-  void* mem = AllocFromBlock(b, n);
-  AddCleanupInBlock(b, mem, cleanup);
+SerialArena::Memory ThreadSafeArena::Free(size_t* space_allocated) {
+  SerialArena::Memory mem = {nullptr, 0};
+  auto deallocator = GetDeallocator(alloc_policy_.get(), space_allocated);
+  PerSerialArena([deallocator, &mem](SerialArena* a) {
+    if (mem.ptr) deallocator(mem);
+    mem = a->Free(deallocator);
+  });
  return mem;
 }

-inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
-ArenaImpl::Block* ArenaImpl::GetBlock(size_t n) {
-  Block* my_block = NULL;
+uint64_t ThreadSafeArena::Reset() {
+  // Have to do this in a first pass, because some of the destructors might
+  // refer to memory in other blocks.
+  CleanupList();

-  // If this thread already owns a block in this arena then try to use that.
-  // This fast path optimizes the case where multiple threads allocate from the
-  // same arena.
-  ThreadCache* tc = &thread_cache();
-  if (tc->last_lifecycle_id_seen == lifecycle_id_) {
-    my_block = tc->last_block_used_;
-    if (my_block->avail() >= n) {
-      return my_block;
+  // Discard all blocks except the special block (if present).
+  size_t space_allocated = 0;
+  auto mem = Free(&space_allocated);
+
+  AllocationPolicy* policy = alloc_policy_.get();
+  if (policy) {
+    auto saved_policy = *policy;
+    if (alloc_policy_.is_user_owned_initial_block()) {
+      space_allocated += mem.size;
+    } else {
+      GetDeallocator(alloc_policy_.get(), &space_allocated)(mem);
+      mem.ptr = nullptr;
+      mem.size = 0;
    }
-  }
-
-  // Check whether we own the last accessed block on this arena.
-  // This fast path optimizes the case where a single thread uses multiple
-  // arenas.
-  Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&hint_));
-  if (b != NULL && b->owner == tc) {
-    my_block = b;
-    if (my_block->avail() >= n) {
-      return my_block;
+    ArenaMetricsCollector* collector = saved_policy.metrics_collector;
+    if (collector) collector->OnReset(space_allocated);
+    InitializeWithPolicy(mem.ptr, mem.size, saved_policy);
+  } else {
+    GOOGLE_DCHECK(!alloc_policy_.should_record_allocs());
+    // Nullptr policy
+    if (alloc_policy_.is_user_owned_initial_block()) {
+      space_allocated += mem.size;
+      InitializeFrom(mem.ptr, mem.size);
+    } else {
+      GetDeallocator(alloc_policy_.get(), &space_allocated)(mem);
+      Init();
    }
  }
-  return GetBlockSlow(tc, my_block, n);
-}
-
-inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
-void* ArenaImpl::AllocFromBlock(Block* b, size_t n) {
-  GOOGLE_DCHECK_EQ(internal::AlignUpTo8(b->pos), b->pos);  // Must be already aligned.
-  GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n);  // Must be already aligned.
-  GOOGLE_DCHECK_GE(b->avail(), n);
-  size_t p = b->pos;
-  b->pos = p + n;
-#ifdef ADDRESS_SANITIZER
-  ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b) + p, n);
-#endif  // ADDRESS_SANITIZER
-  return reinterpret_cast<char*>(b) + p;
-}
-
-ArenaImpl::Block* ArenaImpl::GetBlockSlow(void* me, Block* my_full_block,
-                                          size_t n) {
-  ThreadInfo* info =
-      my_full_block ? my_full_block->thread_info : GetThreadInfo(me, n);
-  GOOGLE_DCHECK(info != NULL);
-  Block* b = info->head;
-  if (b->avail() < n) {
-    Block* new_b = NewBlock(me, b, n);
-    new_b->thread_info = info;
-    new_b->next = b;
-    info->head = new_b;
-    b = new_b;
-  }
-  CacheBlock(b);
-  return b;
-}
-
-uint64 ArenaImpl::SpaceAllocated() const {
-  return google::protobuf::internal::NoBarrier_Load(&space_allocated_);
-}
-
-uint64 ArenaImpl::SpaceUsed() const {
-  ThreadInfo* info =
-      reinterpret_cast<ThreadInfo*>(google::protobuf::internal::Acquire_Load(&threads_));
-  uint64 space_used = 0;
-
-  for ( ; info; info = info->next) {
-    // Remove the overhead of the ThreadInfo itself.
-    space_used -= sizeof(ThreadInfo);
-    for (Block* b = info->head; b; b = b->next) {
-      space_used += (b->pos - kHeaderSize);
-    }
-  }
-
-  return space_used;
-}
-
-uint64 ArenaImpl::FreeBlocks() {
-  uint64 space_allocated = 0;
-  // By omitting an Acquire barrier we ensure that any user code that doesn't
-  // properly synchronize Reset() or the destructor will throw a TSAN warning.
-  ThreadInfo* info =
-      reinterpret_cast<ThreadInfo*>(google::protobuf::internal::NoBarrier_Load(&threads_));
-
-  while (info) {
-    // This is inside the block we are freeing, so we need to read it now.
-    ThreadInfo* next_info = info->next;
-    for (Block* b = info->head; b; ) {
-      // This is inside the block we are freeing, so we need to read it now.
-      Block* next_block = b->next;
-      space_allocated += (b->size);
-
-#ifdef ADDRESS_SANITIZER
-      // This memory was provided by the underlying allocator as unpoisoned, so
-      // return it in an unpoisoned state.
-      ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b), b->size);
-#endif  // ADDRESS_SANITIZER
-
-      if (b != initial_block_) {
-        options_.block_dealloc(b, b->size);
-      }
-
-      b = next_block;
-    }
-    info = next_info;
-  }

  return space_allocated;
 }

-void ArenaImpl::CleanupList() {
-  // By omitting an Acquire barrier we ensure that any user code that doesn't
-  // properly synchronize Reset() or the destructor will throw a TSAN warning.
-  ThreadInfo* info =
-      reinterpret_cast<ThreadInfo*>(google::protobuf::internal::NoBarrier_Load(&threads_));
-
-  for ( ; info; info = info->next) {
-    CleanupChunk* list = info->cleanup;
-    while (list) {
-      size_t n = list->len;
-      CleanupNode* node = &list->nodes[list->len - 1];
-      for (size_t i = 0; i < n; i++, node--) {
-        node->cleanup(node->elem);
-      }
-      list = list->next;
-    }
+std::pair<void*, SerialArena::CleanupNode*>
+ThreadSafeArena::AllocateAlignedWithCleanup(size_t n,
+                                            const std::type_info* type) {
+  SerialArena* arena;
+  if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() &&
+                            GetSerialArenaFast(&arena))) {
+    return arena->AllocateAlignedWithCleanup(n, alloc_policy_.get());
+  } else {
+    return AllocateAlignedWithCleanupFallback(n, type);
  }
 }

-ArenaImpl::ThreadInfo* ArenaImpl::NewThreadInfo(Block* b) {
-  GOOGLE_DCHECK(FindThreadInfo(b->owner) == NULL);
-  ThreadInfo* info =
-      reinterpret_cast<ThreadInfo*>(AllocFromBlock(b, sizeof(ThreadInfo)));
-  b->thread_info = info;
-  info->owner = b->owner;
-  info->head = b;
-  info->cleanup = NULL;
-  return info;
+void ThreadSafeArena::AddCleanup(void* elem, void (*cleanup)(void*)) {
+  SerialArena* arena;
+  if (PROTOBUF_PREDICT_FALSE(!GetSerialArenaFast(&arena))) {
+    arena = GetSerialArenaFallback(&thread_cache());
+  }
+  arena->AddCleanup(elem, cleanup, AllocPolicy());
 }

-ArenaImpl::ThreadInfo* ArenaImpl::FindThreadInfo(void* me) {
-  ThreadInfo* info =
-      reinterpret_cast<ThreadInfo*>(google::protobuf::internal::Acquire_Load(&threads_));
-  for ( ; info; info = info->next) {
-    if (info->owner == me) {
-      return info;
+PROTOBUF_NOINLINE
+void* ThreadSafeArena::AllocateAlignedFallback(size_t n,
+                                               const std::type_info* type) {
+  if (alloc_policy_.should_record_allocs()) {
+    alloc_policy_.RecordAlloc(type, n);
+    SerialArena* arena;
+    if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFast(&arena))) {
+      return arena->AllocateAligned(n, alloc_policy_.get());
+    }
+  }
+  return GetSerialArenaFallback(&thread_cache())
+      ->AllocateAligned(n, alloc_policy_.get());
+}
+
+PROTOBUF_NOINLINE
+std::pair<void*, SerialArena::CleanupNode*>
+ThreadSafeArena::AllocateAlignedWithCleanupFallback(
+    size_t n, const std::type_info* type) {
+  if (alloc_policy_.should_record_allocs()) {
+    alloc_policy_.RecordAlloc(type, n);
+    SerialArena* arena;
+    if (GetSerialArenaFast(&arena)) {
+      return arena->AllocateAlignedWithCleanup(n, alloc_policy_.get());
+    }
+  }
+  return GetSerialArenaFallback(&thread_cache())
+      ->AllocateAlignedWithCleanup(n, alloc_policy_.get());
+}
+
+uint64_t ThreadSafeArena::SpaceAllocated() const {
+  SerialArena* serial = threads_.load(std::memory_order_acquire);
+  uint64_t res = 0;
+  for (; serial; serial = serial->next()) {
+    res += serial->SpaceAllocated();
+  }
+  return res;
+}
+
+uint64_t ThreadSafeArena::SpaceUsed() const {
+  SerialArena* serial = threads_.load(std::memory_order_acquire);
+  uint64_t space_used = 0;
+  for (; serial; serial = serial->next()) {
+    space_used += serial->SpaceUsed();
+  }
+  return space_used - (alloc_policy_.get() ? sizeof(AllocationPolicy) : 0);
+}
+
+void ThreadSafeArena::CleanupList() {
+  PerSerialArena([](SerialArena* a) { a->CleanupList(); });
+}
+
+PROTOBUF_NOINLINE
+SerialArena* ThreadSafeArena::GetSerialArenaFallback(void* me) {
+  // Look for this SerialArena in our linked list.
+  SerialArena* serial = threads_.load(std::memory_order_acquire);
+  for (; serial; serial = serial->next()) {
+    if (serial->owner() == me) {
+      break;
    }
  }

-  return NULL;
-}
+  if (!serial) {
+    // This thread doesn't have any SerialArena, which also means it doesn't
+    // have any blocks yet.  So we'll allocate its first block now.
+    serial = SerialArena::New(
+        AllocateMemory(alloc_policy_.get(), 0, kSerialArenaSize), me);

-ArenaImpl::ThreadInfo* ArenaImpl::GetThreadInfo(void* me, size_t n) {
-  ThreadInfo* info = FindThreadInfo(me);
-
-  if (!info) {
-    // This thread doesn't have any ThreadInfo, which also means it doesn't have
-    // any blocks yet.  So we'll allocate its first block now.
-    Block* b = NewBlock(me, NULL, sizeof(ThreadInfo) + n);
-    info = NewThreadInfo(b);
-
-    google::protobuf::internal::AtomicWord head;
+    SerialArena* head = threads_.load(std::memory_order_relaxed);
    do {
-      head = google::protobuf::internal::NoBarrier_Load(&threads_);
-      info->next = reinterpret_cast<ThreadInfo*>(head);
-    } while (google::protobuf::internal::Release_CompareAndSwap(
-                 &threads_, head, reinterpret_cast<google::protobuf::internal::AtomicWord>(info)) != head);
+      serial->set_next(head);
+    } while (!threads_.compare_exchange_weak(
+        head, serial, std::memory_order_release, std::memory_order_relaxed));
  }

-  return info;
+  CacheSerialArena(serial);
+  return serial;
 }

 }  // namespace internal

-void Arena::CallDestructorHooks() {
-  uint64 space_allocated = impl_.SpaceAllocated();
-  // Call the reset hook
-  if (on_arena_reset_ != NULL) {
-    on_arena_reset_(this, hooks_cookie_, space_allocated);
-  }
-
-  // Call the destruction hook
-  if (on_arena_destruction_ != NULL) {
-    on_arena_destruction_(this, hooks_cookie_, space_allocated);
-  }
+PROTOBUF_FUNC_ALIGN(32)
+void* Arena::AllocateAlignedNoHook(size_t n) {
+  return impl_.AllocateAligned(n, nullptr);
 }

-void Arena::OnArenaAllocation(const std::type_info* allocated_type,
-                              size_t n) const {
-  if (on_arena_allocation_ != NULL) {
-    on_arena_allocation_(allocated_type, n, hooks_cookie_);
-  }
+PROTOBUF_FUNC_ALIGN(32)
+void* Arena::AllocateAlignedWithHook(size_t n, const std::type_info* type) {
+  return impl_.AllocateAligned(n, type);
+}
+
+PROTOBUF_FUNC_ALIGN(32)
+std::pair<void*, internal::SerialArena::CleanupNode*>
+Arena::AllocateAlignedWithCleanup(size_t n, const std::type_info* type) {
+  return impl_.AllocateAlignedWithCleanup(n, type);
 }

 }  // namespace protobuf
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
@@ -33,85 +33,168 @@
 #ifndef GOOGLE_PROTOBUF_ARENA_IMPL_H__
 #define GOOGLE_PROTOBUF_ARENA_IMPL_H__

+#include <atomic>
 #include <limits>
+#include <typeinfo>

-#include <google/protobuf/stubs/atomic_sequence_num.h>
-#include <google/protobuf/stubs/atomicops.h>
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/stubs/logging.h>
-#include <google/protobuf/stubs/mutex.h>
-#include <google/protobuf/stubs/type_traits.h>

-#include <google/protobuf/stubs/port.h>
+#ifdef ADDRESS_SANITIZER
+#include <sanitizer/asan_interface.h>
+#endif  // ADDRESS_SANITIZER
+
+#include <google/protobuf/port_def.inc>
+

 namespace google {
-
 namespace protobuf {
 namespace internal {

-inline size_t AlignUpTo8(size_t n) {
+inline constexpr size_t AlignUpTo8(size_t n) {
  // Align n to next multiple of 8 (from Hacker's Delight, Chapter 3.)
-  return (n + 7) & -8;
+  return (n + 7) & static_cast<size_t>(-8);
 }

-// This class provides the core Arena memory allocation library. Different
-// implementations only need to implement the public interface below.
-// Arena is not a template type as that would only be useful if all protos
-// in turn would be templates, which will/cannot happen. However separating
-// the memory allocation part from the cruft of the API users expect we can
-// use #ifdef the select the best implementation based on hardware / OS.
-class LIBPROTOBUF_EXPORT ArenaImpl {
+using LifecycleIdAtomic = uint64_t;
+
+// MetricsCollector collects stats for a particular arena.
+class PROTOBUF_EXPORT ArenaMetricsCollector {
 public:
-  struct Options {
-    size_t start_block_size;
-    size_t max_block_size;
-    char* initial_block;
-    size_t initial_block_size;
-    void* (*block_alloc)(size_t);
-    void (*block_dealloc)(void*, size_t);
+  ArenaMetricsCollector(bool record_allocs) : record_allocs_(record_allocs) {}

-    template <typename O>
-    explicit Options(const O& options)
-      : start_block_size(options.start_block_size),
-        max_block_size(options.max_block_size),
-        initial_block(options.initial_block),
-        initial_block_size(options.initial_block_size),
-        block_alloc(options.block_alloc),
-        block_dealloc(options.block_dealloc) {}
-  };
+  // Invoked when the arena is about to be destroyed. This method will
+  // typically finalize any metric collection and delete the collector.
+  // space_allocated is the space used by the arena.
+  virtual void OnDestroy(uint64_t space_allocated) = 0;

-  template <typename O>
-  explicit ArenaImpl(const O& options) : options_(options) {
-    if (options_.initial_block != NULL && options_.initial_block_size > 0) {
-      GOOGLE_CHECK_GE(options_.initial_block_size, sizeof(Block))
-          << ": Initial block size too small for header.";
-      initial_block_ = reinterpret_cast<Block*>(options_.initial_block);
-    } else {
-      initial_block_ = NULL;
-    }
+  // OnReset() is called when the associated arena is reset.
+  // space_allocated is the space used by the arena just before the reset.
+  virtual void OnReset(uint64_t space_allocated) = 0;

-    Init();
+  // OnAlloc is called when an allocation happens.
+  // type_info is promised to be static - its lifetime extends to
+  // match program's lifetime (It is given by typeid operator).
+  // Note: typeid(void) will be passed as allocated_type every time we
+  // intentionally want to avoid monitoring an allocation. (i.e. internal
+  // allocations for managing the arena)
+  virtual void OnAlloc(const std::type_info* allocated_type,
+                       uint64_t alloc_size) = 0;
+
+  // Does OnAlloc() need to be called?  If false, metric collection overhead
+  // will be reduced since we will not do extra work per allocation.
+  bool RecordAllocs() { return record_allocs_; }
+
+ protected:
+  // This class is destructed by the call to OnDestroy().
+  ~ArenaMetricsCollector() = default;
+  const bool record_allocs_;
+};
+
+struct AllocationPolicy {
+  static constexpr size_t kDefaultStartBlockSize = 256;
+  static constexpr size_t kDefaultMaxBlockSize = 8192;
+
+  size_t start_block_size = kDefaultStartBlockSize;
+  size_t max_block_size = kDefaultMaxBlockSize;
+  void* (*block_alloc)(size_t) = nullptr;
+  void (*block_dealloc)(void*, size_t) = nullptr;
+  ArenaMetricsCollector* metrics_collector = nullptr;
+
+  bool IsDefault() const {
+    return start_block_size == kDefaultMaxBlockSize &&
+           max_block_size == kDefaultMaxBlockSize && block_alloc == nullptr &&
+           block_dealloc == nullptr && metrics_collector == nullptr;
+  }
+};
+
+// Tagged pointer to an AllocationPolicy.
+class TaggedAllocationPolicyPtr {
+ public:
+  constexpr TaggedAllocationPolicyPtr() : policy_(0) {}
+
+  explicit TaggedAllocationPolicyPtr(AllocationPolicy* policy)
+      : policy_(reinterpret_cast<uintptr_t>(policy)) {}
+
+  void set_policy(AllocationPolicy* policy) {
+    auto bits = policy_ & kTagsMask;
+    policy_ = reinterpret_cast<uintptr_t>(policy) | bits;
  }

-  // Destructor deletes all owned heap allocated objects, and destructs objects
-  // that have non-trivial destructors, except for proto2 message objects whose
-  // destructors can be skipped. Also, frees all blocks except the initial block
-  // if it was passed in.
-  ~ArenaImpl();
+  AllocationPolicy* get() {
+    return reinterpret_cast<AllocationPolicy*>(policy_ & kPtrMask);
+  }
+  const AllocationPolicy* get() const {
+    return reinterpret_cast<const AllocationPolicy*>(policy_ & kPtrMask);
+  }

-  uint64 Reset();
+  AllocationPolicy& operator*() { return *get(); }
+  const AllocationPolicy& operator*() const { return *get(); }

-  uint64 SpaceAllocated() const;
-  uint64 SpaceUsed() const;
+  AllocationPolicy* operator->() { return get(); }
+  const AllocationPolicy* operator->() const { return get(); }

-  void* AllocateAligned(size_t n);
+  bool is_user_owned_initial_block() const {
+    return static_cast<bool>(get_mask<kUserOwnedInitialBlock>());
+  }
+  void set_is_user_owned_initial_block(bool v) {
+    set_mask<kUserOwnedInitialBlock>(v);
+  }

-  void* AllocateAlignedAndAddCleanup(size_t n, void (*cleanup)(void*));
+  bool should_record_allocs() const {
+    return static_cast<bool>(get_mask<kRecordAllocs>());
+  }
+  void set_should_record_allocs(bool v) { set_mask<kRecordAllocs>(v); }

-  // Add object pointer and cleanup function pointer to the list.
-  void AddCleanup(void* elem, void (*cleanup)(void*));
+  uintptr_t get_raw() const { return policy_; }
+
+  inline void RecordAlloc(const std::type_info* allocated_type,
+                          size_t n) const {
+    get()->metrics_collector->OnAlloc(allocated_type, n);
+  }

 private:
+  enum : uintptr_t {
+    kUserOwnedInitialBlock = 1,
+    kRecordAllocs = 2,
+  };
+
+  static constexpr uintptr_t kTagsMask = 7;
+  static constexpr uintptr_t kPtrMask = ~kTagsMask;
+
+  template <uintptr_t kMask>
+  uintptr_t get_mask() const {
+    return policy_ & kMask;
+  }
+  template <uintptr_t kMask>
+  void set_mask(bool v) {
+    if (v) {
+      policy_ |= kMask;
+    } else {
+      policy_ &= ~kMask;
+    }
+  }
+  uintptr_t policy_;
+};
+
+// A simple arena allocator. Calls to allocate functions must be properly
+// serialized by the caller, hence this class cannot be used as a general
+// purpose allocator in a multi-threaded program. It serves as a building block
+// for ThreadSafeArena, which provides a thread-safe arena allocator.
+//
+// This class manages
+// 1) Arena bump allocation + owning memory blocks.
+// 2) Maintaining a cleanup list.
+// It delagetes the actual memory allocation back to ThreadSafeArena, which
+// contains the information on block growth policy and backing memory allocation
+// used.
+class PROTOBUF_EXPORT SerialArena {
+ public:
+  struct Memory {
+    void* ptr;
+    size_t size;
+  };
+
  // Node contains the ptr of the object to be cleaned up and the associated
  // cleanup function ptr.
  struct CleanupNode {
@@ -119,124 +202,361 @@ class LIBPROTOBUF_EXPORT ArenaImpl {
    void (*cleanup)(void*);  // Function pointer to the destructor or deleter.
  };

-  // Cleanup uses a chunked linked list, to reduce pointer chasing.
-  struct CleanupChunk {
-    static size_t SizeOf(size_t i) {
-      return sizeof(CleanupChunk) + (sizeof(CleanupNode) * (i - 1));
+  void CleanupList();
+  uint64_t SpaceAllocated() const {
+    return space_allocated_.load(std::memory_order_relaxed);
+  }
+  uint64_t SpaceUsed() const;
+
+  bool HasSpace(size_t n) { return n <= static_cast<size_t>(limit_ - ptr_); }
+
+  void* AllocateAligned(size_t n, const AllocationPolicy* policy) {
+    GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n);  // Must be already aligned.
+    GOOGLE_DCHECK_GE(limit_, ptr_);
+    if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) {
+      return AllocateAlignedFallback(n, policy);
    }
-    size_t len;            // Number of elements currently present.
-    size_t size;           // Total elements in the list.
-    CleanupChunk* next;    // Next node in the list.
-    CleanupNode nodes[1];  // True length is |size|.
-  };
+    return AllocateFromExisting(n);
+  }

-  struct Block;
+ private:
+  void* AllocateFromExisting(size_t n) {
+    void* ret = ptr_;
+    ptr_ += n;
+#ifdef ADDRESS_SANITIZER
+    ASAN_UNPOISON_MEMORY_REGION(ret, n);
+#endif  // ADDRESS_SANITIZER
+    return ret;
+  }

-  // Tracks per-thread info.  ThreadInfos are kept in a linked list.
-  struct ThreadInfo {
-    void *owner;             // &ThreadCache of this thread;
-    Block* head;             // Head of linked list of blocks.
-    CleanupChunk* cleanup;   // Head of cleanup list.
-    ThreadInfo* next;        // Next ThreadInfo in this linked list.
-  };
+ public:
+  // Allocate space if the current region provides enough space.
+  bool MaybeAllocateAligned(size_t n, void** out) {
+    GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n);  // Must be already aligned.
+    GOOGLE_DCHECK_GE(limit_, ptr_);
+    if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) return false;
+    *out = AllocateFromExisting(n);
+    return true;
+  }
+
+  std::pair<void*, CleanupNode*> AllocateAlignedWithCleanup(
+      size_t n, const AllocationPolicy* policy) {
+    GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n);  // Must be already aligned.
+    if (PROTOBUF_PREDICT_FALSE(!HasSpace(n + kCleanupSize))) {
+      return AllocateAlignedWithCleanupFallback(n, policy);
+    }
+    return AllocateFromExistingWithCleanupFallback(n);
+  }
+
+ private:
+  std::pair<void*, CleanupNode*> AllocateFromExistingWithCleanupFallback(
+      size_t n) {
+    void* ret = ptr_;
+    ptr_ += n;
+    limit_ -= kCleanupSize;
+#ifdef ADDRESS_SANITIZER
+    ASAN_UNPOISON_MEMORY_REGION(ret, n);
+    ASAN_UNPOISON_MEMORY_REGION(limit_, kCleanupSize);
+#endif  // ADDRESS_SANITIZER
+    return CreatePair(ret, reinterpret_cast<CleanupNode*>(limit_));
+  }
+
+ public:
+  void AddCleanup(void* elem, void (*cleanup)(void*),
+                  const AllocationPolicy* policy) {
+    auto res = AllocateAlignedWithCleanup(0, policy);
+    res.second->elem = elem;
+    res.second->cleanup = cleanup;
+  }
+
+  void* owner() const { return owner_; }
+  SerialArena* next() const { return next_; }
+  void set_next(SerialArena* next) { next_ = next; }
+
+ private:
+  friend class ThreadSafeArena;
+  friend class ArenaBenchmark;
+
+  // Creates a new SerialArena inside mem using the remaining memory as for
+  // future allocations.
+  static SerialArena* New(SerialArena::Memory mem, void* owner);
+  // Free SerialArena returning the memory passed in to New
+  template <typename Deallocator>
+  Memory Free(Deallocator deallocator);

  // Blocks are variable length malloc-ed objects.  The following structure
  // describes the common header for all blocks.
  struct Block {
-    void* owner;              // &ThreadCache of thread that owns this block.
-    ThreadInfo* thread_info;  // ThreadInfo of thread that owns this block.
-    Block* next;              // Next block in arena (may have different owner)
-    // ((char*) &block) + pos is next available byte. It is always
-    // aligned at a multiple of 8 bytes.
-    size_t pos;
-    size_t size;  // total size of the block.
-    GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
-    size_t avail() const { return size - pos; }
+    Block(Block* next, size_t size) : next(next), size(size), start(nullptr) {}
+
+    char* Pointer(size_t n) {
+      GOOGLE_DCHECK(n <= size);
+      return reinterpret_cast<char*>(this) + n;
+    }
+
+    Block* const next;
+    const size_t size;
+    CleanupNode* start;
    // data follows
  };

-  struct ThreadCache {
+  void* owner_;            // &ThreadCache of this thread;
+  Block* head_;            // Head of linked list of blocks.
+  SerialArena* next_;      // Next SerialArena in this linked list.
+  size_t space_used_ = 0;  // Necessary for metrics.
+  std::atomic<size_t> space_allocated_;
+
+  // Next pointer to allocate from.  Always 8-byte aligned.  Points inside
+  // head_ (and head_->pos will always be non-canonical).  We keep these
+  // here to reduce indirection.
+  char* ptr_;
+  char* limit_;
+
+  // Constructor is private as only New() should be used.
+  inline SerialArena(Block* b, void* owner);
+  void* AllocateAlignedFallback(size_t n, const AllocationPolicy* policy);
+  std::pair<void*, CleanupNode*> AllocateAlignedWithCleanupFallback(
+      size_t n, const AllocationPolicy* policy);
+  void AllocateNewBlock(size_t n, const AllocationPolicy* policy);
+
+  std::pair<void*, CleanupNode*> CreatePair(void* ptr, CleanupNode* node) {
+    return {ptr, node};
+  }
+
+ public:
+  static constexpr size_t kBlockHeaderSize = AlignUpTo8(sizeof(Block));
+  static constexpr size_t kCleanupSize = AlignUpTo8(sizeof(CleanupNode));
+};
+
+// Tag type used to invoke the constructor of message-owned arena.
+// Only message-owned arenas use this constructor for creation.
+// Such constructors are internal implementation details of the library.
+struct MessageOwned {
+  explicit MessageOwned() = default;
+};
+
+// This class provides the core Arena memory allocation library. Different
+// implementations only need to implement the public interface below.
+// Arena is not a template type as that would only be useful if all protos
+// in turn would be templates, which will/cannot happen. However separating
+// the memory allocation part from the cruft of the API users expect we can
+// use #ifdef the select the best implementation based on hardware / OS.
+class PROTOBUF_EXPORT ThreadSafeArena {
+ public:
+  ThreadSafeArena() { Init(); }
+
+  // Constructor solely used by message-owned arena.
+  ThreadSafeArena(internal::MessageOwned) : tag_and_id_(kMessageOwnedArena) {
+    Init();
+  }
+
+  ThreadSafeArena(char* mem, size_t size) { InitializeFrom(mem, size); }
+
+  explicit ThreadSafeArena(void* mem, size_t size,
+                           const AllocationPolicy& policy) {
+    InitializeWithPolicy(mem, size, policy);
+  }
+
+  // Destructor deletes all owned heap allocated objects, and destructs objects
+  // that have non-trivial destructors, except for proto2 message objects whose
+  // destructors can be skipped. Also, frees all blocks except the initial block
+  // if it was passed in.
+  ~ThreadSafeArena();
+
+  uint64_t Reset();
+
+  uint64_t SpaceAllocated() const;
+  uint64_t SpaceUsed() const;
+
+  void* AllocateAligned(size_t n, const std::type_info* type) {
+    SerialArena* arena;
+    if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() &&
+                              GetSerialArenaFast(&arena))) {
+      return arena->AllocateAligned(n, AllocPolicy());
+    } else {
+      return AllocateAlignedFallback(n, type);
+    }
+  }
+
+  // This function allocates n bytes if the common happy case is true and
+  // returns true. Otherwise does nothing and returns false. This strange
+  // semantics is necessary to allow callers to program functions that only
+  // have fallback function calls in tail position. This substantially improves
+  // code for the happy path.
+  PROTOBUF_NDEBUG_INLINE bool MaybeAllocateAligned(size_t n, void** out) {
+    SerialArena* a;
+    if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() &&
+                              GetSerialArenaFromThreadCache(&a))) {
+      return a->MaybeAllocateAligned(n, out);
+    }
+    return false;
+  }
+
+  std::pair<void*, SerialArena::CleanupNode*> AllocateAlignedWithCleanup(
+      size_t n, const std::type_info* type);
+
+  // Add object pointer and cleanup function pointer to the list.
+  void AddCleanup(void* elem, void (*cleanup)(void*));
+
+  // Checks whether this arena is message-owned.
+  PROTOBUF_ALWAYS_INLINE bool IsMessageOwned() const {
+    return tag_and_id_ & kMessageOwnedArena;
+  }
+
+ private:
+  // Unique for each arena. Changes on Reset().
+  uint64_t tag_and_id_ = 0;
+  // The LSB of tag_and_id_ indicates if the arena is message-owned.
+  enum : uint64_t { kMessageOwnedArena = 1 };
+
+  TaggedAllocationPolicyPtr alloc_policy_;  // Tagged pointer to AllocPolicy.
+
+  // Pointer to a linked list of SerialArena.
+  std::atomic<SerialArena*> threads_;
+  std::atomic<SerialArena*> hint_;  // Fast thread-local block access
+
+  const AllocationPolicy* AllocPolicy() const { return alloc_policy_.get(); }
+  void InitializeFrom(void* mem, size_t size);
+  void InitializeWithPolicy(void* mem, size_t size, AllocationPolicy policy);
+  void* AllocateAlignedFallback(size_t n, const std::type_info* type);
+  std::pair<void*, SerialArena::CleanupNode*>
+  AllocateAlignedWithCleanupFallback(size_t n, const std::type_info* type);
+
+  void Init();
+  void SetInitialBlock(void* mem, size_t size);
+
+  // Delete or Destruct all objects owned by the arena.
+  void CleanupList();
+
+  inline uint64_t LifeCycleId() const {
+    return tag_and_id_ & ~kMessageOwnedArena;
+  }
+
+  inline void CacheSerialArena(SerialArena* serial) {
+    thread_cache().last_serial_arena = serial;
+    thread_cache().last_lifecycle_id_seen = tag_and_id_;
+    // TODO(haberman): evaluate whether we would gain efficiency by getting rid
+    // of hint_.  It's the only write we do to ThreadSafeArena in the allocation
+    // path, which will dirty the cache line.
+
+    hint_.store(serial, std::memory_order_release);
+  }
+
+  PROTOBUF_NDEBUG_INLINE bool GetSerialArenaFast(SerialArena** arena) {
+    if (GetSerialArenaFromThreadCache(arena)) return true;
+
+    // Check whether we own the last accessed SerialArena on this arena.  This
+    // fast path optimizes the case where a single thread uses multiple arenas.
+    ThreadCache* tc = &thread_cache();
+    SerialArena* serial = hint_.load(std::memory_order_acquire);
+    if (PROTOBUF_PREDICT_TRUE(serial != NULL && serial->owner() == tc)) {
+      *arena = serial;
+      return true;
+    }
+    return false;
+  }
+
+  PROTOBUF_NDEBUG_INLINE bool GetSerialArenaFromThreadCache(
+      SerialArena** arena) {
+    // If this thread already owns a block in this arena then try to use that.
+    // This fast path optimizes the case where multiple threads allocate from
+    // the same arena.
+    ThreadCache* tc = &thread_cache();
+    if (PROTOBUF_PREDICT_TRUE(tc->last_lifecycle_id_seen == tag_and_id_)) {
+      *arena = tc->last_serial_arena;
+      return true;
+    }
+    return false;
+  }
+  SerialArena* GetSerialArenaFallback(void* me);
+
+  template <typename Functor>
+  void PerSerialArena(Functor fn) {
+    // By omitting an Acquire barrier we ensure that any user code that doesn't
+    // properly synchronize Reset() or the destructor will throw a TSAN warning.
+    SerialArena* serial = threads_.load(std::memory_order_relaxed);
+
+    for (; serial; serial = serial->next()) fn(serial);
+  }
+
+  // Releases all memory except the first block which it returns. The first
+  // block might be owned by the user and thus need some extra checks before
+  // deleting.
+  SerialArena::Memory Free(size_t* space_allocated);
+
+#ifdef _MSC_VER
+#pragma warning(disable : 4324)
+#endif
+  struct alignas(64) ThreadCache {
 #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
    // If we are using the ThreadLocalStorage class to store the ThreadCache,
    // then the ThreadCache's default constructor has to be responsible for
    // initializing it.
-    ThreadCache() : last_lifecycle_id_seen(-1), last_block_used_(NULL) {}
+    ThreadCache()
+        : next_lifecycle_id(0),
+          last_lifecycle_id_seen(-1),
+          last_serial_arena(NULL) {}
 #endif

+    // Number of per-thread lifecycle IDs to reserve. Must be power of two.
+    // To reduce contention on a global atomic, each thread reserves a batch of
+    // IDs.  The following number is calculated based on a stress test with
+    // ~6500 threads all frequently allocating a new arena.
+    static constexpr size_t kPerThreadIds = 256;
+    // Next lifecycle ID available to this thread. We need to reserve a new
+    // batch, if `next_lifecycle_id & (kPerThreadIds - 1) == 0`.
+    uint64_t next_lifecycle_id;
    // The ThreadCache is considered valid as long as this matches the
    // lifecycle_id of the arena being used.
-    int64 last_lifecycle_id_seen;
-    Block* last_block_used_;
+    uint64_t last_lifecycle_id_seen;
+    SerialArena* last_serial_arena;
  };
-  static google::protobuf::internal::SequenceNumber lifecycle_id_generator_;
+
+  // Lifecycle_id can be highly contended variable in a situation of lots of
+  // arena creation. Make sure that other global variables are not sharing the
+  // cacheline.
+#ifdef _MSC_VER
+#pragma warning(disable : 4324)
+#endif
+  struct alignas(64) CacheAlignedLifecycleIdGenerator {
+    std::atomic<LifecycleIdAtomic> id;
+  };
+  static CacheAlignedLifecycleIdGenerator lifecycle_id_generator_;
 #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
-  // Android ndk does not support GOOGLE_THREAD_LOCAL keyword so we use a custom thread
-  // local storage class we implemented.
-  // iOS also does not support the GOOGLE_THREAD_LOCAL keyword.
+  // iOS does not support __thread keyword so we use a custom thread local
+  // storage class we implemented.
  static ThreadCache& thread_cache();
 #elif defined(PROTOBUF_USE_DLLS)
  // Thread local variables cannot be exposed through DLL interface but we can
  // wrap them in static functions.
  static ThreadCache& thread_cache();
 #else
-  static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_;
+  static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_;
  static ThreadCache& thread_cache() { return thread_cache_; }
 #endif

-  void Init();
-
-  // Free all blocks and return the total space used which is the sums of sizes
-  // of the all the allocated blocks.
-  uint64 FreeBlocks();
-
-  void AddCleanupInBlock(Block* b, void* elem, void (*func)(void*));
-  CleanupChunk* ExpandCleanupList(CleanupChunk* cleanup, Block* b);
-  // Delete or Destruct all objects owned by the arena.
-  void CleanupList();
-
-  inline void CacheBlock(Block* block) {
-    thread_cache().last_block_used_ = block;
-    thread_cache().last_lifecycle_id_seen = lifecycle_id_;
-    // TODO(haberman): evaluate whether we would gain efficiency by getting rid
-    // of hint_.  It's the only write we do to ArenaImpl in the allocation path,
-    // which will dirty the cache line.
-    google::protobuf::internal::Release_Store(&hint_, reinterpret_cast<google::protobuf::internal::AtomicWord>(block));
-  }
-
-  google::protobuf::internal::AtomicWord threads_;          // Pointer to a linked list of ThreadInfo.
-  google::protobuf::internal::AtomicWord hint_;             // Fast thread-local block access
-  google::protobuf::internal::AtomicWord space_allocated_;  // Sum of sizes of all allocated blocks.
-
-  Block *initial_block_;     // If non-NULL, points to the block that came from
-                             // user data.
-
-  // Returns a block owned by this thread.
-  Block* GetBlock(size_t n);
-  Block* GetBlockSlow(void* me, Block* my_full_block, size_t n);
-  Block* NewBlock(void* me, Block* my_last_block, size_t min_bytes);
-  void InitBlock(Block* b, void *me, size_t size);
-  static void* AllocFromBlock(Block* b, size_t n);
-  ThreadInfo* NewThreadInfo(Block* b);
-  ThreadInfo* FindThreadInfo(void* me);
-  ThreadInfo* GetThreadInfo(void* me, size_t n);
-
-  int64 lifecycle_id_;  // Unique for each arena. Changes on Reset().
-
-  Options options_;
-
-  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArenaImpl);
+  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ThreadSafeArena);
+  // All protos have pointers back to the arena hence Arena must have
+  // pointer stability.
+  ThreadSafeArena(ThreadSafeArena&&) = delete;
+  ThreadSafeArena& operator=(ThreadSafeArena&&) = delete;

 public:
-  // kHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 to
-  // protect the invariant that pos is always at a multiple of 8.
-  static const size_t kHeaderSize = (sizeof(Block) + 7) & -8;
-#if LANG_CXX11
-  static_assert(kHeaderSize % 8 == 0, "kHeaderSize must be a multiple of 8.");
-#endif
+  // kBlockHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8
+  // to protect the invariant that pos is always at a multiple of 8.
+  static constexpr size_t kBlockHeaderSize = SerialArena::kBlockHeaderSize;
+  static constexpr size_t kSerialArenaSize =
+      (sizeof(SerialArena) + 7) & static_cast<size_t>(-8);
+  static_assert(kBlockHeaderSize % 8 == 0,
+                "kBlockHeaderSize must be a multiple of 8.");
+  static_assert(kSerialArenaSize % 8 == 0,
+                "kSerialArenaSize must be a multiple of 8.");
 };

 }  // namespace internal
 }  // namespace protobuf
-
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
 #endif  // GOOGLE_PROTOBUF_ARENA_IMPL_H__
@@ -28,16 +28,259 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

-// The ArenaString implementation is not included in the open-source release. Do
-// not include this file in the distribution.
-
 #include <google/protobuf/arenastring.h>

+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/parse_context.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/message_lite.h>
+#include <google/protobuf/stubs/mutex.h>
+#include <google/protobuf/stubs/strutil.h>
+#include <google/protobuf/stubs/stl_util.h>
+
+// clang-format off
+#include <google/protobuf/port_def.inc>
+// clang-format on
+
 namespace google {
 namespace protobuf {
 namespace internal {

+const std::string& LazyString::Init() const {
+  static WrappedMutex mu{GOOGLE_PROTOBUF_LINKER_INITIALIZED};
+  mu.Lock();
+  const std::string* res = inited_.load(std::memory_order_acquire);
+  if (res == nullptr) {
+    auto init_value = init_value_;
+    res = ::new (static_cast<void*>(string_buf_))
+        std::string(init_value.ptr, init_value.size);
+    inited_.store(res, std::memory_order_release);
+  }
+  mu.Unlock();
+  return *res;
+}
+
+
+std::string* ArenaStringPtr::SetAndReturnNewString() {
+  std::string* new_string = new std::string();
+  tagged_ptr_.Set(new_string);
+  return new_string;
+}
+
+void ArenaStringPtr::DestroyNoArenaSlowPath() { delete UnsafeMutablePointer(); }
+
+void ArenaStringPtr::Set(const std::string* default_value,
+                         ConstStringParam value, ::google::protobuf::Arena* arena) {
+  if (IsDefault(default_value)) {
+    tagged_ptr_.Set(Arena::Create<std::string>(arena, value));
+  } else {
+    UnsafeMutablePointer()->assign(value.data(), value.length());
+  }
+}
+
+void ArenaStringPtr::Set(const std::string* default_value, std::string&& value,
+                         ::google::protobuf::Arena* arena) {
+  if (IsDefault(default_value)) {
+    if (arena == nullptr) {
+      tagged_ptr_.Set(new std::string(std::move(value)));
+    } else {
+      tagged_ptr_.Set(Arena::Create<std::string>(arena, std::move(value)));
+    }
+  } else if (IsDonatedString()) {
+    std::string* current = tagged_ptr_.Get();
+    auto* s = new (current) std::string(std::move(value));
+    arena->OwnDestructor(s);
+    tagged_ptr_.Set(s);
+  } else /* !IsDonatedString() */ {
+    *UnsafeMutablePointer() = std::move(value);
+  }
+}
+
+void ArenaStringPtr::Set(EmptyDefault, ConstStringParam value,
+                         ::google::protobuf::Arena* arena) {
+  Set(&GetEmptyStringAlreadyInited(), value, arena);
+}
+
+void ArenaStringPtr::Set(EmptyDefault, std::string&& value,
+                         ::google::protobuf::Arena* arena) {
+  Set(&GetEmptyStringAlreadyInited(), std::move(value), arena);
+}
+
+void ArenaStringPtr::Set(NonEmptyDefault, ConstStringParam value,
+                         ::google::protobuf::Arena* arena) {
+  Set(nullptr, value, arena);
+}
+
+void ArenaStringPtr::Set(NonEmptyDefault, std::string&& value,
+                         ::google::protobuf::Arena* arena) {
+  Set(nullptr, std::move(value), arena);
+}
+
+std::string* ArenaStringPtr::Mutable(EmptyDefault, ::google::protobuf::Arena* arena) {
+  if (!IsDonatedString() && !IsDefault(&GetEmptyStringAlreadyInited())) {
+    return UnsafeMutablePointer();
+  } else {
+    return MutableSlow(arena);
+  }
+}
+
+std::string* ArenaStringPtr::Mutable(const LazyString& default_value,
+                                     ::google::protobuf::Arena* arena) {
+  if (!IsDonatedString() && !IsDefault(nullptr)) {
+    return UnsafeMutablePointer();
+  } else {
+    return MutableSlow(arena, default_value);
+  }
+}
+
+std::string* ArenaStringPtr::MutableNoCopy(const std::string* default_value,
+                                           ::google::protobuf::Arena* arena) {
+  if (!IsDonatedString() && !IsDefault(default_value)) {
+    return UnsafeMutablePointer();
+  } else {
+    GOOGLE_DCHECK(IsDefault(default_value));
+    // Allocate empty. The contents are not relevant.
+    std::string* new_string = Arena::Create<std::string>(arena);
+    tagged_ptr_.Set(new_string);
+    return new_string;
+  }
+}
+
+template <typename... Lazy>
+std::string* ArenaStringPtr::MutableSlow(::google::protobuf::Arena* arena,
+                                         const Lazy&... lazy_default) {
+  const std::string* const default_value =
+      sizeof...(Lazy) == 0 ? &GetEmptyStringAlreadyInited() : nullptr;
+  GOOGLE_DCHECK(IsDefault(default_value));
+  std::string* new_string =
+      Arena::Create<std::string>(arena, lazy_default.get()...);
+  tagged_ptr_.Set(new_string);
+  return new_string;
+}
+
+std::string* ArenaStringPtr::Release(const std::string* default_value,
+                                     ::google::protobuf::Arena* arena) {
+  if (IsDefault(default_value)) {
+    return nullptr;
+  } else {
+    return ReleaseNonDefault(default_value, arena);
+  }
+}
+
+std::string* ArenaStringPtr::ReleaseNonDefault(const std::string* default_value,
+                                               ::google::protobuf::Arena* arena) {
+  GOOGLE_DCHECK(!IsDefault(default_value));
+
+  if (!IsDonatedString()) {
+    std::string* released;
+    if (arena != nullptr) {
+      released = new std::string;
+      released->swap(*UnsafeMutablePointer());
+    } else {
+      released = UnsafeMutablePointer();
+    }
+    tagged_ptr_.Set(const_cast<std::string*>(default_value));
+    return released;
+  } else /* IsDonatedString() */ {
+    GOOGLE_DCHECK(arena != nullptr);
+    std::string* released = new std::string(Get());
+    tagged_ptr_.Set(const_cast<std::string*>(default_value));
+    return released;
+  }
+}
+
+void ArenaStringPtr::SetAllocated(const std::string* default_value,
+                                  std::string* value, ::google::protobuf::Arena* arena) {
+  // Release what we have first.
+  if (arena == nullptr && !IsDefault(default_value)) {
+    delete UnsafeMutablePointer();
+  }
+  if (value == nullptr) {
+    tagged_ptr_.Set(const_cast<std::string*>(default_value));
+  } else {
+#ifdef NDEBUG
+    tagged_ptr_.Set(value);
+    if (arena != nullptr) {
+      arena->Own(value);
+    }
+#else
+    // On debug builds, copy the string so the address differs.  delete will
+    // fail if value was a stack-allocated temporary/etc., which would have
+    // failed when arena ran its cleanup list.
+    std::string* new_value = Arena::Create<std::string>(arena, *value);
+    delete value;
+    tagged_ptr_.Set(new_value);
+#endif
+  }
+}
+
+void ArenaStringPtr::Destroy(const std::string* default_value,
+                             ::google::protobuf::Arena* arena) {
+  if (arena == nullptr) {
+    GOOGLE_DCHECK(!IsDonatedString());
+    if (!IsDefault(default_value)) {
+      delete UnsafeMutablePointer();
+    }
+  }
+}
+
+void ArenaStringPtr::Destroy(EmptyDefault, ::google::protobuf::Arena* arena) {
+  Destroy(&GetEmptyStringAlreadyInited(), arena);
+}
+
+void ArenaStringPtr::Destroy(NonEmptyDefault, ::google::protobuf::Arena* arena) {
+  Destroy(nullptr, arena);
+}
+
+void ArenaStringPtr::ClearToEmpty() {
+  if (IsDefault(&GetEmptyStringAlreadyInited())) {
+    // Already set to default -- do nothing.
+  } else {
+    // Unconditionally mask away the tag.
+    //
+    // UpdateDonatedString uses assign when capacity is larger than the new
+    // value, which is trivially true in the donated string case.
+    // const_cast<std::string*>(PtrValue<std::string>())->clear();
+    tagged_ptr_.Get()->clear();
+  }
+}
+
+void ArenaStringPtr::ClearToDefault(const LazyString& default_value,
+                                    ::google::protobuf::Arena* arena) {
+  (void)arena;
+  if (IsDefault(nullptr)) {
+    // Already set to default -- do nothing.
+  } else if (!IsDonatedString()) {
+    UnsafeMutablePointer()->assign(default_value.get());
+  }
+}
+
+inline void SetStrWithHeapBuffer(std::string* str, ArenaStringPtr* s) {
+  TaggedPtr<std::string> res;
+  res.Set(str);
+  s->UnsafeSetTaggedPointer(res);
+}
+
+const char* EpsCopyInputStream::ReadArenaString(const char* ptr,
+                                                ArenaStringPtr* s,
+                                                Arena* arena) {
+  GOOGLE_DCHECK(arena != nullptr);
+
+  int size = ReadSize(&ptr);
+  if (!ptr) return nullptr;
+
+  auto* str = Arena::Create<std::string>(arena);
+  ptr = ReadString(ptr, size, str);
+  GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+
+  SetStrWithHeapBuffer(str, s);
+
+  return ptr;
+}

 }  // namespace internal
 }  // namespace protobuf
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
@@ -32,307 +32,389 @@
 #define GOOGLE_PROTOBUF_ARENASTRING_H__

 #include <string>
+#include <type_traits>
+#include <utility>

-#include <google/protobuf/arena.h>
-#include <google/protobuf/stubs/common.h>
-#include <google/protobuf/stubs/fastmem.h>
 #include <google/protobuf/stubs/logging.h>
-#include <google/protobuf/stubs/port.h>
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/arena.h>
+#include <google/protobuf/port.h>
+
+#include <google/protobuf/port_def.inc>
+
+#ifdef SWIG
+#error "You cannot SWIG proto headers"
+#endif

-// This is the implementation of arena string fields written for the open-source
-// release. The ArenaStringPtr struct below is an internal implementation class
-// and *should not be used* by user code. It is used to collect string
-// operations together into one place and abstract away the underlying
-// string-field pointer representation, so that (for example) an alternate
-// implementation that knew more about ::std::string's internals could integrate more
-// closely with the arena allocator.

 namespace google {
 namespace protobuf {
 namespace internal {

-struct LIBPROTOBUF_EXPORT ArenaStringPtr {
-  inline void Set(const ::std::string* default_value,
-                  const ::std::string& value, ::google::protobuf::Arena* arena) {
-    if (ptr_ == default_value) {
-      CreateInstance(arena, &value);
-    } else {
-      *ptr_ = value;
-    }
+template <typename T>
+class ExplicitlyConstructed;
+
+class SwapFieldHelper;
+
+// Lazy string instance to support string fields with non-empty default.
+// These are initialized on the first call to .get().
+class PROTOBUF_EXPORT LazyString {
+ public:
+  // We explicitly make LazyString an aggregate so that MSVC can do constant
+  // initialization on it without marking it `constexpr`.
+  // We do not want to use `constexpr` because it makes it harder to have extern
+  // storage for it and causes library bloat.
+  struct InitValue {
+    const char* ptr;
+    size_t size;
+  };
+  // We keep a union of the initialization value and the std::string to save on
+  // space. We don't need the string array after Init() is done.
+  union {
+    mutable InitValue init_value_;
+    alignas(std::string) mutable char string_buf_[sizeof(std::string)];
+  };
+  mutable std::atomic<const std::string*> inited_;
+
+  const std::string& get() const {
+    // This check generates less code than a call-once invocation.
+    auto* res = inited_.load(std::memory_order_acquire);
+    if (PROTOBUF_PREDICT_FALSE(res == nullptr)) return Init();
+    return *res;
  }

-  inline void SetLite(const ::std::string* default_value,
-                      const ::std::string& value,
-                      ::google::protobuf::Arena* arena) {
-    Set(default_value, value, arena);
+ private:
+  // Initialize the string in `string_buf_`, update `inited_` and return it.
+  // We return it here to avoid having to read it again in the inlined code.
+  const std::string& Init() const;
+};
+
+template <typename T>
+class TaggedPtr {
+ public:
+  TaggedPtr() = default;
+  explicit constexpr TaggedPtr(const ExplicitlyConstructed<std::string>* ptr)
+      : ptr_(const_cast<ExplicitlyConstructed<std::string>*>(ptr)) {}
+
+  void SetTagged(T* p) {
+    Set(p);
+    ptr_ = reinterpret_cast<void*>(as_int() | 1);
+  }
+  void Set(T* p) { ptr_ = p; }
+  T* Get() const { return reinterpret_cast<T*>(as_int() & -2); }
+  bool IsTagged() const { return as_int() & 1; }
+
+  // Returned value is only safe to dereference if IsTagged() == false.
+  // It is safe to compare.
+  T* UnsafeGet() const { return static_cast<T*>(ptr_); }
+
+  bool IsNull() { return ptr_ == nullptr; }
+
+ private:
+  uintptr_t as_int() const { return reinterpret_cast<uintptr_t>(ptr_); }
+  void* ptr_;
+};
+
+static_assert(std::is_trivial<TaggedPtr<std::string>>::value,
+              "TaggedPtr must be trivial");
+
+// This class encapsulates a pointer to a std::string with or without a donated
+// buffer, tagged by bottom bit. It is a high-level wrapper that almost directly
+// corresponds to the interface required by string fields in generated
+// code. It replaces the old std::string* pointer in such cases.
+//
+// The object has different but similar code paths for when the default value is
+// the empty string and when it is a non-empty string.
+// The empty string is handled different throughout the library and there is a
+// single global instance of it we can share.
+//
+// For fields with an empty string default value, there are three distinct
+// states:
+//
+// - Pointer set to 'String' tag (LSB is 0), equal to
+//   &GetEmptyStringAlreadyInited(): field is set to its default value. Points
+//   to a true std::string*, but we do not own that std::string* (it's a
+//   globally shared instance).
+//
+// - Pointer set to 'String' tag (LSB is 0), but not equal to the global empty
+//   string: field points to a true std::string* instance that we own. This
+//   instance is either on the heap or on the arena (i.e. registered on
+//   free()/destructor-call list) as appropriate.
+//
+// - Pointer set to 'DonatedString' tag (LSB is 1): points to a std::string
+//   instance with a buffer on the arena (arena is never nullptr in this case).
+//
+// For fields with a non-empty string default value, there are three distinct
+// states:
+//
+// - Pointer set to 'String' tag (LSB is 0), equal to `nullptr`:
+//   Field is in "default" mode and does not point to any actual instance.
+//   Methods that might need to create an instance of the object will pass a
+//   `const LazyString&` for it.
+//
+// - Pointer set to 'String' tag (LSB is 0), but not equal to `nullptr`:
+//   field points to a true std::string* instance that we own. This instance is
+//   either on the heap or on the arena (i.e. registered on
+//   free()/destructor-call list) as appropriate.
+//
+// - Pointer set to 'DonatedString' tag (LSB is 1): points to a std::string
+//   instance with a buffer on the arena (arena is never nullptr in this case).
+//
+// Generated code and reflection code both ensure that ptr_ is never null for
+// fields with an empty default.
+// Because ArenaStringPtr is used in oneof unions, its constructor is a NOP and
+// so the field is always manually initialized via method calls.
+//
+// Side-note: why pass information about the default on every API call? Because
+// we don't want to hold it in a member variable, or else this would go into
+// every proto message instance. This would be a huge waste of space, since the
+// default instance pointer is typically a global (static class field). We want
+// the generated code to be as efficient as possible, and if we take
+// the default value information as a parameter that's in practice taken from a
+// static class field, and compare ptr_ to the default value, we end up with a
+// single "cmp %reg, GLOBAL" in the resulting machine code. (Note that this also
+// requires the String tag to be 0 so we can avoid the mask before comparing.)
+struct PROTOBUF_EXPORT ArenaStringPtr {
+  ArenaStringPtr() = default;
+  explicit constexpr ArenaStringPtr(
+      const ExplicitlyConstructed<std::string>* default_value)
+      : tagged_ptr_(default_value) {}
+
+  // Some methods below are overloaded on a `default_value` and on tags.
+  // The tagged overloads help reduce code size in the callers in generated
+  // code, while the `default_value` overloads are useful from reflection.
+  // By-value empty struct arguments are elided in the ABI.
+  struct EmptyDefault {};
+  struct NonEmptyDefault {};
+
+  void Set(const std::string* default_value, ConstStringParam value,
+           ::google::protobuf::Arena* arena);
+  void Set(const std::string* default_value, std::string&& value,
+           ::google::protobuf::Arena* arena);
+  void Set(EmptyDefault, ConstStringParam value, ::google::protobuf::Arena* arena);
+  void Set(EmptyDefault, std::string&& value, ::google::protobuf::Arena* arena);
+  void Set(NonEmptyDefault, ConstStringParam value, ::google::protobuf::Arena* arena);
+  void Set(NonEmptyDefault, std::string&& value, ::google::protobuf::Arena* arena);
+  template <typename FirstParam>
+  void Set(FirstParam p1, const char* str, ::google::protobuf::Arena* arena) {
+    Set(p1, ConstStringParam(str), arena);
+  }
+  template <typename FirstParam>
+  void Set(FirstParam p1, const char* str, size_t size,
+           ::google::protobuf::Arena* arena) {
+    ConstStringParam sp{str, size};  // for string_view and `const string &`
+    Set(p1, sp, arena);
+  }
+  template <typename FirstParam, typename RefWrappedType>
+  void Set(FirstParam p1,
+           std::reference_wrapper<RefWrappedType> const_string_ref,
+           ::google::protobuf::Arena* arena) {
+    Set(p1, const_string_ref.get(), arena);
+  }
+
+  template <typename FirstParam, typename SecondParam>
+  void SetBytes(FirstParam p1, SecondParam&& p2, ::google::protobuf::Arena* arena) {
+    Set(p1, static_cast<SecondParam&&>(p2), arena);
+  }
+  template <typename FirstParam>
+  void SetBytes(FirstParam p1, const void* str, size_t size,
+                ::google::protobuf::Arena* arena) {
+    // must work whether ConstStringParam is string_view or `const string &`
+    ConstStringParam sp{static_cast<const char*>(str), size};
+    Set(p1, sp, arena);
  }

  // Basic accessors.
-  inline const ::std::string& Get() const { return *ptr_; }
-
-  inline ::std::string* Mutable(const ::std::string* default_value,
-                           ::google::protobuf::Arena* arena) {
-    if (ptr_ == default_value) {
-      CreateInstance(arena, default_value);
-    }
-    return ptr_;
+  PROTOBUF_NDEBUG_INLINE const std::string& Get() const {
+    // Unconditionally mask away the tag.
+    return *tagged_ptr_.Get();
+  }
+  PROTOBUF_NDEBUG_INLINE const std::string* GetPointer() const {
+    // Unconditionally mask away the tag.
+    return tagged_ptr_.Get();
  }

-  // Release returns a ::std::string* instance that is heap-allocated and is not
-  // Own()'d by any arena. If the field was not set, it returns NULL. The caller
-  // retains ownership. Clears this field back to NULL state. Used to implement
-  // release_<field>() methods on generated classes.
-  inline ::std::string* Release(const ::std::string* default_value,
-                           ::google::protobuf::Arena* arena) {
-    if (ptr_ == default_value) {
-      return NULL;
-    }
-    ::std::string* released = NULL;
-    if (arena != NULL) {
-      // ptr_ is owned by the arena.
-      released = new ::std::string;
-      released->swap(*ptr_);
-    } else {
-      released = ptr_;
-    }
-    ptr_ = const_cast< ::std::string* >(default_value);
-    return released;
-  }
+  // For fields with an empty default value.
+  std::string* Mutable(EmptyDefault, ::google::protobuf::Arena* arena);
+  // For fields with a non-empty default value.
+  std::string* Mutable(const LazyString& default_value, ::google::protobuf::Arena* arena);

-  // UnsafeArenaRelease returns a ::std::string*, but it may be arena-owned (i.e.
-  // have its destructor already registered) if arena != NULL. If the field was
-  // not set, this returns NULL. This method clears this field back to NULL
-  // state. Used to implement unsafe_arena_release_<field>() methods on
-  // generated classes.
-  inline ::std::string* UnsafeArenaRelease(const ::std::string* default_value,
-                                      ::google::protobuf::Arena* /* arena */) {
-    if (ptr_ == default_value) {
-      return NULL;
-    }
-    ::std::string* released = ptr_;
-    ptr_ = const_cast< ::std::string* >(default_value);
-    return released;
-  }
+  // Release returns a std::string* instance that is heap-allocated and is not
+  // Own()'d by any arena. If the field is not set, this returns nullptr. The
+  // caller retains ownership. Clears this field back to nullptr state. Used to
+  // implement release_<field>() methods on generated classes.
+  PROTOBUF_NODISCARD std::string* Release(const std::string* default_value,
+                                          ::google::protobuf::Arena* arena);
+  PROTOBUF_NODISCARD std::string* ReleaseNonDefault(
+      const std::string* default_value, ::google::protobuf::Arena* arena);

-  // Takes a string that is heap-allocated, and takes ownership. The string's
-  // destructor is registered with the arena. Used to implement
+  // Takes a std::string that is heap-allocated, and takes ownership. The
+  // std::string's destructor is registered with the arena. Used to implement
  // set_allocated_<field> in generated classes.
-  inline void SetAllocated(const ::std::string* default_value,
-                           ::std::string* value, ::google::protobuf::Arena* arena) {
-    if (arena == NULL && ptr_ != default_value) {
-      Destroy(default_value, arena);
-    }
-    if (value != NULL) {
-      ptr_ = value;
-      if (arena != NULL) {
-        arena->Own(value);
-      }
-    } else {
-      ptr_ = const_cast< ::std::string* >(default_value);
-    }
-  }
-
-  // Takes a string that has lifetime equal to the arena's lifetime. The arena
-  // must be non-null. It is safe only to pass this method a value returned by
-  // UnsafeArenaRelease() on another field of a message in the same arena. Used
-  // to implement unsafe_arena_set_allocated_<field> in generated classes.
-  inline void UnsafeArenaSetAllocated(const ::std::string* default_value,
-                                      ::std::string* value,
-                                      ::google::protobuf::Arena* /* arena */) {
-    if (value != NULL) {
-      ptr_ = value;
-    } else {
-      ptr_ = const_cast< ::std::string* >(default_value);
-    }
-  }
+  void SetAllocated(const std::string* default_value, std::string* value,
+                    ::google::protobuf::Arena* arena);

  // Swaps internal pointers. Arena-safety semantics: this is guarded by the
  // logic in Swap()/UnsafeArenaSwap() at the message level, so this method is
  // 'unsafe' if called directly.
-  GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void Swap(ArenaStringPtr* other) {
-    std::swap(ptr_, other->ptr_);
-  }
+  inline PROTOBUF_NDEBUG_INLINE static void InternalSwap(
+      const std::string* default_value, ArenaStringPtr* rhs, Arena* rhs_arena,
+      ArenaStringPtr* lhs, Arena* lhs_arena);

  // Frees storage (if not on an arena).
-  inline void Destroy(const ::std::string* default_value,
-                      ::google::protobuf::Arena* arena) {
-    if (arena == NULL && ptr_ != default_value) {
-      delete ptr_;
-    }
-  }
+  void Destroy(const std::string* default_value, ::google::protobuf::Arena* arena);
+  void Destroy(EmptyDefault, ::google::protobuf::Arena* arena);
+  void Destroy(NonEmptyDefault, ::google::protobuf::Arena* arena);

-  // Clears content, but keeps allocated string if arena != NULL, to avoid the
-  // overhead of heap operations. After this returns, the content (as seen by
-  // the user) will always be the empty string. Assumes that |default_value|
-  // is an empty string.
-  inline void ClearToEmpty(const ::std::string* default_value,
-                           ::google::protobuf::Arena* /* arena */) {
-    if (ptr_ == default_value) {
-      // Already set to default (which is empty) -- do nothing.
-    } else {
-      ptr_->clear();
-    }
-  }
+  // Clears content, but keeps allocated std::string, to avoid the overhead of
+  // heap operations. After this returns, the content (as seen by the user) will
+  // always be the empty std::string. Assumes that |default_value| is an empty
+  // std::string.
+  void ClearToEmpty();

-  // Clears content, but keeps allocated string if arena != NULL, to avoid the
-  // overhead of heap operations. After this returns, the content (as seen by
-  // the user) will always be equal to |default_value|.
-  inline void ClearToDefault(const ::std::string* default_value,
-                             ::google::protobuf::Arena* /* arena */) {
-    if (ptr_ == default_value) {
-      // Already set to default -- do nothing.
-    } else {
-      // Have another allocated string -- rather than throwing this away and
-      // resetting ptr_ to the canonical default string instance, we just reuse
-      // this instance.
-      *ptr_ = *default_value;
-    }
-  }
+  // Clears content, assuming that the current value is not the empty
+  // string default.
+  void ClearNonDefaultToEmpty();
+
+  // Clears content, but keeps allocated std::string if arena != nullptr, to
+  // avoid the overhead of heap operations. After this returns, the content
+  // (as seen by the user) will always be equal to |default_value|.
+  void ClearToDefault(const LazyString& default_value, ::google::protobuf::Arena* arena);

  // Called from generated code / reflection runtime only. Resets value to point
-  // to a default string pointer, with the semantics that this ArenaStringPtr
-  // does not own the pointed-to memory. Disregards initial value of ptr_ (so
-  // this is the *ONLY* safe method to call after construction or when
-  // reinitializing after becoming the active field in a oneof union).
-  inline void UnsafeSetDefault(const ::std::string* default_value) {
-    // Casting away 'const' is safe here: accessors ensure that ptr_ is only
-    // returned as a const if it is equal to default_value.
-    ptr_ = const_cast< ::std::string* >(default_value);
+  // to a default string pointer, with the semantics that this
+  // ArenaStringPtr does not own the pointed-to memory. Disregards initial value
+  // of ptr_ (so this is the *ONLY* safe method to call after construction or
+  // when reinitializing after becoming the active field in a oneof union).
+  inline void UnsafeSetDefault(const std::string* default_value);
+
+  // Returns a mutable pointer, but doesn't initialize the string to the
+  // default value.
+  std::string* MutableNoArenaNoDefault(const std::string* default_value);
+
+  // Get a mutable pointer with unspecified contents.
+  // Similar to `MutableNoArenaNoDefault`, but also handles the arena case.
+  // If the value was donated, the contents are discarded.
+  std::string* MutableNoCopy(const std::string* default_value,
+                             ::google::protobuf::Arena* arena);
+
+  // Destroy the string. Assumes `arena == nullptr`.
+  void DestroyNoArena(const std::string* default_value);
+
+  // Internal setter used only at parse time to directly set a donated string
+  // value.
+  void UnsafeSetTaggedPointer(TaggedPtr<std::string> value) {
+    tagged_ptr_ = value;
  }
+  // Generated code only! An optimization, in certain cases the generated
+  // code is certain we can obtain a std::string with no default checks and
+  // tag tests.
+  std::string* UnsafeMutablePointer() PROTOBUF_RETURNS_NONNULL;

-  // The 'NoArena' variants of methods below assume arena == NULL and are
-  // optimized to provide very little overhead relative to a raw string pointer
-  // (while still being in-memory compatible with other code that assumes
-  // ArenaStringPtr). Note the invariant that a class instance that has only
-  // ever been mutated by NoArena methods must *only* be in the String state
-  // (i.e., tag bits are not used), *NEVER* ArenaString. This allows all
-  // tagged-pointer manipulations to be avoided.
-  inline void SetNoArena(const ::std::string* default_value,
-                         const ::std::string& value) {
-    if (ptr_ == default_value) {
-      CreateInstanceNoArena(&value);
-    } else {
-      *ptr_ = value;
-    }
-  }
-
-#if LANG_CXX11
-  void SetNoArena(const ::std::string* default_value, ::std::string&& value) {
-    if (IsDefault(default_value)) {
-      ptr_ = new ::std::string(std::move(value));
-    } else {
-      *ptr_ = std::move(value);
-    }
-  }
-#endif
-
-  void AssignWithDefault(const ::std::string* default_value, ArenaStringPtr value);
-
-  inline const ::std::string& GetNoArena() const { return *ptr_; }
-
-  inline ::std::string* MutableNoArena(const ::std::string* default_value) {
-    if (ptr_ == default_value) {
-      CreateInstanceNoArena(default_value);
-    }
-    return ptr_;
-  }
-
-  inline ::std::string* ReleaseNoArena(const ::std::string* default_value) {
-    if (ptr_ == default_value) {
-      return NULL;
-    } else {
-      ::std::string* released = ptr_;
-      ptr_ = const_cast< ::std::string* >(default_value);
-      return released;
-    }
-  }
-
-  inline void SetAllocatedNoArena(const ::std::string* default_value,
-                                  ::std::string* value) {
-    if (ptr_ != default_value) {
-      delete ptr_;
-    }
-    if (value != NULL) {
-      ptr_ = value;
-    } else {
-      ptr_ = const_cast< ::std::string* >(default_value);
-    }
-  }
-
-  inline void DestroyNoArena(const ::std::string* default_value) {
-    if (ptr_ != default_value) {
-      delete ptr_;
-    }
-  }
-
-  inline void ClearToEmptyNoArena(const ::std::string* default_value) {
-    if (ptr_ == default_value) {
-      // Nothing: already equal to default (which is the empty string).
-    } else {
-      ptr_->clear();
-    }
-  }
-
-  inline void ClearToDefaultNoArena(const ::std::string* default_value) {
-    if (ptr_ == default_value) {
-      // Nothing: already set to default.
-    } else {
-      // Reuse existing allocated instance.
-      *ptr_ = *default_value;
-    }
-  }
-
-  // Internal accessor used only at parse time to provide direct access to the
-  // raw pointer from the shared parse routine (in the non-arenas case). The
-  // parse routine does the string allocation in order to save code size in the
-  // generated parsing code.
-  inline ::std::string** UnsafeRawStringPointer() {
-    return &ptr_;
-  }
-
-  inline bool IsDefault(const ::std::string* default_value) const {
-    return ptr_ == default_value;
+  inline bool IsDefault(const std::string* default_value) const {
+    // Relies on the fact that kPtrTagString == 0, so if IsString(), ptr_ is the
+    // actual std::string pointer (and if !IsString(), ptr_ will never be equal
+    // to any aligned |default_value| pointer). The key is that we want to avoid
+    // masking in the fastpath const-pointer Get() case for non-arena code.
+    return tagged_ptr_.UnsafeGet() == default_value;
  }

 private:
-  ::std::string* ptr_;
+  TaggedPtr<std::string> tagged_ptr_;

-  GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
-  void CreateInstance(::google::protobuf::Arena* arena,
-                      const ::std::string* initial_value) {
-    GOOGLE_DCHECK(initial_value != NULL);
-    ptr_ = new ::std::string(*initial_value);
-    if (arena != NULL) {
-      arena->Own(ptr_);
-    }
-  }
-  GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
-  void CreateInstanceNoArena(const ::std::string* initial_value) {
-    GOOGLE_DCHECK(initial_value != NULL);
-    ptr_ = new ::std::string(*initial_value);
+  bool IsDonatedString() const { return false; }
+
+  // Swaps tagged pointer without debug hardening. This is to allow python
+  // protobuf to maintain pointer stability even in DEBUG builds.
+  inline PROTOBUF_NDEBUG_INLINE static void UnsafeShallowSwap(
+      ArenaStringPtr* rhs, ArenaStringPtr* lhs) {
+    std::swap(lhs->tagged_ptr_, rhs->tagged_ptr_);
  }
+
+  friend class ::google::protobuf::internal::SwapFieldHelper;
+
+  // Slow paths.
+
+  // MutableSlow requires that !IsString() || IsDefault
+  // Variadic to support 0 args for EmptyDefault and 1 arg for LazyString.
+  template <typename... Lazy>
+  std::string* MutableSlow(::google::protobuf::Arena* arena, const Lazy&... lazy_default);
+
+  // Sets value to a newly allocated string and returns it
+  std::string* SetAndReturnNewString();
+
+  // Destroys the non-default string value out-of-line
+  void DestroyNoArenaSlowPath();
+
 };

-}  // namespace internal
-}  // namespace protobuf
+inline void ArenaStringPtr::UnsafeSetDefault(const std::string* value) {
+  tagged_ptr_.Set(const_cast<std::string*>(value));
+}

+// Make sure rhs_arena allocated rhs, and lhs_arena allocated lhs.
+inline PROTOBUF_NDEBUG_INLINE void ArenaStringPtr::InternalSwap(  //
+    const std::string* default_value,                             //
+    ArenaStringPtr* rhs, Arena* rhs_arena,                        //
+    ArenaStringPtr* lhs, Arena* lhs_arena) {
+  // Silence unused variable warnings in release buildls.
+  (void)default_value;
+  (void)rhs_arena;
+  (void)lhs_arena;
+  std::swap(lhs->tagged_ptr_, rhs->tagged_ptr_);
+#ifdef PROTOBUF_FORCE_COPY_IN_SWAP
+  auto force_realloc = [default_value](ArenaStringPtr* p, Arena* arena) {
+    if (p->IsDefault(default_value)) return;
+    std::string* old_value = p->tagged_ptr_.Get();
+    std::string* new_value =
+        p->IsDonatedString()
+            ? Arena::Create<std::string>(arena, *old_value)
+            : Arena::Create<std::string>(arena, std::move(*old_value));
+    if (arena == nullptr) delete old_value;
+    p->tagged_ptr_.Set(new_value);
+  };
+  // Because, at this point, tagged_ptr_ has been swapped, arena should also be
+  // swapped.
+  force_realloc(lhs, rhs_arena);
+  force_realloc(rhs, lhs_arena);
+#endif  // PROTOBUF_FORCE_COPY_IN_SWAP
+}

+inline void ArenaStringPtr::ClearNonDefaultToEmpty() {
+  // Unconditionally mask away the tag.
+  tagged_ptr_.Get()->clear();
+}

-namespace protobuf {
-namespace internal {
-
-inline void ArenaStringPtr::AssignWithDefault(const ::std::string* default_value,
-                                       ArenaStringPtr value) {
-  const ::std::string* me = *UnsafeRawStringPointer();
-  const ::std::string* other = *value.UnsafeRawStringPointer();
-  // If the pointers are the same then do nothing.
-  if (me != other) {
-    SetNoArena(default_value, value.GetNoArena());
+inline std::string* ArenaStringPtr::MutableNoArenaNoDefault(
+    const std::string* default_value) {
+  // VERY IMPORTANT for performance and code size: this will reduce to a member
+  // variable load, a pointer check (against |default_value|, in practice a
+  // static global) and a branch to the slowpath (which calls operator new and
+  // the ctor). DO NOT add any tagged-pointer operations here.
+  if (IsDefault(default_value)) {
+    return SetAndReturnNewString();
+  } else {
+    return UnsafeMutablePointer();
  }
 }

+inline void ArenaStringPtr::DestroyNoArena(const std::string* default_value) {
+  if (!IsDefault(default_value)) {
+    DestroyNoArenaSlowPath();
+  }
+}
+
+inline std::string* ArenaStringPtr::UnsafeMutablePointer() {
+  GOOGLE_DCHECK(!tagged_ptr_.IsTagged());
+  GOOGLE_DCHECK(tagged_ptr_.UnsafeGet() != nullptr);
+  return tagged_ptr_.UnsafeGet();
+}
+
+
 }  // namespace internal
 }  // namespace protobuf
-
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
 #endif  // GOOGLE_PROTOBUF_ARENASTRING_H__
@@ -44,6 +44,12 @@
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/descriptor.h>

+#include <google/protobuf/port_def.inc>
+
+#ifdef SWIG
+#error "You cannot SWIG proto headers"
+#endif
+
 namespace google {
 namespace protobuf {

@@ -62,27 +68,27 @@ class MergedDescriptorDatabase;
 // calling DescriptorPool::BuildFile() for each one.  Instead, a DescriptorPool
 // can be created which wraps a DescriptorDatabase and only builds particular
 // descriptors when they are needed.
-class LIBPROTOBUF_EXPORT DescriptorDatabase {
+class PROTOBUF_EXPORT DescriptorDatabase {
 public:
  inline DescriptorDatabase() {}
  virtual ~DescriptorDatabase();

  // Find a file by file name.  Fills in in *output and returns true if found.
  // Otherwise, returns false, leaving the contents of *output undefined.
-  virtual bool FindFileByName(const string& filename,
+  virtual bool FindFileByName(const std::string& filename,
                              FileDescriptorProto* output) = 0;

  // Find the file that declares the given fully-qualified symbol name.
  // If found, fills in *output and returns true, otherwise returns false
  // and leaves *output undefined.
-  virtual bool FindFileContainingSymbol(const string& symbol_name,
+  virtual bool FindFileContainingSymbol(const std::string& symbol_name,
                                        FileDescriptorProto* output) = 0;

  // Find the file which defines an extension extending the given message type
  // with the given field number.  If found, fills in *output and returns true,
  // otherwise returns false and leaves *output undefined.  containing_type
  // must be a fully-qualified type name.
-  virtual bool FindFileContainingExtension(const string& containing_type,
+  virtual bool FindFileContainingExtension(const std::string& containing_type,
                                           int field_number,
                                           FileDescriptorProto* output) = 0;

@@ -96,7 +102,7 @@ class LIBPROTOBUF_EXPORT DescriptorDatabase {
  //
  // This method has a default implementation that always returns
  // false.
-  virtual bool FindAllExtensionNumbers(const string& /* extendee_type */,
+  virtual bool FindAllExtensionNumbers(const std::string& /* extendee_type */,
                                       std::vector<int>* /* output */) {
    return false;
  }
@@ -110,10 +116,24 @@ class LIBPROTOBUF_EXPORT DescriptorDatabase {
  //
  // This method has a default implementation that always returns
  // false.
-  virtual bool FindAllFileNames(std::vector<string>* output) {
+  virtual bool FindAllFileNames(std::vector<std::string>* /*output*/) {
    return false;
  }

+  // Finds the package names and appends them to the output in an
+  // undefined order. This method is best-effort: it's not guaranteed that the
+  // database will find all packages. Returns true if the database supports
+  // searching all package names, otherwise returns false and leaves output
+  // unchanged.
+  bool FindAllPackageNames(std::vector<std::string>* output);
+
+  // Finds the message names and appends them to the output in an
+  // undefined order. This method is best-effort: it's not guaranteed that the
+  // database will find all messages. Returns true if the database supports
+  // searching all message names, otherwise returns false and leaves output
+  // unchanged.
+  bool FindAllMessageNames(std::vector<std::string>* output);
+
 private:
  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(DescriptorDatabase);
 };
@@ -139,10 +159,10 @@ class LIBPROTOBUF_EXPORT DescriptorDatabase {
 // FileDescriptor::CopyTo()) will always use fully-qualified names for all
 // types.  You only need to worry if you are constructing FileDescriptorProtos
 // yourself, or are calling compiler::Parser directly.
-class LIBPROTOBUF_EXPORT SimpleDescriptorDatabase : public DescriptorDatabase {
+class PROTOBUF_EXPORT SimpleDescriptorDatabase : public DescriptorDatabase {
 public:
  SimpleDescriptorDatabase();
-  ~SimpleDescriptorDatabase();
+  ~SimpleDescriptorDatabase() override;

  // Adds the FileDescriptorProto to the database, making a copy.  The object
  // can be deleted after Add() returns.  Returns false if the file conflicted
@@ -154,20 +174,19 @@ class LIBPROTOBUF_EXPORT SimpleDescriptorDatabase : public DescriptorDatabase {
  bool AddAndOwn(const FileDescriptorProto* file);

  // implements DescriptorDatabase -----------------------------------
-  bool FindFileByName(const string& filename,
-                      FileDescriptorProto* output);
-  bool FindFileContainingSymbol(const string& symbol_name,
-                                FileDescriptorProto* output);
-  bool FindFileContainingExtension(const string& containing_type,
+  bool FindFileByName(const std::string& filename,
+                      FileDescriptorProto* output) override;
+  bool FindFileContainingSymbol(const std::string& symbol_name,
+                                FileDescriptorProto* output) override;
+  bool FindFileContainingExtension(const std::string& containing_type,
                                   int field_number,
-                                   FileDescriptorProto* output);
-  bool FindAllExtensionNumbers(const string& extendee_type,
-                               std::vector<int>* output);
+                                   FileDescriptorProto* output) override;
+  bool FindAllExtensionNumbers(const std::string& extendee_type,
+                               std::vector<int>* output) override;
+
+  bool FindAllFileNames(std::vector<std::string>* output) override;

 private:
-  // So that it can use DescriptorIndex.
-  friend class EncodedDescriptorDatabase;
-
  // An index mapping file names, symbol names, and extension numbers to
  // some sort of values.
  template <typename Value>
@@ -175,24 +194,24 @@ class LIBPROTOBUF_EXPORT SimpleDescriptorDatabase : public DescriptorDatabase {
   public:
    // Helpers to recursively add particular descriptors and all their contents
    // to the index.
-    bool AddFile(const FileDescriptorProto& file,
-                 Value value);
-    bool AddSymbol(const string& name, Value value);
-    bool AddNestedExtensions(const DescriptorProto& message_type,
-                             Value value);
-    bool AddExtension(const FieldDescriptorProto& field,
-                      Value value);
+    bool AddFile(const FileDescriptorProto& file, Value value);
+    bool AddSymbol(const std::string& name, Value value);
+    bool AddNestedExtensions(const std::string& filename,
+                             const DescriptorProto& message_type, Value value);
+    bool AddExtension(const std::string& filename,
+                      const FieldDescriptorProto& field, Value value);

-    Value FindFile(const string& filename);
-    Value FindSymbol(const string& name);
-    Value FindExtension(const string& containing_type, int field_number);
-    bool FindAllExtensionNumbers(const string& containing_type,
+    Value FindFile(const std::string& filename);
+    Value FindSymbol(const std::string& name);
+    Value FindExtension(const std::string& containing_type, int field_number);
+    bool FindAllExtensionNumbers(const std::string& containing_type,
                                 std::vector<int>* output);
+    void FindAllFileNames(std::vector<std::string>* output);

   private:
-    std::map<string, Value> by_name_;
-    std::map<string, Value> by_symbol_;
-    std::map<std::pair<string, int>, Value> by_extension_;
+    std::map<std::string, Value> by_name_;
+    std::map<std::string, Value> by_symbol_;
+    std::map<std::pair<std::string, int>, Value> by_extension_;

    // Invariant:  The by_symbol_ map does not contain any symbols which are
    // prefixes of other symbols in the map.  For example, "foo.bar" is a
@@ -214,7 +233,7 @@ class LIBPROTOBUF_EXPORT SimpleDescriptorDatabase : public DescriptorDatabase {
    // will find it.  Proof:
    // 1) Define the "search key" to be the key we are looking for, the "found
    //    key" to be the key found in step (1), and the "match key" to be the
-    //    key which actually matches the serach key (i.e. the key we're trying
+    //    key which actually matches the search key (i.e. the key we're trying
    //    to find).
    // 2) The found key must be less than or equal to the search key by
    //    definition.
@@ -244,30 +263,14 @@ class LIBPROTOBUF_EXPORT SimpleDescriptorDatabase : public DescriptorDatabase {
    // That symbol cannot be a super-symbol of the search key since if it were,
    // then it would be a match, and we're assuming the match key doesn't exist.
    // Therefore, step 2 will correctly return no match.
-
-    // Find the last entry in the by_symbol_ map whose key is less than or
-    // equal to the given name.
-    typename std::map<string, Value>::iterator FindLastLessOrEqual(
-        const string& name);
-
-    // True if either the arguments are equal or super_symbol identifies a
-    // parent symbol of sub_symbol (e.g. "foo.bar" is a parent of
-    // "foo.bar.baz", but not a parent of "foo.barbaz").
-    bool IsSubSymbol(const string& sub_symbol, const string& super_symbol);
-
-    // Returns true if and only if all characters in the name are alphanumerics,
-    // underscores, or periods.
-    bool ValidateSymbolName(const string& name);
  };

-
  DescriptorIndex<const FileDescriptorProto*> index_;
-  std::vector<const FileDescriptorProto*> files_to_delete_;
+  std::vector<std::unique_ptr<const FileDescriptorProto>> files_to_delete_;

-  // If file is non-NULL, copy it into *output and return true, otherwise
+  // If file is non-nullptr, copy it into *output and return true, otherwise
  // return false.
-  bool MaybeCopy(const FileDescriptorProto* file,
-                 FileDescriptorProto* output);
+  bool MaybeCopy(const FileDescriptorProto* file, FileDescriptorProto* output);

  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(SimpleDescriptorDatabase);
 };
@@ -277,10 +280,10 @@ class LIBPROTOBUF_EXPORT SimpleDescriptorDatabase : public DescriptorDatabase {
 //
 // The same caveats regarding FindFileContainingExtension() apply as with
 // SimpleDescriptorDatabase.
-class LIBPROTOBUF_EXPORT EncodedDescriptorDatabase : public DescriptorDatabase {
+class PROTOBUF_EXPORT EncodedDescriptorDatabase : public DescriptorDatabase {
 public:
  EncodedDescriptorDatabase();
-  ~EncodedDescriptorDatabase();
+  ~EncodedDescriptorDatabase() override;

  // Adds the FileDescriptorProto to the database.  The descriptor is provided
  // in encoded form.  The database does not make a copy of the bytes, nor
@@ -295,27 +298,30 @@ class LIBPROTOBUF_EXPORT EncodedDescriptorDatabase : public DescriptorDatabase {
  bool AddCopy(const void* encoded_file_descriptor, int size);

  // Like FindFileContainingSymbol but returns only the name of the file.
-  bool FindNameOfFileContainingSymbol(const string& symbol_name,
-                                      string* output);
+  bool FindNameOfFileContainingSymbol(const std::string& symbol_name,
+                                      std::string* output);

  // implements DescriptorDatabase -----------------------------------
-  bool FindFileByName(const string& filename,
-                      FileDescriptorProto* output);
-  bool FindFileContainingSymbol(const string& symbol_name,
-                                FileDescriptorProto* output);
-  bool FindFileContainingExtension(const string& containing_type,
+  bool FindFileByName(const std::string& filename,
+                      FileDescriptorProto* output) override;
+  bool FindFileContainingSymbol(const std::string& symbol_name,
+                                FileDescriptorProto* output) override;
+  bool FindFileContainingExtension(const std::string& containing_type,
                                   int field_number,
-                                   FileDescriptorProto* output);
-  bool FindAllExtensionNumbers(const string& extendee_type,
-                               std::vector<int>* output);
+                                   FileDescriptorProto* output) override;
+  bool FindAllExtensionNumbers(const std::string& extendee_type,
+                               std::vector<int>* output) override;
+  bool FindAllFileNames(std::vector<std::string>* output) override;

 private:
-  SimpleDescriptorDatabase::DescriptorIndex<std::pair<const void*, int> >
-      index_;
+  class DescriptorIndex;
+  // Keep DescriptorIndex by pointer to hide the implementation to keep a
+  // cleaner header.
+  std::unique_ptr<DescriptorIndex> index_;
  std::vector<void*> files_to_delete_;

-  // If encoded_file.first is non-NULL, parse the data into *output and return
-  // true, otherwise return false.
+  // If encoded_file.first is non-nullptr, parse the data into *output and
+  // return true, otherwise return false.
  bool MaybeParse(std::pair<const void*, int> encoded_file,
                  FileDescriptorProto* output);

@@ -323,21 +329,21 @@ class LIBPROTOBUF_EXPORT EncodedDescriptorDatabase : public DescriptorDatabase {
 };

 // A DescriptorDatabase that fetches files from a given pool.
-class LIBPROTOBUF_EXPORT DescriptorPoolDatabase : public DescriptorDatabase {
+class PROTOBUF_EXPORT DescriptorPoolDatabase : public DescriptorDatabase {
 public:
  explicit DescriptorPoolDatabase(const DescriptorPool& pool);
-  ~DescriptorPoolDatabase();
+  ~DescriptorPoolDatabase() override;

  // implements DescriptorDatabase -----------------------------------
-  bool FindFileByName(const string& filename,
-                      FileDescriptorProto* output);
-  bool FindFileContainingSymbol(const string& symbol_name,
-                                FileDescriptorProto* output);
-  bool FindFileContainingExtension(const string& containing_type,
+  bool FindFileByName(const std::string& filename,
+                      FileDescriptorProto* output) override;
+  bool FindFileContainingSymbol(const std::string& symbol_name,
+                                FileDescriptorProto* output) override;
+  bool FindFileContainingExtension(const std::string& containing_type,
                                   int field_number,
-                                   FileDescriptorProto* output);
-  bool FindAllExtensionNumbers(const string& extendee_type,
-                               std::vector<int>* output);
+                                   FileDescriptorProto* output) override;
+  bool FindAllExtensionNumbers(const std::string& extendee_type,
+                               std::vector<int>* output) override;

 private:
  const DescriptorPool& pool_;
@@ -346,7 +352,7 @@ class LIBPROTOBUF_EXPORT DescriptorPoolDatabase : public DescriptorDatabase {

 // A DescriptorDatabase that wraps two or more others.  It first searches the
 // first database and, if that fails, tries the second, and so on.
-class LIBPROTOBUF_EXPORT MergedDescriptorDatabase : public DescriptorDatabase {
+class PROTOBUF_EXPORT MergedDescriptorDatabase : public DescriptorDatabase {
 public:
  // Merge just two databases.  The sources remain property of the caller.
  MergedDescriptorDatabase(DescriptorDatabase* source1,
@@ -356,20 +362,20 @@ class LIBPROTOBUF_EXPORT MergedDescriptorDatabase : public DescriptorDatabase {
  // DescriptorDatabases need to stick around.
  explicit MergedDescriptorDatabase(
      const std::vector<DescriptorDatabase*>& sources);
-  ~MergedDescriptorDatabase();
+  ~MergedDescriptorDatabase() override;

  // implements DescriptorDatabase -----------------------------------
-  bool FindFileByName(const string& filename,
-                      FileDescriptorProto* output);
-  bool FindFileContainingSymbol(const string& symbol_name,
-                                FileDescriptorProto* output);
-  bool FindFileContainingExtension(const string& containing_type,
+  bool FindFileByName(const std::string& filename,
+                      FileDescriptorProto* output) override;
+  bool FindFileContainingSymbol(const std::string& symbol_name,
+                                FileDescriptorProto* output) override;
+  bool FindFileContainingExtension(const std::string& containing_type,
                                   int field_number,
-                                   FileDescriptorProto* output);
+                                   FileDescriptorProto* output) override;
  // Merges the results of calling all databases. Returns true iff any
  // of the databases returned true.
-  bool FindAllExtensionNumbers(const string& extendee_type,
-                               std::vector<int>* output);
+  bool FindAllExtensionNumbers(const std::string& extendee_type,
+                               std::vector<int>* output) override;


 private:
@@ -378,6 +384,8 @@ class LIBPROTOBUF_EXPORT MergedDescriptorDatabase : public DescriptorDatabase {
 };

 }  // namespace protobuf
-
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
 #endif  // GOOGLE_PROTOBUF_DESCRIPTOR_DATABASE_H__
@@ -1,431 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: google/protobuf/duration.proto
-
-#include <google/protobuf/duration.pb.h>
-
-#include <algorithm>
-
-#include <google/protobuf/stubs/common.h>
-#include <google/protobuf/stubs/port.h>
-#include <google/protobuf/stubs/once.h>
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/wire_format_lite_inl.h>
-#include <google/protobuf/descriptor.h>
-#include <google/protobuf/generated_message_reflection.h>
-#include <google/protobuf/reflection_ops.h>
-#include <google/protobuf/wire_format.h>
-// This is a temporary google only hack
-#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-#include "third_party/protobuf/version.h"
-#endif
-// @@protoc_insertion_point(includes)
-namespace google {
-namespace protobuf {
-class DurationDefaultTypeInternal {
- public:
-  ::google::protobuf::internal::ExplicitlyConstructed<Duration>
-      _instance;
-} _Duration_default_instance_;
-}  // namespace protobuf
-}  // namespace google
-namespace protobuf_google_2fprotobuf_2fduration_2eproto {
-void InitDefaultsDurationImpl() {
-  GOOGLE_PROTOBUF_VERIFY_VERSION;
-
-#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
-#else
-  ::google::protobuf::internal::InitProtobufDefaults();
-#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-  {
-    void* ptr = &::google::protobuf::_Duration_default_instance_;
-    new (ptr) ::google::protobuf::Duration();
-    ::google::protobuf::internal::OnShutdownDestroyMessage(ptr);
-  }
-  ::google::protobuf::Duration::InitAsDefaultInstance();
-}
-
-void InitDefaultsDuration() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsDurationImpl);
-}
-
-::google::protobuf::Metadata file_level_metadata[1];
-
-const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  ~0u,  // no _has_bits_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Duration, _internal_metadata_),
-  ~0u,  // no _extensions_
-  ~0u,  // no _oneof_case_
-  ~0u,  // no _weak_field_map_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Duration, seconds_),
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Duration, nanos_),
-};
-static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  { 0, -1, sizeof(::google::protobuf::Duration)},
-};
-
-static ::google::protobuf::Message const * const file_default_instances[] = {
-  reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Duration_default_instance_),
-};
-
-void protobuf_AssignDescriptors() {
-  AddDescriptors();
-  ::google::protobuf::MessageFactory* factory = NULL;
-  AssignDescriptors(
-      "google/protobuf/duration.proto", schemas, file_default_instances, TableStruct::offsets, factory,
-      file_level_metadata, NULL, NULL);
-}
-
-void protobuf_AssignDescriptorsOnce() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &protobuf_AssignDescriptors);
-}
-
-void protobuf_RegisterTypes(const ::std::string&) GOOGLE_PROTOBUF_ATTRIBUTE_COLD;
-void protobuf_RegisterTypes(const ::std::string&) {
-  protobuf_AssignDescriptorsOnce();
-  ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1);
-}
-
-void AddDescriptorsImpl() {
-  InitDefaults();
-  static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-      "\n\036google/protobuf/duration.proto\022\017google"
-      ".protobuf\"*\n\010Duration\022\017\n\007seconds\030\001 \001(\003\022\r"
-      "\n\005nanos\030\002 \001(\005B|\n\023com.google.protobufB\rDu"
-      "rationProtoP\001Z*github.com/golang/protobu"
-      "f/ptypes/duration\370\001\001\242\002\003GPB\252\002\036Google.Prot"
-      "obuf.WellKnownTypesb\006proto3"
-  };
-  ::google::protobuf::DescriptorPool::InternalAddGeneratedFile(
-      descriptor, 227);
-  ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
-    "google/protobuf/duration.proto", &protobuf_RegisterTypes);
-}
-
-void AddDescriptors() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &AddDescriptorsImpl);
-}
-// Force AddDescriptors() to be called at dynamic initialization time.
-struct StaticDescriptorInitializer {
-  StaticDescriptorInitializer() {
-    AddDescriptors();
-  }
-} static_descriptor_initializer;
-}  // namespace protobuf_google_2fprotobuf_2fduration_2eproto
-namespace google {
-namespace protobuf {
-
-// ===================================================================
-
-void Duration::InitAsDefaultInstance() {
-}
-#if !defined(_MSC_VER) || _MSC_VER >= 1900
-const int Duration::kSecondsFieldNumber;
-const int Duration::kNanosFieldNumber;
-#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
-
-Duration::Duration()
-  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
-  if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) {
-    ::protobuf_google_2fprotobuf_2fduration_2eproto::InitDefaultsDuration();
-  }
-  SharedCtor();
-  // @@protoc_insertion_point(constructor:google.protobuf.Duration)
-}
-Duration::Duration(::google::protobuf::Arena* arena)
-  : ::google::protobuf::Message(),
-  _internal_metadata_(arena) {
-  ::protobuf_google_2fprotobuf_2fduration_2eproto::InitDefaultsDuration();
-  SharedCtor();
-  RegisterArenaDtor(arena);
-  // @@protoc_insertion_point(arena_constructor:google.protobuf.Duration)
-}
-Duration::Duration(const Duration& from)
-  : ::google::protobuf::Message(),
-      _internal_metadata_(NULL),
-      _cached_size_(0) {
-  _internal_metadata_.MergeFrom(from._internal_metadata_);
-  ::memcpy(&seconds_, &from.seconds_,
-    static_cast<size_t>(reinterpret_cast<char*>(&nanos_) -
-    reinterpret_cast<char*>(&seconds_)) + sizeof(nanos_));
-  // @@protoc_insertion_point(copy_constructor:google.protobuf.Duration)
-}
-
-void Duration::SharedCtor() {
-  ::memset(&seconds_, 0, static_cast<size_t>(
-      reinterpret_cast<char*>(&nanos_) -
-      reinterpret_cast<char*>(&seconds_)) + sizeof(nanos_));
-  _cached_size_ = 0;
-}
-
-Duration::~Duration() {
-  // @@protoc_insertion_point(destructor:google.protobuf.Duration)
-  SharedDtor();
-}
-
-void Duration::SharedDtor() {
-  GOOGLE_DCHECK(GetArenaNoVirtual() == NULL);
-}
-
-void Duration::ArenaDtor(void* object) {
-  Duration* _this = reinterpret_cast< Duration* >(object);
-  (void)_this;
-}
-void Duration::RegisterArenaDtor(::google::protobuf::Arena* arena) {
-}
-void Duration::SetCachedSize(int size) const {
-  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
-  _cached_size_ = size;
-  GOOGLE_SAFE_CONCURRENT_WRITES_END();
-}
-const ::google::protobuf::Descriptor* Duration::descriptor() {
-  ::protobuf_google_2fprotobuf_2fduration_2eproto::protobuf_AssignDescriptorsOnce();
-  return ::protobuf_google_2fprotobuf_2fduration_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
-}
-
-const Duration& Duration::default_instance() {
-  ::protobuf_google_2fprotobuf_2fduration_2eproto::InitDefaultsDuration();
-  return *internal_default_instance();
-}
-
-Duration* Duration::New(::google::protobuf::Arena* arena) const {
-  return ::google::protobuf::Arena::CreateMessage<Duration>(arena);
-}
-
-void Duration::Clear() {
-// @@protoc_insertion_point(message_clear_start:google.protobuf.Duration)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  // Prevent compiler warnings about cached_has_bits being unused
-  (void) cached_has_bits;
-
-  ::memset(&seconds_, 0, static_cast<size_t>(
-      reinterpret_cast<char*>(&nanos_) -
-      reinterpret_cast<char*>(&seconds_)) + sizeof(nanos_));
-  _internal_metadata_.Clear();
-}
-
-bool Duration::MergePartialFromCodedStream(
-    ::google::protobuf::io::CodedInputStream* input) {
-#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
-  ::google::protobuf::uint32 tag;
-  // @@protoc_insertion_point(parse_start:google.protobuf.Duration)
-  for (;;) {
-    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u);
-    tag = p.first;
-    if (!p.second) goto handle_unusual;
-    switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
-      // int64 seconds = 1;
-      case 1: {
-        if (static_cast< ::google::protobuf::uint8>(tag) ==
-            static_cast< ::google::protobuf::uint8>(8u /* 8 & 0xFF */)) {
-
-          DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive<
-                   ::google::protobuf::int64, ::google::protobuf::internal::WireFormatLite::TYPE_INT64>(
-                 input, &seconds_)));
-        } else {
-          goto handle_unusual;
-        }
-        break;
-      }
-
-      // int32 nanos = 2;
-      case 2: {
-        if (static_cast< ::google::protobuf::uint8>(tag) ==
-            static_cast< ::google::protobuf::uint8>(16u /* 16 & 0xFF */)) {
-
-          DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive<
-                   ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>(
-                 input, &nanos_)));
-        } else {
-          goto handle_unusual;
-        }
-        break;
-      }
-
-      default: {
-      handle_unusual:
-        if (tag == 0) {
-          goto success;
-        }
-        DO_(::google::protobuf::internal::WireFormat::SkipField(
-              input, tag, _internal_metadata_.mutable_unknown_fields()));
-        break;
-      }
-    }
-  }
-success:
-  // @@protoc_insertion_point(parse_success:google.protobuf.Duration)
-  return true;
-failure:
-  // @@protoc_insertion_point(parse_failure:google.protobuf.Duration)
-  return false;
-#undef DO_
-}
-
-void Duration::SerializeWithCachedSizes(
-    ::google::protobuf::io::CodedOutputStream* output) const {
-  // @@protoc_insertion_point(serialize_start:google.protobuf.Duration)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-  // int64 seconds = 1;
-  if (this->seconds() != 0) {
-    ::google::protobuf::internal::WireFormatLite::WriteInt64(1, this->seconds(), output);
-  }
-
-  // int32 nanos = 2;
-  if (this->nanos() != 0) {
-    ::google::protobuf::internal::WireFormatLite::WriteInt32(2, this->nanos(), output);
-  }
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    ::google::protobuf::internal::WireFormat::SerializeUnknownFields(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()), output);
-  }
-  // @@protoc_insertion_point(serialize_end:google.protobuf.Duration)
-}
-
-::google::protobuf::uint8* Duration::InternalSerializeWithCachedSizesToArray(
-    bool deterministic, ::google::protobuf::uint8* target) const {
-  (void)deterministic; // Unused
-  // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.Duration)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-  // int64 seconds = 1;
-  if (this->seconds() != 0) {
-    target = ::google::protobuf::internal::WireFormatLite::WriteInt64ToArray(1, this->seconds(), target);
-  }
-
-  // int32 nanos = 2;
-  if (this->nanos() != 0) {
-    target = ::google::protobuf::internal::WireFormatLite::WriteInt32ToArray(2, this->nanos(), target);
-  }
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()), target);
-  }
-  // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Duration)
-  return target;
-}
-
-size_t Duration::ByteSizeLong() const {
-// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Duration)
-  size_t total_size = 0;
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    total_size +=
-      ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()));
-  }
-  // int64 seconds = 1;
-  if (this->seconds() != 0) {
-    total_size += 1 +
-      ::google::protobuf::internal::WireFormatLite::Int64Size(
-        this->seconds());
-  }
-
-  // int32 nanos = 2;
-  if (this->nanos() != 0) {
-    total_size += 1 +
-      ::google::protobuf::internal::WireFormatLite::Int32Size(
-        this->nanos());
-  }
-
-  int cached_size = ::google::protobuf::internal::ToCachedSize(total_size);
-  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
-  _cached_size_ = cached_size;
-  GOOGLE_SAFE_CONCURRENT_WRITES_END();
-  return total_size;
-}
-
-void Duration::MergeFrom(const ::google::protobuf::Message& from) {
-// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.Duration)
-  GOOGLE_DCHECK_NE(&from, this);
-  const Duration* source =
-      ::google::protobuf::internal::DynamicCastToGenerated<const Duration>(
-          &from);
-  if (source == NULL) {
-  // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.Duration)
-    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
-  } else {
-  // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.Duration)
-    MergeFrom(*source);
-  }
-}
-
-void Duration::MergeFrom(const Duration& from) {
-// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.Duration)
-  GOOGLE_DCHECK_NE(&from, this);
-  _internal_metadata_.MergeFrom(from._internal_metadata_);
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-  if (from.seconds() != 0) {
-    set_seconds(from.seconds());
-  }
-  if (from.nanos() != 0) {
-    set_nanos(from.nanos());
-  }
-}
-
-void Duration::CopyFrom(const ::google::protobuf::Message& from) {
-// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.Duration)
-  if (&from == this) return;
-  Clear();
-  MergeFrom(from);
-}
-
-void Duration::CopyFrom(const Duration& from) {
-// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.Duration)
-  if (&from == this) return;
-  Clear();
-  MergeFrom(from);
-}
-
-bool Duration::IsInitialized() const {
-  return true;
-}
-
-void Duration::Swap(Duration* other) {
-  if (other == this) return;
-  if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) {
-    InternalSwap(other);
-  } else {
-    Duration* temp = New(GetArenaNoVirtual());
-    temp->MergeFrom(*other);
-    other->CopyFrom(*this);
-    InternalSwap(temp);
-    if (GetArenaNoVirtual() == NULL) {
-      delete temp;
-    }
-  }
-}
-void Duration::UnsafeArenaSwap(Duration* other) {
-  if (other == this) return;
-  GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual());
-  InternalSwap(other);
-}
-void Duration::InternalSwap(Duration* other) {
-  using std::swap;
-  swap(seconds_, other->seconds_);
-  swap(nanos_, other->nanos_);
-  _internal_metadata_.Swap(&other->_internal_metadata_);
-  swap(_cached_size_, other->_cached_size_);
-}
-
-::google::protobuf::Metadata Duration::GetMetadata() const {
-  protobuf_google_2fprotobuf_2fduration_2eproto::protobuf_AssignDescriptorsOnce();
-  return ::protobuf_google_2fprotobuf_2fduration_2eproto::file_level_metadata[kIndexInFileMessages];
-}
-
-
-// @@protoc_insertion_point(namespace_scope)
-}  // namespace protobuf
-}  // namespace google
-
-// @@protoc_insertion_point(global_scope)
@@ -1,232 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: google/protobuf/duration.proto
-
-#ifndef PROTOBUF_google_2fprotobuf_2fduration_2eproto__INCLUDED
-#define PROTOBUF_google_2fprotobuf_2fduration_2eproto__INCLUDED
-
-#include <string>
-
-#include <google/protobuf/stubs/common.h>
-
-#if GOOGLE_PROTOBUF_VERSION < 3005000
-#error This file was generated by a newer version of protoc which is
-#error incompatible with your Protocol Buffer headers.  Please update
-#error your headers.
-#endif
-#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION
-#error This file was generated by an older version of protoc which is
-#error incompatible with your Protocol Buffer headers.  Please
-#error regenerate this file with a newer version of protoc.
-#endif
-
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/arena.h>
-#include <google/protobuf/arenastring.h>
-#include <google/protobuf/generated_message_table_driven.h>
-#include <google/protobuf/generated_message_util.h>
-#include <google/protobuf/metadata.h>
-#include <google/protobuf/message.h>
-#include <google/protobuf/repeated_field.h>  // IWYU pragma: export
-#include <google/protobuf/extension_set.h>  // IWYU pragma: export
-#include <google/protobuf/unknown_field_set.h>
-// @@protoc_insertion_point(includes)
-
-namespace protobuf_google_2fprotobuf_2fduration_2eproto {
-// Internal implementation detail -- do not use these members.
-struct LIBPROTOBUF_EXPORT TableStruct {
-  static const ::google::protobuf::internal::ParseTableField entries[];
-  static const ::google::protobuf::internal::AuxillaryParseTableField aux[];
-  static const ::google::protobuf::internal::ParseTable schema[1];
-  static const ::google::protobuf::internal::FieldMetadata field_metadata[];
-  static const ::google::protobuf::internal::SerializationTable serialization_table[];
-  static const ::google::protobuf::uint32 offsets[];
-};
-void LIBPROTOBUF_EXPORT AddDescriptors();
-void LIBPROTOBUF_EXPORT InitDefaultsDurationImpl();
-void LIBPROTOBUF_EXPORT InitDefaultsDuration();
-inline void LIBPROTOBUF_EXPORT InitDefaults() {
-  InitDefaultsDuration();
-}
-}  // namespace protobuf_google_2fprotobuf_2fduration_2eproto
-namespace google {
-namespace protobuf {
-class Duration;
-class DurationDefaultTypeInternal;
-LIBPROTOBUF_EXPORT extern DurationDefaultTypeInternal _Duration_default_instance_;
-}  // namespace protobuf
-}  // namespace google
-namespace google {
-namespace protobuf {
-
-// ===================================================================
-
-class LIBPROTOBUF_EXPORT Duration : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Duration) */ {
- public:
-  Duration();
-  virtual ~Duration();
-
-  Duration(const Duration& from);
-
-  inline Duration& operator=(const Duration& from) {
-    CopyFrom(from);
-    return *this;
-  }
-  #if LANG_CXX11
-  Duration(Duration&& from) noexcept
-    : Duration() {
-    *this = ::std::move(from);
-  }
-
-  inline Duration& operator=(Duration&& from) noexcept {
-    if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
-      if (this != &from) InternalSwap(&from);
-    } else {
-      CopyFrom(from);
-    }
-    return *this;
-  }
-  #endif
-  inline ::google::protobuf::Arena* GetArena() const PROTOBUF_FINAL {
-    return GetArenaNoVirtual();
-  }
-  inline void* GetMaybeArenaPointer() const PROTOBUF_FINAL {
-    return MaybeArenaPtr();
-  }
-  static const ::google::protobuf::Descriptor* descriptor();
-  static const Duration& default_instance();
-
-  static void InitAsDefaultInstance();  // FOR INTERNAL USE ONLY
-  static inline const Duration* internal_default_instance() {
-    return reinterpret_cast<const Duration*>(
-               &_Duration_default_instance_);
-  }
-  static PROTOBUF_CONSTEXPR int const kIndexInFileMessages =
-    0;
-
-  void UnsafeArenaSwap(Duration* other);
-  void Swap(Duration* other);
-  friend void swap(Duration& a, Duration& b) {
-    a.Swap(&b);
-  }
-
-  // implements Message ----------------------------------------------
-
-  inline Duration* New() const PROTOBUF_FINAL { return New(NULL); }
-
-  Duration* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL;
-  void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
-  void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
-  void CopyFrom(const Duration& from);
-  void MergeFrom(const Duration& from);
-  void Clear() PROTOBUF_FINAL;
-  bool IsInitialized() const PROTOBUF_FINAL;
-
-  size_t ByteSizeLong() const PROTOBUF_FINAL;
-  bool MergePartialFromCodedStream(
-      ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL;
-  void SerializeWithCachedSizes(
-      ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL;
-  ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(
-      bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL;
-  int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; }
-  private:
-  void SharedCtor();
-  void SharedDtor();
-  void SetCachedSize(int size) const PROTOBUF_FINAL;
-  void InternalSwap(Duration* other);
-  protected:
-  explicit Duration(::google::protobuf::Arena* arena);
-  private:
-  static void ArenaDtor(void* object);
-  inline void RegisterArenaDtor(::google::protobuf::Arena* arena);
-  private:
-  inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
-    return _internal_metadata_.arena();
-  }
-  inline void* MaybeArenaPtr() const {
-    return _internal_metadata_.raw_arena_ptr();
-  }
-  public:
-
-  ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL;
-
-  // nested types ----------------------------------------------------
-
-  // accessors -------------------------------------------------------
-
-  // int64 seconds = 1;
-  void clear_seconds();
-  static const int kSecondsFieldNumber = 1;
-  ::google::protobuf::int64 seconds() const;
-  void set_seconds(::google::protobuf::int64 value);
-
-  // int32 nanos = 2;
-  void clear_nanos();
-  static const int kNanosFieldNumber = 2;
-  ::google::protobuf::int32 nanos() const;
-  void set_nanos(::google::protobuf::int32 value);
-
-  // @@protoc_insertion_point(class_scope:google.protobuf.Duration)
- private:
-
-  ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_;
-  template <typename T> friend class ::google::protobuf::Arena::InternalHelper;
-  typedef void InternalArenaConstructable_;
-  typedef void DestructorSkippable_;
-  ::google::protobuf::int64 seconds_;
-  ::google::protobuf::int32 nanos_;
-  mutable int _cached_size_;
-  friend struct ::protobuf_google_2fprotobuf_2fduration_2eproto::TableStruct;
-  friend void ::protobuf_google_2fprotobuf_2fduration_2eproto::InitDefaultsDurationImpl();
-};
-// ===================================================================
-
-
-// ===================================================================
-
-#ifdef __GNUC__
-  #pragma GCC diagnostic push
-  #pragma GCC diagnostic ignored "-Wstrict-aliasing"
-#endif  // __GNUC__
-// Duration
-
-// int64 seconds = 1;
-inline void Duration::clear_seconds() {
-  seconds_ = GOOGLE_LONGLONG(0);
-}
-inline ::google::protobuf::int64 Duration::seconds() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Duration.seconds)
-  return seconds_;
-}
-inline void Duration::set_seconds(::google::protobuf::int64 value) {
-  
-  seconds_ = value;
-  // @@protoc_insertion_point(field_set:google.protobuf.Duration.seconds)
-}
-
-// int32 nanos = 2;
-inline void Duration::clear_nanos() {
-  nanos_ = 0;
-}
-inline ::google::protobuf::int32 Duration::nanos() const {
-  // @@protoc_insertion_point(field_get:google.protobuf.Duration.nanos)
-  return nanos_;
-}
-inline void Duration::set_nanos(::google::protobuf::int32 value) {
-  
-  nanos_ = value;
-  // @@protoc_insertion_point(field_set:google.protobuf.Duration.nanos)
-}
-
-#ifdef __GNUC__
-  #pragma GCC diagnostic pop
-#endif  // __GNUC__
-
-// @@protoc_insertion_point(namespace_scope)
-
-}  // namespace protobuf
-}  // namespace google
-
-// @@protoc_insertion_point(global_scope)
-
-#endif  // PROTOBUF_google_2fprotobuf_2fduration_2eproto__INCLUDED
@@ -40,22 +40,27 @@

 #include <algorithm>
 #include <memory>
-#ifndef _SHARED_PTR_H
-#include <google/protobuf/stubs/shared_ptr.h>
-#endif
+#include <unordered_map>
 #include <vector>

-#include <google/protobuf/message.h>
-#include <google/protobuf/repeated_field.h>
 #include <google/protobuf/stubs/common.h>
+#include <google/protobuf/message.h>
 #include <google/protobuf/stubs/mutex.h>
+#include <google/protobuf/reflection.h>
+#include <google/protobuf/repeated_field.h>
+
+#ifdef SWIG
+#error "You cannot SWIG proto headers"
+#endif
+
+#include <google/protobuf/port_def.inc>

 namespace google {
 namespace protobuf {

 // Defined in other files.
-class Descriptor;        // descriptor.h
-class DescriptorPool;    // descriptor.h
+class Descriptor;      // descriptor.h
+class DescriptorPool;  // descriptor.h

 // Constructs implementations of Message which can emulate types which are not
 // known at compile-time.
@@ -74,7 +79,7 @@ class DescriptorPool;    // descriptor.h
 // encapsulates this "cache".  All DynamicMessages of the same type created
 // from the same factory will share the same support data.  Any Descriptors
 // used with a particular factory must outlive the factory.
-class LIBPROTOBUF_EXPORT DynamicMessageFactory : public MessageFactory {
+class PROTOBUF_EXPORT DynamicMessageFactory : public MessageFactory {
 public:
  // Construct a DynamicMessageFactory that will search for extensions in
  // the DescriptorPool in which the extendee is defined.
@@ -118,62 +123,44 @@ class LIBPROTOBUF_EXPORT DynamicMessageFactory : public MessageFactory {
  // outlive the DynamicMessageFactory.
  //
  // The method is thread-safe.
-  const Message* GetPrototype(const Descriptor* type);
+  const Message* GetPrototype(const Descriptor* type) override;

 private:
  const DescriptorPool* pool_;
  bool delegate_to_generated_factory_;

-  // This struct just contains a hash_map.  We can't #include <google/protobuf/stubs/hash.h> from
-  // this header due to hacks needed for hash_map portability in the open source
-  // release.  Namely, stubs/hash.h, which defines hash_map portably, is not a
-  // public header (for good reason), but dynamic_message.h is, and public
-  // headers may only #include other public headers.
-  struct PrototypeMap;
-  google::protobuf::scoped_ptr<PrototypeMap> prototypes_;
-  mutable Mutex prototypes_mutex_;
+  struct TypeInfo;
+  std::unordered_map<const Descriptor*, const TypeInfo*> prototypes_;
+  mutable internal::WrappedMutex prototypes_mutex_;

  friend class DynamicMessage;
  const Message* GetPrototypeNoLock(const Descriptor* type);

-  // Construct default oneof instance for reflection usage if oneof
-  // is defined.
-  static void ConstructDefaultOneofInstance(const Descriptor* type,
-                                            const uint32 offsets[],
-                                            void* default_oneof_instance);
-  // Delete default oneof instance. Called by ~DynamicMessageFactory.
-  static void DeleteDefaultOneofInstance(const Descriptor* type,
-                                         const uint32 offsets[],
-                                         const void* default_oneof_instance);
-
  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(DynamicMessageFactory);
 };

 // Helper for computing a sorted list of map entries via reflection.
-class LIBPROTOBUF_EXPORT DynamicMapSorter {
+class PROTOBUF_EXPORT DynamicMapSorter {
 public:
-  static std::vector<const Message*> Sort(const Message& message,
-                                          int map_size,
+  static std::vector<const Message*> Sort(const Message& message, int map_size,
                                          const Reflection* reflection,
                                          const FieldDescriptor* field) {
-    std::vector<const Message*> result(static_cast<size_t>(map_size));
-    const RepeatedPtrField<Message>& map_field =
-        reflection->GetRepeatedPtrField<Message>(message, field);
-    size_t i = 0;
-    for (RepeatedPtrField<Message>::const_pointer_iterator it =
-             map_field.pointer_begin(); it != map_field.pointer_end(); ) {
-      result[i++] = *it++;
+    std::vector<const Message*> result;
+    result.reserve(map_size);
+    RepeatedFieldRef<Message> map_field =
+        reflection->GetRepeatedFieldRef<Message>(message, field);
+    for (auto it = map_field.begin(); it != map_field.end(); ++it) {
+      result.push_back(&*it);
    }
-    GOOGLE_DCHECK_EQ(result.size(), i);
    MapEntryMessageComparator comparator(field->message_type());
    std::stable_sort(result.begin(), result.end(), comparator);
    // Complain if the keys aren't in ascending order.
 #ifndef NDEBUG
    for (size_t j = 1; j < static_cast<size_t>(map_size); j++) {
      if (!comparator(result[j - 1], result[j])) {
-        GOOGLE_LOG(ERROR) << (comparator(result[j], result[j - 1]) ?
-                      "internal error in map key sorting" :
-                      "map keys are not unique");
+        GOOGLE_LOG(ERROR) << (comparator(result[j], result[j - 1])
+                           ? "internal error in map key sorting"
+                           : "map keys are not unique");
      }
    }
 #endif
@@ -181,7 +168,7 @@ class LIBPROTOBUF_EXPORT DynamicMapSorter {
  }

 private:
-  class LIBPROTOBUF_EXPORT MapEntryMessageComparator {
+  class PROTOBUF_EXPORT MapEntryMessageComparator {
   public:
    explicit MapEntryMessageComparator(const Descriptor* descriptor)
        : field_(descriptor->field(0)) {}
@@ -195,28 +182,28 @@ class LIBPROTOBUF_EXPORT DynamicMapSorter {
          return first < second;
        }
        case FieldDescriptor::CPPTYPE_INT32: {
-          int32 first = reflection->GetInt32(*a, field_);
-          int32 second = reflection->GetInt32(*b, field_);
+          int32_t first = reflection->GetInt32(*a, field_);
+          int32_t second = reflection->GetInt32(*b, field_);
          return first < second;
        }
        case FieldDescriptor::CPPTYPE_INT64: {
-          int64 first = reflection->GetInt64(*a, field_);
-          int64 second = reflection->GetInt64(*b, field_);
+          int64_t first = reflection->GetInt64(*a, field_);
+          int64_t second = reflection->GetInt64(*b, field_);
          return first < second;
        }
        case FieldDescriptor::CPPTYPE_UINT32: {
-          uint32 first = reflection->GetUInt32(*a, field_);
-          uint32 second = reflection->GetUInt32(*b, field_);
+          uint32_t first = reflection->GetUInt32(*a, field_);
+          uint32_t second = reflection->GetUInt32(*b, field_);
          return first < second;
        }
        case FieldDescriptor::CPPTYPE_UINT64: {
-          uint64 first = reflection->GetUInt64(*a, field_);
-          uint64 second = reflection->GetUInt64(*b, field_);
+          uint64_t first = reflection->GetUInt64(*a, field_);
+          uint64_t second = reflection->GetUInt64(*b, field_);
          return first < second;
        }
        case FieldDescriptor::CPPTYPE_STRING: {
-          string first = reflection->GetString(*a, field_);
-          string second = reflection->GetString(*b, field_);
+          std::string first = reflection->GetString(*a, field_);
+          std::string second = reflection->GetString(*b, field_);
          return first < second;
        }
        default:
@@ -231,6 +218,8 @@ class LIBPROTOBUF_EXPORT DynamicMapSorter {
 };

 }  // namespace protobuf
-
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
 #endif  // GOOGLE_PROTOBUF_DYNAMIC_MESSAGE_H__
@@ -1,342 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: google/protobuf/empty.proto
-
-#include <google/protobuf/empty.pb.h>
-
-#include <algorithm>
-
-#include <google/protobuf/stubs/common.h>
-#include <google/protobuf/stubs/port.h>
-#include <google/protobuf/stubs/once.h>
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/wire_format_lite_inl.h>
-#include <google/protobuf/descriptor.h>
-#include <google/protobuf/generated_message_reflection.h>
-#include <google/protobuf/reflection_ops.h>
-#include <google/protobuf/wire_format.h>
-// This is a temporary google only hack
-#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-#include "third_party/protobuf/version.h"
-#endif
-// @@protoc_insertion_point(includes)
-namespace google {
-namespace protobuf {
-class EmptyDefaultTypeInternal {
- public:
-  ::google::protobuf::internal::ExplicitlyConstructed<Empty>
-      _instance;
-} _Empty_default_instance_;
-}  // namespace protobuf
-}  // namespace google
-namespace protobuf_google_2fprotobuf_2fempty_2eproto {
-void InitDefaultsEmptyImpl() {
-  GOOGLE_PROTOBUF_VERIFY_VERSION;
-
-#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
-#else
-  ::google::protobuf::internal::InitProtobufDefaults();
-#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-  {
-    void* ptr = &::google::protobuf::_Empty_default_instance_;
-    new (ptr) ::google::protobuf::Empty();
-    ::google::protobuf::internal::OnShutdownDestroyMessage(ptr);
-  }
-  ::google::protobuf::Empty::InitAsDefaultInstance();
-}
-
-void InitDefaultsEmpty() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsEmptyImpl);
-}
-
-::google::protobuf::Metadata file_level_metadata[1];
-
-const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  ~0u,  // no _has_bits_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Empty, _internal_metadata_),
-  ~0u,  // no _extensions_
-  ~0u,  // no _oneof_case_
-  ~0u,  // no _weak_field_map_
-};
-static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  { 0, -1, sizeof(::google::protobuf::Empty)},
-};
-
-static ::google::protobuf::Message const * const file_default_instances[] = {
-  reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Empty_default_instance_),
-};
-
-void protobuf_AssignDescriptors() {
-  AddDescriptors();
-  ::google::protobuf::MessageFactory* factory = NULL;
-  AssignDescriptors(
-      "google/protobuf/empty.proto", schemas, file_default_instances, TableStruct::offsets, factory,
-      file_level_metadata, NULL, NULL);
-}
-
-void protobuf_AssignDescriptorsOnce() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &protobuf_AssignDescriptors);
-}
-
-void protobuf_RegisterTypes(const ::std::string&) GOOGLE_PROTOBUF_ATTRIBUTE_COLD;
-void protobuf_RegisterTypes(const ::std::string&) {
-  protobuf_AssignDescriptorsOnce();
-  ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1);
-}
-
-void AddDescriptorsImpl() {
-  InitDefaults();
-  static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-      "\n\033google/protobuf/empty.proto\022\017google.pr"
-      "otobuf\"\007\n\005EmptyBv\n\023com.google.protobufB\n"
-      "EmptyProtoP\001Z\'github.com/golang/protobuf"
-      "/ptypes/empty\370\001\001\242\002\003GPB\252\002\036Google.Protobuf"
-      ".WellKnownTypesb\006proto3"
-  };
-  ::google::protobuf::DescriptorPool::InternalAddGeneratedFile(
-      descriptor, 183);
-  ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
-    "google/protobuf/empty.proto", &protobuf_RegisterTypes);
-}
-
-void AddDescriptors() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &AddDescriptorsImpl);
-}
-// Force AddDescriptors() to be called at dynamic initialization time.
-struct StaticDescriptorInitializer {
-  StaticDescriptorInitializer() {
-    AddDescriptors();
-  }
-} static_descriptor_initializer;
-}  // namespace protobuf_google_2fprotobuf_2fempty_2eproto
-namespace google {
-namespace protobuf {
-
-// ===================================================================
-
-void Empty::InitAsDefaultInstance() {
-}
-#if !defined(_MSC_VER) || _MSC_VER >= 1900
-#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
-
-Empty::Empty()
-  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
-  if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) {
-    ::protobuf_google_2fprotobuf_2fempty_2eproto::InitDefaultsEmpty();
-  }
-  SharedCtor();
-  // @@protoc_insertion_point(constructor:google.protobuf.Empty)
-}
-Empty::Empty(::google::protobuf::Arena* arena)
-  : ::google::protobuf::Message(),
-  _internal_metadata_(arena) {
-  ::protobuf_google_2fprotobuf_2fempty_2eproto::InitDefaultsEmpty();
-  SharedCtor();
-  RegisterArenaDtor(arena);
-  // @@protoc_insertion_point(arena_constructor:google.protobuf.Empty)
-}
-Empty::Empty(const Empty& from)
-  : ::google::protobuf::Message(),
-      _internal_metadata_(NULL),
-      _cached_size_(0) {
-  _internal_metadata_.MergeFrom(from._internal_metadata_);
-  // @@protoc_insertion_point(copy_constructor:google.protobuf.Empty)
-}
-
-void Empty::SharedCtor() {
-  _cached_size_ = 0;
-}
-
-Empty::~Empty() {
-  // @@protoc_insertion_point(destructor:google.protobuf.Empty)
-  SharedDtor();
-}
-
-void Empty::SharedDtor() {
-  GOOGLE_DCHECK(GetArenaNoVirtual() == NULL);
-}
-
-void Empty::ArenaDtor(void* object) {
-  Empty* _this = reinterpret_cast< Empty* >(object);
-  (void)_this;
-}
-void Empty::RegisterArenaDtor(::google::protobuf::Arena* arena) {
-}
-void Empty::SetCachedSize(int size) const {
-  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
-  _cached_size_ = size;
-  GOOGLE_SAFE_CONCURRENT_WRITES_END();
-}
-const ::google::protobuf::Descriptor* Empty::descriptor() {
-  ::protobuf_google_2fprotobuf_2fempty_2eproto::protobuf_AssignDescriptorsOnce();
-  return ::protobuf_google_2fprotobuf_2fempty_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
-}
-
-const Empty& Empty::default_instance() {
-  ::protobuf_google_2fprotobuf_2fempty_2eproto::InitDefaultsEmpty();
-  return *internal_default_instance();
-}
-
-Empty* Empty::New(::google::protobuf::Arena* arena) const {
-  return ::google::protobuf::Arena::CreateMessage<Empty>(arena);
-}
-
-void Empty::Clear() {
-// @@protoc_insertion_point(message_clear_start:google.protobuf.Empty)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  // Prevent compiler warnings about cached_has_bits being unused
-  (void) cached_has_bits;
-
-  _internal_metadata_.Clear();
-}
-
-bool Empty::MergePartialFromCodedStream(
-    ::google::protobuf::io::CodedInputStream* input) {
-#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
-  ::google::protobuf::uint32 tag;
-  // @@protoc_insertion_point(parse_start:google.protobuf.Empty)
-  for (;;) {
-    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u);
-    tag = p.first;
-    if (!p.second) goto handle_unusual;
-  handle_unusual:
-    if (tag == 0) {
-      goto success;
-    }
-    DO_(::google::protobuf::internal::WireFormat::SkipField(
-          input, tag, _internal_metadata_.mutable_unknown_fields()));
-  }
-success:
-  // @@protoc_insertion_point(parse_success:google.protobuf.Empty)
-  return true;
-failure:
-  // @@protoc_insertion_point(parse_failure:google.protobuf.Empty)
-  return false;
-#undef DO_
-}
-
-void Empty::SerializeWithCachedSizes(
-    ::google::protobuf::io::CodedOutputStream* output) const {
-  // @@protoc_insertion_point(serialize_start:google.protobuf.Empty)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    ::google::protobuf::internal::WireFormat::SerializeUnknownFields(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()), output);
-  }
-  // @@protoc_insertion_point(serialize_end:google.protobuf.Empty)
-}
-
-::google::protobuf::uint8* Empty::InternalSerializeWithCachedSizesToArray(
-    bool deterministic, ::google::protobuf::uint8* target) const {
-  (void)deterministic; // Unused
-  // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.Empty)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()), target);
-  }
-  // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Empty)
-  return target;
-}
-
-size_t Empty::ByteSizeLong() const {
-// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Empty)
-  size_t total_size = 0;
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    total_size +=
-      ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()));
-  }
-  int cached_size = ::google::protobuf::internal::ToCachedSize(total_size);
-  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
-  _cached_size_ = cached_size;
-  GOOGLE_SAFE_CONCURRENT_WRITES_END();
-  return total_size;
-}
-
-void Empty::MergeFrom(const ::google::protobuf::Message& from) {
-// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.Empty)
-  GOOGLE_DCHECK_NE(&from, this);
-  const Empty* source =
-      ::google::protobuf::internal::DynamicCastToGenerated<const Empty>(
-          &from);
-  if (source == NULL) {
-  // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.Empty)
-    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
-  } else {
-  // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.Empty)
-    MergeFrom(*source);
-  }
-}
-
-void Empty::MergeFrom(const Empty& from) {
-// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.Empty)
-  GOOGLE_DCHECK_NE(&from, this);
-  _internal_metadata_.MergeFrom(from._internal_metadata_);
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-}
-
-void Empty::CopyFrom(const ::google::protobuf::Message& from) {
-// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.Empty)
-  if (&from == this) return;
-  Clear();
-  MergeFrom(from);
-}
-
-void Empty::CopyFrom(const Empty& from) {
-// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.Empty)
-  if (&from == this) return;
-  Clear();
-  MergeFrom(from);
-}
-
-bool Empty::IsInitialized() const {
-  return true;
-}
-
-void Empty::Swap(Empty* other) {
-  if (other == this) return;
-  if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) {
-    InternalSwap(other);
-  } else {
-    Empty* temp = New(GetArenaNoVirtual());
-    temp->MergeFrom(*other);
-    other->CopyFrom(*this);
-    InternalSwap(temp);
-    if (GetArenaNoVirtual() == NULL) {
-      delete temp;
-    }
-  }
-}
-void Empty::UnsafeArenaSwap(Empty* other) {
-  if (other == this) return;
-  GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual());
-  InternalSwap(other);
-}
-void Empty::InternalSwap(Empty* other) {
-  using std::swap;
-  _internal_metadata_.Swap(&other->_internal_metadata_);
-  swap(_cached_size_, other->_cached_size_);
-}
-
-::google::protobuf::Metadata Empty::GetMetadata() const {
-  protobuf_google_2fprotobuf_2fempty_2eproto::protobuf_AssignDescriptorsOnce();
-  return ::protobuf_google_2fprotobuf_2fempty_2eproto::file_level_metadata[kIndexInFileMessages];
-}
-
-
-// @@protoc_insertion_point(namespace_scope)
-}  // namespace protobuf
-}  // namespace google
-
-// @@protoc_insertion_point(global_scope)
@@ -1,190 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: google/protobuf/empty.proto
-
-#ifndef PROTOBUF_google_2fprotobuf_2fempty_2eproto__INCLUDED
-#define PROTOBUF_google_2fprotobuf_2fempty_2eproto__INCLUDED
-
-#include <string>
-
-#include <google/protobuf/stubs/common.h>
-
-#if GOOGLE_PROTOBUF_VERSION < 3005000
-#error This file was generated by a newer version of protoc which is
-#error incompatible with your Protocol Buffer headers.  Please update
-#error your headers.
-#endif
-#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION
-#error This file was generated by an older version of protoc which is
-#error incompatible with your Protocol Buffer headers.  Please
-#error regenerate this file with a newer version of protoc.
-#endif
-
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/arena.h>
-#include <google/protobuf/arenastring.h>
-#include <google/protobuf/generated_message_table_driven.h>
-#include <google/protobuf/generated_message_util.h>
-#include <google/protobuf/metadata.h>
-#include <google/protobuf/message.h>
-#include <google/protobuf/repeated_field.h>  // IWYU pragma: export
-#include <google/protobuf/extension_set.h>  // IWYU pragma: export
-#include <google/protobuf/unknown_field_set.h>
-// @@protoc_insertion_point(includes)
-
-namespace protobuf_google_2fprotobuf_2fempty_2eproto {
-// Internal implementation detail -- do not use these members.
-struct LIBPROTOBUF_EXPORT TableStruct {
-  static const ::google::protobuf::internal::ParseTableField entries[];
-  static const ::google::protobuf::internal::AuxillaryParseTableField aux[];
-  static const ::google::protobuf::internal::ParseTable schema[1];
-  static const ::google::protobuf::internal::FieldMetadata field_metadata[];
-  static const ::google::protobuf::internal::SerializationTable serialization_table[];
-  static const ::google::protobuf::uint32 offsets[];
-};
-void LIBPROTOBUF_EXPORT AddDescriptors();
-void LIBPROTOBUF_EXPORT InitDefaultsEmptyImpl();
-void LIBPROTOBUF_EXPORT InitDefaultsEmpty();
-inline void LIBPROTOBUF_EXPORT InitDefaults() {
-  InitDefaultsEmpty();
-}
-}  // namespace protobuf_google_2fprotobuf_2fempty_2eproto
-namespace google {
-namespace protobuf {
-class Empty;
-class EmptyDefaultTypeInternal;
-LIBPROTOBUF_EXPORT extern EmptyDefaultTypeInternal _Empty_default_instance_;
-}  // namespace protobuf
-}  // namespace google
-namespace google {
-namespace protobuf {
-
-// ===================================================================
-
-class LIBPROTOBUF_EXPORT Empty : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Empty) */ {
- public:
-  Empty();
-  virtual ~Empty();
-
-  Empty(const Empty& from);
-
-  inline Empty& operator=(const Empty& from) {
-    CopyFrom(from);
-    return *this;
-  }
-  #if LANG_CXX11
-  Empty(Empty&& from) noexcept
-    : Empty() {
-    *this = ::std::move(from);
-  }
-
-  inline Empty& operator=(Empty&& from) noexcept {
-    if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
-      if (this != &from) InternalSwap(&from);
-    } else {
-      CopyFrom(from);
-    }
-    return *this;
-  }
-  #endif
-  inline ::google::protobuf::Arena* GetArena() const PROTOBUF_FINAL {
-    return GetArenaNoVirtual();
-  }
-  inline void* GetMaybeArenaPointer() const PROTOBUF_FINAL {
-    return MaybeArenaPtr();
-  }
-  static const ::google::protobuf::Descriptor* descriptor();
-  static const Empty& default_instance();
-
-  static void InitAsDefaultInstance();  // FOR INTERNAL USE ONLY
-  static inline const Empty* internal_default_instance() {
-    return reinterpret_cast<const Empty*>(
-               &_Empty_default_instance_);
-  }
-  static PROTOBUF_CONSTEXPR int const kIndexInFileMessages =
-    0;
-
-  void UnsafeArenaSwap(Empty* other);
-  void Swap(Empty* other);
-  friend void swap(Empty& a, Empty& b) {
-    a.Swap(&b);
-  }
-
-  // implements Message ----------------------------------------------
-
-  inline Empty* New() const PROTOBUF_FINAL { return New(NULL); }
-
-  Empty* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL;
-  void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
-  void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
-  void CopyFrom(const Empty& from);
-  void MergeFrom(const Empty& from);
-  void Clear() PROTOBUF_FINAL;
-  bool IsInitialized() const PROTOBUF_FINAL;
-
-  size_t ByteSizeLong() const PROTOBUF_FINAL;
-  bool MergePartialFromCodedStream(
-      ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL;
-  void SerializeWithCachedSizes(
-      ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL;
-  ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(
-      bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL;
-  int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; }
-  private:
-  void SharedCtor();
-  void SharedDtor();
-  void SetCachedSize(int size) const PROTOBUF_FINAL;
-  void InternalSwap(Empty* other);
-  protected:
-  explicit Empty(::google::protobuf::Arena* arena);
-  private:
-  static void ArenaDtor(void* object);
-  inline void RegisterArenaDtor(::google::protobuf::Arena* arena);
-  private:
-  inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
-    return _internal_metadata_.arena();
-  }
-  inline void* MaybeArenaPtr() const {
-    return _internal_metadata_.raw_arena_ptr();
-  }
-  public:
-
-  ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL;
-
-  // nested types ----------------------------------------------------
-
-  // accessors -------------------------------------------------------
-
-  // @@protoc_insertion_point(class_scope:google.protobuf.Empty)
- private:
-
-  ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_;
-  template <typename T> friend class ::google::protobuf::Arena::InternalHelper;
-  typedef void InternalArenaConstructable_;
-  typedef void DestructorSkippable_;
-  mutable int _cached_size_;
-  friend struct ::protobuf_google_2fprotobuf_2fempty_2eproto::TableStruct;
-  friend void ::protobuf_google_2fprotobuf_2fempty_2eproto::InitDefaultsEmptyImpl();
-};
-// ===================================================================
-
-
-// ===================================================================
-
-#ifdef __GNUC__
-  #pragma GCC diagnostic push
-  #pragma GCC diagnostic ignored "-Wstrict-aliasing"
-#endif  // __GNUC__
-// Empty
-
-#ifdef __GNUC__
-  #pragma GCC diagnostic pop
-#endif  // __GNUC__
-
-// @@protoc_insertion_point(namespace_scope)
-
-}  // namespace protobuf
-}  // namespace google
-
-// @@protoc_insertion_point(global_scope)
-
-#endif  // PROTOBUF_google_2fprotobuf_2fempty_2eproto__INCLUDED
@@ -27,49 +27,65 @@
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#ifndef GOOGLE_PROTOBUF_STUBS_TIME_H_
-#define GOOGLE_PROTOBUF_STUBS_TIME_H_

+#ifndef GOOGLE_PROTOBUF_EXPLICITLY_CONSTRUCTED_H__
+#define GOOGLE_PROTOBUF_EXPLICITLY_CONSTRUCTED_H__
+
+#include <stdint.h>
+
+#include <utility>
+
+#include <google/protobuf/stubs/logging.h>
 #include <google/protobuf/stubs/common.h>

+// clang-format off
+#include <google/protobuf/port_def.inc>
+// clang-format on
+
 namespace google {
 namespace protobuf {
 namespace internal {

-struct DateTime {
-  int year;
-  int month;
-  int day;
-  int hour;
-  int minute;
-  int second;
+// Wraps a variable whose constructor and destructor are explicitly
+// called. It is particularly useful for a global variable, without its
+// constructor and destructor run on start and end of the program lifetime.
+// This circumvents the initial construction order fiasco, while keeping
+// the address of the empty string a compile time constant.
+//
+// Pay special attention to the initialization state of the object.
+// 1. The object is "uninitialized" to begin with.
+// 2. Call Construct() or DefaultConstruct() only if the object is
+//    uninitialized. After the call, the object becomes "initialized".
+// 3. Call get() and get_mutable() only if the object is initialized.
+// 4. Call Destruct() only if the object is initialized.
+//    After the call, the object becomes uninitialized.
+template <typename T>
+class ExplicitlyConstructed {
+ public:
+  void DefaultConstruct() { new (&union_) T(); }
+
+  template <typename... Args>
+  void Construct(Args&&... args) {
+    new (&union_) T(std::forward<Args>(args)...);
+  }
+
+  void Destruct() { get_mutable()->~T(); }
+
+  constexpr const T& get() const { return reinterpret_cast<const T&>(union_); }
+  T* get_mutable() { return reinterpret_cast<T*>(&union_); }
+
+ private:
+  union AlignedUnion {
+    alignas(T) char space[sizeof(T)];
+    int64_t align_to_int64;
+    void* align_to_ptr;
+  } union_;
 };

-// Converts a timestamp (seconds elapsed since 1970-01-01T00:00:00, could be
-// negative to represent time before 1970-01-01) to DateTime. Returns false
-// if the timestamp is not in the range between 0001-01-01T00:00:00 and
-// 9999-12-31T23:59:59.
-bool LIBPROTOBUF_EXPORT SecondsToDateTime(int64 seconds, DateTime* time);
-// Converts DateTime to a timestamp (seconds since 1970-01-01T00:00:00).
-// Returns false if the DateTime is not valid or is not in the valid range.
-bool LIBPROTOBUF_EXPORT DateTimeToSeconds(const DateTime& time, int64* seconds);
-
-void LIBPROTOBUF_EXPORT GetCurrentTime(int64* seconds, int32* nanos);
-
-// Formats a time string in RFC3339 fromat.
-//
-// For example, "2015-05-20T13:29:35.120Z". For nanos, 0, 3, 6 or 9 fractional
-// digits will be used depending on how many are required to represent the exact
-// value.
-//
-// Note that "nanos" must in the range of [0, 999999999].
-string LIBPROTOBUF_EXPORT FormatTime(int64 seconds, int32 nanos);
-// Parses a time string. This method accepts RFC3339 date/time string with UTC
-// offset. For example, "2015-05-20T13:29:35.120-08:00".
-bool LIBPROTOBUF_EXPORT ParseTime(const string& value, int64* seconds, int32* nanos);
-
 }  // namespace internal
 }  // namespace protobuf
 }  // namespace google

-#endif  // GOOGLE_PROTOBUF_STUBS_TIME_H_
+#include <google/protobuf/port_undef.inc>
+
+#endif  // GOOGLE_PROTOBUF_EXPLICITLY_CONSTRUCTED_H__
@@ -35,42 +35,47 @@
 // Contains methods defined in extension_set.h which cannot be part of the
 // lite library because they use descriptors or reflection.

-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <google/protobuf/stubs/casts.h>
 #include <google/protobuf/descriptor.pb.h>
+#include <google/protobuf/extension_set_inl.h>
+#include <google/protobuf/parse_context.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/arena.h>
 #include <google/protobuf/descriptor.h>
 #include <google/protobuf/extension_set.h>
 #include <google/protobuf/message.h>
+#include <google/protobuf/message_lite.h>
 #include <google/protobuf/repeated_field.h>
 #include <google/protobuf/unknown_field_set.h>
 #include <google/protobuf/wire_format.h>
-#include <google/protobuf/wire_format_lite_inl.h>
+#include <google/protobuf/wire_format_lite.h>
+
+
+#include <google/protobuf/port_def.inc>

 namespace google {
-
 namespace protobuf {
 namespace internal {

 // A FieldSkipper used to store unknown MessageSet fields into UnknownFieldSet.
-class MessageSetFieldSkipper
-    : public UnknownFieldSetFieldSkipper {
+class MessageSetFieldSkipper : public UnknownFieldSetFieldSkipper {
 public:
  explicit MessageSetFieldSkipper(UnknownFieldSet* unknown_fields)
      : UnknownFieldSetFieldSkipper(unknown_fields) {}
-  virtual ~MessageSetFieldSkipper() {}
+  ~MessageSetFieldSkipper() override {}

  virtual bool SkipMessageSetField(io::CodedInputStream* input,
                                   int field_number);
 };
-bool MessageSetFieldSkipper::SkipMessageSetField(
-    io::CodedInputStream* input, int field_number) {
-  uint32 length;
+bool MessageSetFieldSkipper::SkipMessageSetField(io::CodedInputStream* input,
+                                                 int field_number) {
+  uint32_t length;
  if (!input->ReadVarint32(&length)) return false;
-  if (unknown_fields_ == NULL) {
+  if (unknown_fields_ == nullptr) {
    return input->Skip(length);
  } else {
-    return input->ReadString(
-        unknown_fields_->AddLengthDelimited(field_number), length);
+    return input->ReadString(unknown_fields_->AddLengthDelimited(field_number),
+                             length);
  }
 }

@@ -84,9 +89,9 @@ class DescriptorPoolExtensionFinder : public ExtensionFinder {
                                MessageFactory* factory,
                                const Descriptor* containing_type)
      : pool_(pool), factory_(factory), containing_type_(containing_type) {}
-  virtual ~DescriptorPoolExtensionFinder() {}
+  ~DescriptorPoolExtensionFinder() override {}

-  virtual bool Find(int number, ExtensionInfo* output);
+  bool Find(int number, ExtensionInfo* output) override;

 private:
  const DescriptorPool* pool_;
@@ -95,16 +100,14 @@ class DescriptorPoolExtensionFinder : public ExtensionFinder {
 };

 void ExtensionSet::AppendToList(
-    const Descriptor* containing_type,
-    const DescriptorPool* pool,
+    const Descriptor* containing_type, const DescriptorPool* pool,
    std::vector<const FieldDescriptor*>* output) const {
-  for (ExtensionMap::const_iterator iter = extensions_.begin();
-       iter != extensions_.end(); ++iter) {
+  ForEach([containing_type, pool, &output](int number, const Extension& ext) {
    bool has = false;
-    if (iter->second.is_repeated) {
-      has = iter->second.GetSize() > 0;
+    if (ext.is_repeated) {
+      has = ext.GetSize() > 0;
    } else {
-      has = !iter->second.is_cleared;
+      has = !ext.is_cleared;
    }

    if (has) {
@@ -113,14 +116,13 @@ void ExtensionSet::AppendToList(
      //   initialized, so they might not even be constructed until
      //   AppendToList() is called.

-      if (iter->second.descriptor == NULL) {
-        output->push_back(pool->FindExtensionByNumber(
-            containing_type, iter->first));
+      if (ext.descriptor == nullptr) {
+        output->push_back(pool->FindExtensionByNumber(containing_type, number));
      } else {
-        output->push_back(iter->second.descriptor);
+        output->push_back(ext.descriptor);
      }
    }
-  }
+  });
 }

 inline FieldDescriptor::Type real_type(FieldType type) {
@@ -138,26 +140,26 @@ inline WireFormatLite::FieldType field_type(FieldType type) {
  return static_cast<WireFormatLite::FieldType>(type);
 }

-#define GOOGLE_DCHECK_TYPE(EXTENSION, LABEL, CPPTYPE)                            \
-  GOOGLE_DCHECK_EQ((EXTENSION).is_repeated ? FieldDescriptor::LABEL_REPEATED     \
-                                  : FieldDescriptor::LABEL_OPTIONAL,      \
-            FieldDescriptor::LABEL_##LABEL);                              \
+#define GOOGLE_DCHECK_TYPE(EXTENSION, LABEL, CPPTYPE)                         \
+  GOOGLE_DCHECK_EQ((EXTENSION).is_repeated ? FieldDescriptor::LABEL_REPEATED  \
+                                    : FieldDescriptor::LABEL_OPTIONAL, \
+            FieldDescriptor::LABEL_##LABEL);                           \
  GOOGLE_DCHECK_EQ(cpp_type((EXTENSION).type), FieldDescriptor::CPPTYPE_##CPPTYPE)

 const MessageLite& ExtensionSet::GetMessage(int number,
                                            const Descriptor* message_type,
                                            MessageFactory* factory) const {
-  ExtensionMap::const_iterator iter = extensions_.find(number);
-  if (iter == extensions_.end() || iter->second.is_cleared) {
+  const Extension* extension = FindOrNull(number);
+  if (extension == nullptr || extension->is_cleared) {
    // Not present.  Return the default value.
    return *factory->GetPrototype(message_type);
  } else {
-    GOOGLE_DCHECK_TYPE(iter->second, OPTIONAL, MESSAGE);
-    if (iter->second.is_lazy) {
-      return iter->second.lazymessage_value->GetMessage(
-          *factory->GetPrototype(message_type));
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE);
+    if (extension->is_lazy) {
+      return extension->lazymessage_value->GetMessage(
+          *factory->GetPrototype(message_type), arena_);
    } else {
-      return *iter->second.message_value;
+      return *extension->message_value;
    }
  }
 }
@@ -181,7 +183,7 @@ MessageLite* ExtensionSet::MutableMessage(const FieldDescriptor* descriptor,
    extension->is_cleared = false;
    if (extension->is_lazy) {
      return extension->lazymessage_value->MutableMessage(
-          *factory->GetPrototype(descriptor->message_type()));
+          *factory->GetPrototype(descriptor->message_type()), arena_);
    } else {
      return extension->message_value;
    }
@@ -190,63 +192,64 @@ MessageLite* ExtensionSet::MutableMessage(const FieldDescriptor* descriptor,

 MessageLite* ExtensionSet::ReleaseMessage(const FieldDescriptor* descriptor,
                                          MessageFactory* factory) {
-  ExtensionMap::iterator iter = extensions_.find(descriptor->number());
-  if (iter == extensions_.end()) {
-    // Not present.  Return NULL.
-    return NULL;
+  Extension* extension = FindOrNull(descriptor->number());
+  if (extension == nullptr) {
+    // Not present.  Return nullptr.
+    return nullptr;
  } else {
-    GOOGLE_DCHECK_TYPE(iter->second, OPTIONAL, MESSAGE);
-    MessageLite* ret = NULL;
-    if (iter->second.is_lazy) {
-      ret = iter->second.lazymessage_value->ReleaseMessage(
-          *factory->GetPrototype(descriptor->message_type()));
-      if (arena_ == NULL) {
-        delete iter->second.lazymessage_value;
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE);
+    MessageLite* ret = nullptr;
+    if (extension->is_lazy) {
+      ret = extension->lazymessage_value->ReleaseMessage(
+          *factory->GetPrototype(descriptor->message_type()), arena_);
+      if (arena_ == nullptr) {
+        delete extension->lazymessage_value;
      }
    } else {
-      if (arena_ != NULL) {
-        ret = (iter->second.message_value)->New();
-        ret->CheckTypeAndMergeFrom(*(iter->second.message_value));
+      if (arena_ != nullptr) {
+        ret = extension->message_value->New();
+        ret->CheckTypeAndMergeFrom(*extension->message_value);
      } else {
-        ret = iter->second.message_value;
+        ret = extension->message_value;
      }
    }
-    extensions_.erase(descriptor->number());
+    Erase(descriptor->number());
    return ret;
  }
 }

 MessageLite* ExtensionSet::UnsafeArenaReleaseMessage(
    const FieldDescriptor* descriptor, MessageFactory* factory) {
-  ExtensionMap::iterator iter = extensions_.find(descriptor->number());
-  if (iter == extensions_.end()) {
-    // Not present.  Return NULL.
-    return NULL;
+  Extension* extension = FindOrNull(descriptor->number());
+  if (extension == nullptr) {
+    // Not present.  Return nullptr.
+    return nullptr;
  } else {
-    GOOGLE_DCHECK_TYPE(iter->second, OPTIONAL, MESSAGE);
-    MessageLite* ret = NULL;
-    if (iter->second.is_lazy) {
-      ret = iter->second.lazymessage_value->UnsafeArenaReleaseMessage(
-          *factory->GetPrototype(descriptor->message_type()));
-      if (arena_ == NULL) {
-        delete iter->second.lazymessage_value;
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE);
+    MessageLite* ret = nullptr;
+    if (extension->is_lazy) {
+      ret = extension->lazymessage_value->UnsafeArenaReleaseMessage(
+          *factory->GetPrototype(descriptor->message_type()), arena_);
+      if (arena_ == nullptr) {
+        delete extension->lazymessage_value;
      }
    } else {
-      ret = iter->second.message_value;
+      ret = extension->message_value;
    }
-    extensions_.erase(descriptor->number());
+    Erase(descriptor->number());
    return ret;
  }
 }

-ExtensionSet::Extension* ExtensionSet::MaybeNewRepeatedExtension(const FieldDescriptor* descriptor) {
+ExtensionSet::Extension* ExtensionSet::MaybeNewRepeatedExtension(
+    const FieldDescriptor* descriptor) {
  Extension* extension;
  if (MaybeNewExtension(descriptor->number(), descriptor, &extension)) {
    extension->type = descriptor->type();
    GOOGLE_DCHECK_EQ(cpp_type(extension->type), FieldDescriptor::CPPTYPE_MESSAGE);
    extension->is_repeated = true;
    extension->repeated_message_value =
-        ::google::protobuf::Arena::CreateMessage<RepeatedPtrField<MessageLite> >(arena_);
+        Arena::CreateMessage<RepeatedPtrField<MessageLite> >(arena_);
  } else {
    GOOGLE_DCHECK_TYPE(*extension, REPEATED, MESSAGE);
  }
@@ -259,13 +262,15 @@ MessageLite* ExtensionSet::AddMessage(const FieldDescriptor* descriptor,

  // RepeatedPtrField<Message> does not know how to Add() since it cannot
  // allocate an abstract object, so we have to be tricky.
-  MessageLite* result = extension->repeated_message_value
-      ->AddFromCleared<GenericTypeHandler<MessageLite> >();
-  if (result == NULL) {
+  MessageLite* result =
+      reinterpret_cast<internal::RepeatedPtrFieldBase*>(
+          extension->repeated_message_value)
+          ->AddFromCleared<GenericTypeHandler<MessageLite> >();
+  if (result == nullptr) {
    const MessageLite* prototype;
-    if (extension->repeated_message_value->size() == 0) {
+    if (extension->repeated_message_value->empty()) {
      prototype = factory->GetPrototype(descriptor->message_type());
-      GOOGLE_CHECK(prototype != NULL);
+      GOOGLE_CHECK(prototype != nullptr);
    } else {
      prototype = &extension->repeated_message_value->Get(0);
    }
@@ -282,15 +287,22 @@ void ExtensionSet::AddAllocatedMessage(const FieldDescriptor* descriptor,
  extension->repeated_message_value->AddAllocated(new_entry);
 }

+void ExtensionSet::UnsafeArenaAddAllocatedMessage(
+    const FieldDescriptor* descriptor, MessageLite* new_entry) {
+  Extension* extension = MaybeNewRepeatedExtension(descriptor);
+
+  extension->repeated_message_value->UnsafeArenaAddAllocated(new_entry);
+}
+
 static bool ValidateEnumUsingDescriptor(const void* arg, int number) {
-  return reinterpret_cast<const EnumDescriptor*>(arg)
-      ->FindValueByNumber(number) != NULL;
+  return reinterpret_cast<const EnumDescriptor*>(arg)->FindValueByNumber(
+             number) != nullptr;
 }

 bool DescriptorPoolExtensionFinder::Find(int number, ExtensionInfo* output) {
  const FieldDescriptor* extension =
      pool_->FindExtensionByNumber(containing_type_, number);
-  if (extension == NULL) {
+  if (extension == nullptr) {
    return false;
  } else {
    output->type = extension->type();
@@ -298,10 +310,10 @@ bool DescriptorPoolExtensionFinder::Find(int number, ExtensionInfo* output) {
    output->is_packed = extension->options().packed();
    output->descriptor = extension;
    if (extension->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
-      output->message_prototype =
+      output->message_info.prototype =
          factory_->GetPrototype(extension->message_type());
-      GOOGLE_CHECK(output->message_prototype != NULL)
-          << "Extension factory's GetPrototype() returned NULL for extension: "
+      GOOGLE_CHECK(output->message_info.prototype != nullptr)
+          << "Extension factory's GetPrototype() returned nullptr; extension: "
          << extension->full_name();
    } else if (extension->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) {
      output->enum_validity_check.func = ValidateEnumUsingDescriptor;
@@ -312,11 +324,63 @@ bool DescriptorPoolExtensionFinder::Find(int number, ExtensionInfo* output) {
  }
 }

-bool ExtensionSet::ParseField(uint32 tag, io::CodedInputStream* input,
+
+bool ExtensionSet::FindExtension(int wire_type, uint32_t field,
+                                 const Message* containing_type,
+                                 const internal::ParseContext* ctx,
+                                 ExtensionInfo* extension,
+                                 bool* was_packed_on_wire) {
+  if (ctx->data().pool == nullptr) {
+    GeneratedExtensionFinder finder(containing_type);
+    if (!FindExtensionInfoFromFieldNumber(wire_type, field, &finder, extension,
+                                          was_packed_on_wire)) {
+      return false;
+    }
+  } else {
+    DescriptorPoolExtensionFinder finder(ctx->data().pool, ctx->data().factory,
+                                         containing_type->GetDescriptor());
+    if (!FindExtensionInfoFromFieldNumber(wire_type, field, &finder, extension,
+                                          was_packed_on_wire)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+const char* ExtensionSet::ParseField(uint64_t tag, const char* ptr,
+                                     const Message* containing_type,
+                                     internal::InternalMetadata* metadata,
+                                     internal::ParseContext* ctx) {
+  int number = tag >> 3;
+  bool was_packed_on_wire;
+  ExtensionInfo extension;
+  if (!FindExtension(tag & 7, number, containing_type, ctx, &extension,
+                     &was_packed_on_wire)) {
+    return UnknownFieldParse(
+        tag, metadata->mutable_unknown_fields<UnknownFieldSet>(), ptr, ctx);
+  }
+  return ParseFieldWithExtensionInfo<UnknownFieldSet>(
+      number, was_packed_on_wire, extension, metadata, ptr, ctx);
+}
+
+const char* ExtensionSet::ParseFieldMaybeLazily(
+    uint64_t tag, const char* ptr, const Message* containing_type,
+    internal::InternalMetadata* metadata, internal::ParseContext* ctx) {
+  return ParseField(tag, ptr, containing_type, metadata, ctx);
+}
+
+const char* ExtensionSet::ParseMessageSetItem(
+    const char* ptr, const Message* containing_type,
+    internal::InternalMetadata* metadata, internal::ParseContext* ctx) {
+  return ParseMessageSetItemTmpl<Message, UnknownFieldSet>(ptr, containing_type,
+                                                           metadata, ctx);
+}
+
+bool ExtensionSet::ParseField(uint32_t tag, io::CodedInputStream* input,
                              const Message* containing_type,
                              UnknownFieldSet* unknown_fields) {
  UnknownFieldSetFieldSkipper skipper(unknown_fields);
-  if (input->GetExtensionPool() == NULL) {
+  if (input->GetExtensionPool() == nullptr) {
    GeneratedExtensionFinder finder(containing_type);
    return ParseField(tag, input, &finder, &skipper);
  } else {
@@ -331,7 +395,7 @@ bool ExtensionSet::ParseMessageSet(io::CodedInputStream* input,
                                   const Message* containing_type,
                                   UnknownFieldSet* unknown_fields) {
  MessageSetFieldSkipper skipper(unknown_fields);
-  if (input->GetExtensionPool() == NULL) {
+  if (input->GetExtensionPool() == nullptr) {
    GeneratedExtensionFinder finder(containing_type);
    return ParseMessageSet(input, &finder, &skipper);
  } else {
@@ -347,14 +411,10 @@ int ExtensionSet::SpaceUsedExcludingSelf() const {
 }

 size_t ExtensionSet::SpaceUsedExcludingSelfLong() const {
-  size_t total_size =
-      extensions_.size() * sizeof(ExtensionMap::value_type);
-  for (ExtensionMap::const_iterator iter = extensions_.begin(),
-       end = extensions_.end();
-       iter != end;
-       ++iter) {
-    total_size += iter->second.SpaceUsedExcludingSelfLong();
-  }
+  size_t total_size = Size() * sizeof(KeyValue);
+  ForEach([&total_size](int /* number */, const Extension& ext) {
+    total_size += ext.SpaceUsedExcludingSelfLong();
+  });
  return total_size;
 }

@@ -373,15 +433,15 @@ size_t ExtensionSet::Extension::SpaceUsedExcludingSelfLong() const {
                  repeated_##LOWERCASE##_value->SpaceUsedExcludingSelfLong(); \
    break

-      HANDLE_TYPE(  INT32,   int32);
-      HANDLE_TYPE(  INT64,   int64);
-      HANDLE_TYPE( UINT32,  uint32);
-      HANDLE_TYPE( UINT64,  uint64);
-      HANDLE_TYPE(  FLOAT,   float);
-      HANDLE_TYPE( DOUBLE,  double);
-      HANDLE_TYPE(   BOOL,    bool);
-      HANDLE_TYPE(   ENUM,    enum);
-      HANDLE_TYPE( STRING,  string);
+      HANDLE_TYPE(INT32, int32_t);
+      HANDLE_TYPE(INT64, int64_t);
+      HANDLE_TYPE(UINT32, uint32_t);
+      HANDLE_TYPE(UINT64, uint64_t);
+      HANDLE_TYPE(FLOAT, float);
+      HANDLE_TYPE(DOUBLE, double);
+      HANDLE_TYPE(BOOL, bool);
+      HANDLE_TYPE(ENUM, enum);
+      HANDLE_TYPE(STRING, string);
 #undef HANDLE_TYPE

      case FieldDescriptor::CPPTYPE_MESSAGE:
@@ -389,9 +449,10 @@ size_t ExtensionSet::Extension::SpaceUsedExcludingSelfLong() const {
        // but MessageLite has no SpaceUsedLong(), so we must directly call
        // RepeatedPtrFieldBase::SpaceUsedExcludingSelfLong() with a different
        // type handler.
-        total_size +=
-            sizeof(*repeated_message_value) +
-            RepeatedMessage_SpaceUsedExcludingSelfLong(repeated_message_value);
+        total_size += sizeof(*repeated_message_value) +
+                      RepeatedMessage_SpaceUsedExcludingSelfLong(
+                          reinterpret_cast<internal::RepeatedPtrFieldBase*>(
+                              repeated_message_value));
        break;
    }
  } else {
@@ -415,218 +476,29 @@ size_t ExtensionSet::Extension::SpaceUsedExcludingSelfLong() const {
  return total_size;
 }

-// The Serialize*ToArray methods are only needed in the heavy library, as
-// the lite library only generates SerializeWithCachedSizes.
-uint8* ExtensionSet::SerializeWithCachedSizesToArray(int start_field_number,
-                                                     int end_field_number,
-                                                     uint8* target) const {
-  return InternalSerializeWithCachedSizesToArray(
-      start_field_number, end_field_number,
-      google::protobuf::io::CodedOutputStream::IsDefaultSerializationDeterministic(),
-      target);
+uint8_t* ExtensionSet::SerializeMessageSetWithCachedSizesToArray(
+    const MessageLite* extendee, uint8_t* target) const {
+  io::EpsCopyOutputStream stream(
+      target, MessageSetByteSize(),
+      io::CodedOutputStream::IsDefaultSerializationDeterministic());
+  return InternalSerializeMessageSetWithCachedSizesToArray(extendee, target,
+                                                           &stream);
 }

-uint8* ExtensionSet::SerializeMessageSetWithCachedSizesToArray(
-    uint8* target) const {
-  return InternalSerializeMessageSetWithCachedSizesToArray(
-      google::protobuf::io::CodedOutputStream::IsDefaultSerializationDeterministic(),
-      target);
-}
-
-uint8* ExtensionSet::InternalSerializeWithCachedSizesToArray(
-    int start_field_number, int end_field_number,
-    bool deterministic, uint8* target) const {
-  ExtensionMap::const_iterator iter;
-  for (iter = extensions_.lower_bound(start_field_number);
-       iter != extensions_.end() && iter->first < end_field_number;
-       ++iter) {
-    target = iter->second.InternalSerializeFieldWithCachedSizesToArray(
-        iter->first, deterministic, target);
-  }
-  return target;
-}
-
-uint8* ExtensionSet::InternalSerializeMessageSetWithCachedSizesToArray(
-    bool deterministic, uint8* target) const {
-  ExtensionMap::const_iterator iter;
-  for (iter = extensions_.begin(); iter != extensions_.end(); ++iter) {
-    target = iter->second.InternalSerializeMessageSetItemWithCachedSizesToArray(
-        iter->first, deterministic, target);
-  }
-  return target;
-}
-
-uint8* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray(
-    int number, bool deterministic, uint8* target) const {
-  if (is_repeated) {
-    if (is_packed) {
-      if (cached_size == 0) return target;
-
-      target = WireFormatLite::WriteTagToArray(number,
-          WireFormatLite::WIRETYPE_LENGTH_DELIMITED, target);
-      target = WireFormatLite::WriteInt32NoTagToArray(cached_size, target);
-
-      switch (real_type(type)) {
-#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE)                        \
-        case FieldDescriptor::TYPE_##UPPERCASE:                             \
-          for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) {  \
-            target = WireFormatLite::Write##CAMELCASE##NoTagToArray(        \
-              repeated_##LOWERCASE##_value->Get(i), target);                \
-          }                                                                 \
-          break
-
-        HANDLE_TYPE(   INT32,    Int32,   int32);
-        HANDLE_TYPE(   INT64,    Int64,   int64);
-        HANDLE_TYPE(  UINT32,   UInt32,  uint32);
-        HANDLE_TYPE(  UINT64,   UInt64,  uint64);
-        HANDLE_TYPE(  SINT32,   SInt32,   int32);
-        HANDLE_TYPE(  SINT64,   SInt64,   int64);
-        HANDLE_TYPE( FIXED32,  Fixed32,  uint32);
-        HANDLE_TYPE( FIXED64,  Fixed64,  uint64);
-        HANDLE_TYPE(SFIXED32, SFixed32,   int32);
-        HANDLE_TYPE(SFIXED64, SFixed64,   int64);
-        HANDLE_TYPE(   FLOAT,    Float,   float);
-        HANDLE_TYPE(  DOUBLE,   Double,  double);
-        HANDLE_TYPE(    BOOL,     Bool,    bool);
-        HANDLE_TYPE(    ENUM,     Enum,    enum);
-#undef HANDLE_TYPE
-
-        case FieldDescriptor::TYPE_STRING:
-        case FieldDescriptor::TYPE_BYTES:
-        case FieldDescriptor::TYPE_GROUP:
-        case FieldDescriptor::TYPE_MESSAGE:
-          GOOGLE_LOG(FATAL) << "Non-primitive types can't be packed.";
-          break;
-      }
-    } else {
-      switch (real_type(type)) {
-#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE)                        \
-        case FieldDescriptor::TYPE_##UPPERCASE:                             \
-          for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) {  \
-            target = WireFormatLite::Write##CAMELCASE##ToArray(number,      \
-              repeated_##LOWERCASE##_value->Get(i), target);                \
-          }                                                                 \
-          break
-
-        HANDLE_TYPE(   INT32,    Int32,   int32);
-        HANDLE_TYPE(   INT64,    Int64,   int64);
-        HANDLE_TYPE(  UINT32,   UInt32,  uint32);
-        HANDLE_TYPE(  UINT64,   UInt64,  uint64);
-        HANDLE_TYPE(  SINT32,   SInt32,   int32);
-        HANDLE_TYPE(  SINT64,   SInt64,   int64);
-        HANDLE_TYPE( FIXED32,  Fixed32,  uint32);
-        HANDLE_TYPE( FIXED64,  Fixed64,  uint64);
-        HANDLE_TYPE(SFIXED32, SFixed32,   int32);
-        HANDLE_TYPE(SFIXED64, SFixed64,   int64);
-        HANDLE_TYPE(   FLOAT,    Float,   float);
-        HANDLE_TYPE(  DOUBLE,   Double,  double);
-        HANDLE_TYPE(    BOOL,     Bool,    bool);
-        HANDLE_TYPE(  STRING,   String,  string);
-        HANDLE_TYPE(   BYTES,    Bytes,  string);
-        HANDLE_TYPE(    ENUM,     Enum,    enum);
-#undef HANDLE_TYPE
-#define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE)                        \
-        case FieldDescriptor::TYPE_##UPPERCASE:                             \
-          for (int i = 0; i < repeated_##LOWERCASE##_value->size(); i++) {  \
-            target = WireFormatLite::InternalWrite##CAMELCASE##ToArray(     \
-                      number, repeated_##LOWERCASE##_value->Get(i),         \
-                      deterministic, target);                               \
-          }                                                                 \
-          break
-
-        HANDLE_TYPE(   GROUP,    Group, message);
-        HANDLE_TYPE( MESSAGE,  Message, message);
-#undef HANDLE_TYPE
-      }
-    }
-  } else if (!is_cleared) {
-    switch (real_type(type)) {
-#define HANDLE_TYPE(UPPERCASE, CAMELCASE, VALUE)                 \
-      case FieldDescriptor::TYPE_##UPPERCASE:                    \
-        target = WireFormatLite::Write##CAMELCASE##ToArray(      \
-            number, VALUE, target); \
-        break
-
-      HANDLE_TYPE(   INT32,    Int32,    int32_value);
-      HANDLE_TYPE(   INT64,    Int64,    int64_value);
-      HANDLE_TYPE(  UINT32,   UInt32,   uint32_value);
-      HANDLE_TYPE(  UINT64,   UInt64,   uint64_value);
-      HANDLE_TYPE(  SINT32,   SInt32,    int32_value);
-      HANDLE_TYPE(  SINT64,   SInt64,    int64_value);
-      HANDLE_TYPE( FIXED32,  Fixed32,   uint32_value);
-      HANDLE_TYPE( FIXED64,  Fixed64,   uint64_value);
-      HANDLE_TYPE(SFIXED32, SFixed32,    int32_value);
-      HANDLE_TYPE(SFIXED64, SFixed64,    int64_value);
-      HANDLE_TYPE(   FLOAT,    Float,    float_value);
-      HANDLE_TYPE(  DOUBLE,   Double,   double_value);
-      HANDLE_TYPE(    BOOL,     Bool,     bool_value);
-      HANDLE_TYPE(  STRING,   String,  *string_value);
-      HANDLE_TYPE(   BYTES,    Bytes,  *string_value);
-      HANDLE_TYPE(    ENUM,     Enum,     enum_value);
-      HANDLE_TYPE(   GROUP,    Group, *message_value);
-#undef HANDLE_TYPE
-      case FieldDescriptor::TYPE_MESSAGE:
-        if (is_lazy) {
-          target = lazymessage_value->InternalWriteMessageToArray(
-              number, deterministic, target);
-        } else {
-          target = WireFormatLite::InternalWriteMessageToArray(
-              number, *message_value, deterministic, target);
-        }
-        break;
-    }
-  }
-  return target;
-}
-
-uint8*
-ExtensionSet::Extension::InternalSerializeMessageSetItemWithCachedSizesToArray(
-    int number, bool deterministic, uint8* target) const {
-  if (type != WireFormatLite::TYPE_MESSAGE || is_repeated) {
-    // Not a valid MessageSet extension, but serialize it the normal way.
-    GOOGLE_LOG(WARNING) << "Invalid message set extension.";
-    return InternalSerializeFieldWithCachedSizesToArray(number, deterministic,
-                                                        target);
-  }
-
-  if (is_cleared) return target;
-
-  // Start group.
-  target = io::CodedOutputStream::WriteTagToArray(
-      WireFormatLite::kMessageSetItemStartTag, target);
-  // Write type ID.
-  target = WireFormatLite::WriteUInt32ToArray(
-      WireFormatLite::kMessageSetTypeIdNumber, number, target);
-  // Write message.
-  if (is_lazy) {
-    target = lazymessage_value->InternalWriteMessageToArray(
-        WireFormatLite::kMessageSetMessageNumber, deterministic, target);
-  } else {
-    target = WireFormatLite::InternalWriteMessageToArray(
-        WireFormatLite::kMessageSetMessageNumber, *message_value, deterministic,
-        target);
-  }
-  // End group.
-  target = io::CodedOutputStream::WriteTagToArray(
-      WireFormatLite::kMessageSetItemEndTag, target);
-  return target;
-}
-
-
 bool ExtensionSet::ParseFieldMaybeLazily(
    int wire_type, int field_number, io::CodedInputStream* input,
-    ExtensionFinder* extension_finder,
-    MessageSetFieldSkipper* field_skipper) {
-  return ParseField(WireFormatLite::MakeTag(
-      field_number, static_cast<WireFormatLite::WireType>(wire_type)),
-                    input, extension_finder, field_skipper);
+    ExtensionFinder* extension_finder, MessageSetFieldSkipper* field_skipper) {
+  return ParseField(
+      WireFormatLite::MakeTag(field_number,
+                              static_cast<WireFormatLite::WireType>(wire_type)),
+      input, extension_finder, field_skipper);
 }

 bool ExtensionSet::ParseMessageSet(io::CodedInputStream* input,
                                   ExtensionFinder* extension_finder,
                                   MessageSetFieldSkipper* field_skipper) {
  while (true) {
-    const uint32 tag = input->ReadTag();
+    const uint32_t tag = input->ReadTag();
    switch (tag) {
      case 0:
        return true;
@@ -644,171 +516,31 @@ bool ExtensionSet::ParseMessageSet(io::CodedInputStream* input,
  }
 }

-bool ExtensionSet::ParseMessageSet(io::CodedInputStream* input,
-                                   const MessageLite* containing_type) {
-  MessageSetFieldSkipper skipper(NULL);
-  GeneratedExtensionFinder finder(containing_type);
-  return ParseMessageSet(input, &finder, &skipper);
-}
-
 bool ExtensionSet::ParseMessageSetItem(io::CodedInputStream* input,
                                       ExtensionFinder* extension_finder,
                                       MessageSetFieldSkipper* field_skipper) {
-  // TODO(kenton):  It would be nice to share code between this and
-  // WireFormatLite::ParseAndMergeMessageSetItem(), but I think the
-  // differences would be hard to factor out.
-
-  // This method parses a group which should contain two fields:
-  //   required int32 type_id = 2;
-  //   required data message = 3;
-
-  uint32 last_type_id = 0;
-
-  // If we see message data before the type_id, we'll append it to this so
-  // we can parse it later.
-  string message_data;
-
-  while (true) {
-    const uint32 tag = input->ReadTagNoLastTag();
-    if (tag == 0) return false;
-
-    switch (tag) {
-      case WireFormatLite::kMessageSetTypeIdTag: {
-        uint32 type_id;
-        if (!input->ReadVarint32(&type_id)) return false;
-        last_type_id = type_id;
-
-        if (!message_data.empty()) {
-          // We saw some message data before the type_id.  Have to parse it
-          // now.
-          io::CodedInputStream sub_input(
-              reinterpret_cast<const uint8*>(message_data.data()),
-              message_data.size());
-          if (!ParseFieldMaybeLazily(WireFormatLite::WIRETYPE_LENGTH_DELIMITED,
-                                     last_type_id, &sub_input,
-                                     extension_finder, field_skipper)) {
-            return false;
-          }
-          message_data.clear();
-        }
-
-        break;
-      }
-
-      case WireFormatLite::kMessageSetMessageTag: {
-        if (last_type_id == 0) {
-          // We haven't seen a type_id yet.  Append this data to message_data.
-          string temp;
-          uint32 length;
-          if (!input->ReadVarint32(&length)) return false;
-          if (!input->ReadString(&temp, length)) return false;
-          io::StringOutputStream output_stream(&message_data);
-          io::CodedOutputStream coded_output(&output_stream);
-          coded_output.WriteVarint32(length);
-          coded_output.WriteString(temp);
-        } else {
-          // Already saw type_id, so we can parse this directly.
-          if (!ParseFieldMaybeLazily(WireFormatLite::WIRETYPE_LENGTH_DELIMITED,
-                                     last_type_id, input,
-                                     extension_finder, field_skipper)) {
-            return false;
-          }
-        }
-
-        break;
-      }
-
-      case WireFormatLite::kMessageSetItemEndTag: {
-        return true;
-      }
-
-      default: {
-        if (!field_skipper->SkipField(input, tag)) return false;
-      }
+  struct MSFull {
+    bool ParseField(int type_id, io::CodedInputStream* input) {
+      return me->ParseFieldMaybeLazily(
+          WireFormatLite::WIRETYPE_LENGTH_DELIMITED, type_id, input,
+          extension_finder, field_skipper);
    }
-  }
-}

-void ExtensionSet::Extension::SerializeMessageSetItemWithCachedSizes(
-    int number,
-    io::CodedOutputStream* output) const {
-  if (type != WireFormatLite::TYPE_MESSAGE || is_repeated) {
-    // Not a valid MessageSet extension, but serialize it the normal way.
-    SerializeFieldWithCachedSizes(number, output);
-    return;
-  }
+    bool SkipField(uint32_t tag, io::CodedInputStream* input) {
+      return field_skipper->SkipField(input, tag);
+    }

-  if (is_cleared) return;
+    ExtensionSet* me;
+    ExtensionFinder* extension_finder;
+    MessageSetFieldSkipper* field_skipper;
+  };

-  // Start group.
-  output->WriteTag(WireFormatLite::kMessageSetItemStartTag);
-
-  // Write type ID.
-  WireFormatLite::WriteUInt32(WireFormatLite::kMessageSetTypeIdNumber,
-                              number,
-                              output);
-  // Write message.
-  if (is_lazy) {
-    lazymessage_value->WriteMessage(
-        WireFormatLite::kMessageSetMessageNumber, output);
-  } else {
-    WireFormatLite::WriteMessageMaybeToArray(
-        WireFormatLite::kMessageSetMessageNumber,
-        *message_value,
-        output);
-  }
-
-  // End group.
-  output->WriteTag(WireFormatLite::kMessageSetItemEndTag);
-}
-
-size_t ExtensionSet::Extension::MessageSetItemByteSize(int number) const {
-  if (type != WireFormatLite::TYPE_MESSAGE || is_repeated) {
-    // Not a valid MessageSet extension, but compute the byte size for it the
-    // normal way.
-    return ByteSize(number);
-  }
-
-  if (is_cleared) return 0;
-
-  size_t our_size = WireFormatLite::kMessageSetItemTagsSize;
-
-  // type_id
-  our_size += io::CodedOutputStream::VarintSize32(number);
-
-  // message
-  size_t message_size = 0;
-  if (is_lazy) {
-    message_size = lazymessage_value->ByteSize();
-  } else {
-    message_size = message_value->ByteSize();
-  }
-
-  our_size += io::CodedOutputStream::VarintSize32(message_size);
-  our_size += message_size;
-
-  return our_size;
-}
-
-void ExtensionSet::SerializeMessageSetWithCachedSizes(
-    io::CodedOutputStream* output) const {
-  for (ExtensionMap::const_iterator iter = extensions_.begin();
-       iter != extensions_.end(); ++iter) {
-    iter->second.SerializeMessageSetItemWithCachedSizes(iter->first, output);
-  }
-}
-
-size_t ExtensionSet::MessageSetByteSize() const {
-  size_t total_size = 0;
-
-  for (ExtensionMap::const_iterator iter = extensions_.begin();
-       iter != extensions_.end(); ++iter) {
-    total_size += iter->second.MessageSetItemByteSize(iter->first);
-  }
-
-  return total_size;
+  return ParseMessageSetItemImpl(input,
+                                 MSFull{this, extension_finder, field_skipper});
 }

 }  // namespace internal
 }  // namespace protobuf
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
@@ -0,0 +1,276 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_EXTENSION_SET_INL_H__
+#define GOOGLE_PROTOBUF_EXTENSION_SET_INL_H__
+
+#include <google/protobuf/parse_context.h>
+#include <google/protobuf/extension_set.h>
+#include <google/protobuf/metadata_lite.h>
+
+namespace google {
+namespace protobuf {
+namespace internal {
+
+template <typename T>
+const char* ExtensionSet::ParseFieldWithExtensionInfo(
+    int number, bool was_packed_on_wire, const ExtensionInfo& extension,
+    InternalMetadata* metadata, const char* ptr, internal::ParseContext* ctx) {
+  if (was_packed_on_wire) {
+    switch (extension.type) {
+#define HANDLE_TYPE(UPPERCASE, CPP_CAMELCASE)                                \
+  case WireFormatLite::TYPE_##UPPERCASE:                                     \
+    return internal::Packed##CPP_CAMELCASE##Parser(                          \
+        MutableRawRepeatedField(number, extension.type, extension.is_packed, \
+                                extension.descriptor),                       \
+        ptr, ctx);
+      HANDLE_TYPE(INT32, Int32);
+      HANDLE_TYPE(INT64, Int64);
+      HANDLE_TYPE(UINT32, UInt32);
+      HANDLE_TYPE(UINT64, UInt64);
+      HANDLE_TYPE(SINT32, SInt32);
+      HANDLE_TYPE(SINT64, SInt64);
+      HANDLE_TYPE(FIXED32, Fixed32);
+      HANDLE_TYPE(FIXED64, Fixed64);
+      HANDLE_TYPE(SFIXED32, SFixed32);
+      HANDLE_TYPE(SFIXED64, SFixed64);
+      HANDLE_TYPE(FLOAT, Float);
+      HANDLE_TYPE(DOUBLE, Double);
+      HANDLE_TYPE(BOOL, Bool);
+#undef HANDLE_TYPE
+
+      case WireFormatLite::TYPE_ENUM:
+        return internal::PackedEnumParserArg<T>(
+            MutableRawRepeatedField(number, extension.type, extension.is_packed,
+                                    extension.descriptor),
+            ptr, ctx, extension.enum_validity_check.func,
+            extension.enum_validity_check.arg, metadata, number);
+      case WireFormatLite::TYPE_STRING:
+      case WireFormatLite::TYPE_BYTES:
+      case WireFormatLite::TYPE_GROUP:
+      case WireFormatLite::TYPE_MESSAGE:
+        GOOGLE_LOG(FATAL) << "Non-primitive types can't be packed.";
+        break;
+    }
+  } else {
+    switch (extension.type) {
+#define HANDLE_VARINT_TYPE(UPPERCASE, CPP_CAMELCASE)                        \
+  case WireFormatLite::TYPE_##UPPERCASE: {                                  \
+    uint64_t value;                                                         \
+    ptr = VarintParse(ptr, &value);                                         \
+    GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);                                    \
+    if (extension.is_repeated) {                                            \
+      Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE,          \
+                         extension.is_packed, value, extension.descriptor); \
+    } else {                                                                \
+      Set##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, value,   \
+                         extension.descriptor);                             \
+    }                                                                       \
+  } break
+
+      HANDLE_VARINT_TYPE(INT32, Int32);
+      HANDLE_VARINT_TYPE(INT64, Int64);
+      HANDLE_VARINT_TYPE(UINT32, UInt32);
+      HANDLE_VARINT_TYPE(UINT64, UInt64);
+      HANDLE_VARINT_TYPE(BOOL, Bool);
+#undef HANDLE_VARINT_TYPE
+#define HANDLE_SVARINT_TYPE(UPPERCASE, CPP_CAMELCASE, SIZE)                 \
+  case WireFormatLite::TYPE_##UPPERCASE: {                                  \
+    uint64_t val;                                                           \
+    ptr = VarintParse(ptr, &val);                                           \
+    GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);                                    \
+    auto value = WireFormatLite::ZigZagDecode##SIZE(val);                   \
+    if (extension.is_repeated) {                                            \
+      Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE,          \
+                         extension.is_packed, value, extension.descriptor); \
+    } else {                                                                \
+      Set##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, value,   \
+                         extension.descriptor);                             \
+    }                                                                       \
+  } break
+
+      HANDLE_SVARINT_TYPE(SINT32, Int32, 32);
+      HANDLE_SVARINT_TYPE(SINT64, Int64, 64);
+#undef HANDLE_SVARINT_TYPE
+#define HANDLE_FIXED_TYPE(UPPERCASE, CPP_CAMELCASE, CPPTYPE)                \
+  case WireFormatLite::TYPE_##UPPERCASE: {                                  \
+    auto value = UnalignedLoad<CPPTYPE>(ptr);                               \
+    ptr += sizeof(CPPTYPE);                                                 \
+    if (extension.is_repeated) {                                            \
+      Add##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE,          \
+                         extension.is_packed, value, extension.descriptor); \
+    } else {                                                                \
+      Set##CPP_CAMELCASE(number, WireFormatLite::TYPE_##UPPERCASE, value,   \
+                         extension.descriptor);                             \
+    }                                                                       \
+  } break
+
+      HANDLE_FIXED_TYPE(FIXED32, UInt32, uint32_t);
+      HANDLE_FIXED_TYPE(FIXED64, UInt64, uint64_t);
+      HANDLE_FIXED_TYPE(SFIXED32, Int32, int32_t);
+      HANDLE_FIXED_TYPE(SFIXED64, Int64, int64_t);
+      HANDLE_FIXED_TYPE(FLOAT, Float, float);
+      HANDLE_FIXED_TYPE(DOUBLE, Double, double);
+#undef HANDLE_FIXED_TYPE
+
+      case WireFormatLite::TYPE_ENUM: {
+        uint64_t val;
+        ptr = VarintParse(ptr, &val);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+        int value = val;
+
+        if (!extension.enum_validity_check.func(
+                extension.enum_validity_check.arg, value)) {
+          WriteVarint(number, val, metadata->mutable_unknown_fields<T>());
+        } else if (extension.is_repeated) {
+          AddEnum(number, WireFormatLite::TYPE_ENUM, extension.is_packed, value,
+                  extension.descriptor);
+        } else {
+          SetEnum(number, WireFormatLite::TYPE_ENUM, value,
+                  extension.descriptor);
+        }
+        break;
+      }
+
+      case WireFormatLite::TYPE_BYTES:
+      case WireFormatLite::TYPE_STRING: {
+        std::string* value =
+            extension.is_repeated
+                ? AddString(number, WireFormatLite::TYPE_STRING,
+                            extension.descriptor)
+                : MutableString(number, WireFormatLite::TYPE_STRING,
+                                extension.descriptor);
+        int size = ReadSize(&ptr);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+        return ctx->ReadString(ptr, size, value);
+      }
+
+      case WireFormatLite::TYPE_GROUP: {
+        MessageLite* value =
+            extension.is_repeated
+                ? AddMessage(number, WireFormatLite::TYPE_GROUP,
+                             *extension.message_info.prototype,
+                             extension.descriptor)
+                : MutableMessage(number, WireFormatLite::TYPE_GROUP,
+                                 *extension.message_info.prototype,
+                                 extension.descriptor);
+        uint32_t tag = (number << 3) + WireFormatLite::WIRETYPE_START_GROUP;
+        return ctx->ParseGroup(value, ptr, tag);
+      }
+
+      case WireFormatLite::TYPE_MESSAGE: {
+        MessageLite* value =
+            extension.is_repeated
+                ? AddMessage(number, WireFormatLite::TYPE_MESSAGE,
+                             *extension.message_info.prototype,
+                             extension.descriptor)
+                : MutableMessage(number, WireFormatLite::TYPE_MESSAGE,
+                                 *extension.message_info.prototype,
+                                 extension.descriptor);
+        return ctx->ParseMessage(value, ptr);
+      }
+    }
+  }
+  return ptr;
+}
+
+template <typename Msg, typename T>
+const char* ExtensionSet::ParseMessageSetItemTmpl(
+    const char* ptr, const Msg* extendee, internal::InternalMetadata* metadata,
+    internal::ParseContext* ctx) {
+  std::string payload;
+  uint32_t type_id = 0;
+  bool payload_read = false;
+  while (!ctx->Done(&ptr)) {
+    uint32_t tag = static_cast<uint8_t>(*ptr++);
+    if (tag == WireFormatLite::kMessageSetTypeIdTag) {
+      uint64_t tmp;
+      ptr = ParseBigVarint(ptr, &tmp);
+      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+      type_id = tmp;
+      if (payload_read) {
+        ExtensionInfo extension;
+        bool was_packed_on_wire;
+        if (!FindExtension(2, type_id, extendee, ctx, &extension,
+                           &was_packed_on_wire)) {
+          WriteLengthDelimited(type_id, payload,
+                               metadata->mutable_unknown_fields<T>());
+        } else {
+          MessageLite* value =
+              extension.is_repeated
+                  ? AddMessage(type_id, WireFormatLite::TYPE_MESSAGE,
+                               *extension.message_info.prototype,
+                               extension.descriptor)
+                  : MutableMessage(type_id, WireFormatLite::TYPE_MESSAGE,
+                                   *extension.message_info.prototype,
+                                   extension.descriptor);
+
+          const char* p;
+          // We can't use regular parse from string as we have to track
+          // proper recursion depth and descriptor pools.
+          ParseContext tmp_ctx(ctx->depth(), false, &p, payload);
+          tmp_ctx.data().pool = ctx->data().pool;
+          tmp_ctx.data().factory = ctx->data().factory;
+          GOOGLE_PROTOBUF_PARSER_ASSERT(value->_InternalParse(p, &tmp_ctx) &&
+                                         tmp_ctx.EndedAtLimit());
+        }
+        type_id = 0;
+      }
+    } else if (tag == WireFormatLite::kMessageSetMessageTag) {
+      if (type_id != 0) {
+        ptr = ParseFieldMaybeLazily(static_cast<uint64_t>(type_id) * 8 + 2, ptr,
+                                    extendee, metadata, ctx);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr != nullptr);
+        type_id = 0;
+      } else {
+        int32_t size = ReadSize(&ptr);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+        ptr = ctx->ReadString(ptr, size, &payload);
+        GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+        payload_read = true;
+      }
+    } else {
+      ptr = ReadTag(ptr - 1, &tag);
+      if (tag == 0 || (tag & 7) == 4) {
+        ctx->SetLastTag(tag);
+        return ptr;
+      }
+      ptr = ParseField(tag, ptr, extendee, metadata, ctx);
+      GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
+    }
+  }
+  return ptr;
+}
+
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_EXTENSION_SET_INL_H__
@@ -0,0 +1,172 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_FIELD_ACCESS_LISTENER_H__
+#define GOOGLE_PROTOBUF_FIELD_ACCESS_LISTENER_H__
+
+#include <cstddef>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/message_lite.h>
+
+
+namespace google {
+namespace protobuf {
+
+// A default/no-op implementation of message hooks.
+//
+// See go/statically-dispatched-message-hooks for details.
+template <typename Proto>
+struct NoOpAccessListener {
+  // Number of fields are provided at compile time for the trackers to be able
+  // to have stack allocated bitmaps for the fields. This is useful for
+  // performance critical trackers. This is also to avoid cyclic dependencies
+  // if the number of fields is needed.
+  static constexpr int kFields = Proto::_kInternalFieldNumber;
+  // Default constructor is called during the static global initialization of
+  // the program.
+  // We provide a pointer to extract the name of the proto not to get cyclic
+  // dependencies on GetDescriptor() and OnGetMetadata() calls. If you want
+  // to differentiate the protos during the runtime before the start of the
+  // program, use this functor to get its name. We either way need it for
+  // LITE_RUNTIME protos as they don't have descriptors at all.
+  explicit NoOpAccessListener(StringPiece (*name_extractor)()) {}
+  // called repeatedly during serialization/deserialization/ByteSize of
+  // Reflection as:
+  //   AccessListener<MessageT>::OnSerialize(this);
+  static void OnSerialize(const MessageLite* msg) {}
+  static void OnDeserialize(const MessageLite* msg) {}
+  static void OnByteSize(const MessageLite* msg) {}
+  static void OnMergeFrom(const MessageLite* to, const MessageLite* from) {}
+
+  // NOTE: This function can be called pre-main. Make sure it does not make
+  // the state of the listener invalid.
+  static void OnGetMetadata() {}
+
+  // called from accessors as:
+  //   AccessListener<MessageT>::On$operation(this, &field_storage_);
+  // If you need to override this with type, in your hook implementation
+  // introduce
+  // template <int kFieldNum, typename T>
+  // static void On$operation(const MessageLite* msg,
+  //                          const T* field) {}
+  // And overloads for std::nullptr_t for incomplete types such as Messages,
+  // Maps. Extract them using reflection if you need. Consequently, second
+  // argument can be null pointer.
+  // For an example, see proto_hooks/testing/memory_test_field_listener.h
+  // And argument template deduction will deduce the type itself without
+  // changing the generated code.
+
+  // add_<field>(f)
+  template <int kFieldNum>
+  static void OnAdd(const MessageLite* msg, const void* field) {}
+
+  // add_<field>()
+  template <int kFieldNum>
+  static void OnAddMutable(const MessageLite* msg, const void* field) {}
+
+  // <field>() and <repeated_field>(i)
+  template <int kFieldNum>
+  static void OnGet(const MessageLite* msg, const void* field) {}
+
+  // clear_<field>()
+  template <int kFieldNum>
+  static void OnClear(const MessageLite* msg, const void* field) {}
+
+  // has_<field>()
+  template <int kFieldNum>
+  static void OnHas(const MessageLite* msg, const void* field) {}
+
+  // <repeated_field>()
+  template <int kFieldNum>
+  static void OnList(const MessageLite* msg, const void* field) {}
+
+  // mutable_<field>()
+  template <int kFieldNum>
+  static void OnMutable(const MessageLite* msg, const void* field) {}
+
+  // mutable_<repeated_field>()
+  template <int kFieldNum>
+  static void OnMutableList(const MessageLite* msg, const void* field) {}
+
+  // release_<field>()
+  template <int kFieldNum>
+  static void OnRelease(const MessageLite* msg, const void* field) {}
+
+  // set_<field>() and set_<repeated_field>(i)
+  template <int kFieldNum>
+  static void OnSet(const MessageLite* msg, const void* field) {}
+
+  // <repeated_field>_size()
+  template <int kFieldNum>
+  static void OnSize(const MessageLite* msg, const void* field) {}
+
+  static void OnHasExtension(const MessageLite* msg, int extension_tag,
+                             const void* field) {}
+  // TODO(b/190614678): Support clear in the proto compiler.
+  static void OnClearExtension(const MessageLite* msg, int extension_tag,
+                               const void* field) {}
+  static void OnExtensionSize(const MessageLite* msg, int extension_tag,
+                              const void* field) {}
+  static void OnGetExtension(const MessageLite* msg, int extension_tag,
+                             const void* field) {}
+  static void OnMutableExtension(const MessageLite* msg, int extension_tag,
+                                 const void* field) {}
+  static void OnSetExtension(const MessageLite* msg, int extension_tag,
+                             const void* field) {}
+  static void OnReleaseExtension(const MessageLite* msg, int extension_tag,
+                                 const void* field) {}
+  static void OnAddExtension(const MessageLite* msg, int extension_tag,
+                             const void* field) {}
+  static void OnAddMutableExtension(const MessageLite* msg, int extension_tag,
+                                    const void* field) {}
+  static void OnListExtension(const MessageLite* msg, int extension_tag,
+                              const void* field) {}
+  static void OnMutableListExtension(const MessageLite* msg, int extension_tag,
+                                     const void* field) {}
+};
+
+}  // namespace protobuf
+}  // namespace google
+
+#ifndef REPLACE_PROTO_LISTENER_IMPL
+namespace google {
+namespace protobuf {
+template <class T>
+using AccessListener = NoOpAccessListener<T>;
+}  // namespace protobuf
+}  // namespace google
+#else
+// You can put your implementations of hooks/listeners here.
+// All hooks are subject to approval by protobuf-team@.
+
+#endif  // !REPLACE_PROTO_LISTENER_IMPL
+
+#endif  // GOOGLE_PROTOBUF_FIELD_ACCESS_LISTENER_H__
@@ -1,373 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: google/protobuf/field_mask.proto
-
-#include <google/protobuf/field_mask.pb.h>
-
-#include <algorithm>
-
-#include <google/protobuf/stubs/common.h>
-#include <google/protobuf/stubs/port.h>
-#include <google/protobuf/stubs/once.h>
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/wire_format_lite_inl.h>
-#include <google/protobuf/descriptor.h>
-#include <google/protobuf/generated_message_reflection.h>
-#include <google/protobuf/reflection_ops.h>
-#include <google/protobuf/wire_format.h>
-// This is a temporary google only hack
-#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-#include "third_party/protobuf/version.h"
-#endif
-// @@protoc_insertion_point(includes)
-namespace google {
-namespace protobuf {
-class FieldMaskDefaultTypeInternal {
- public:
-  ::google::protobuf::internal::ExplicitlyConstructed<FieldMask>
-      _instance;
-} _FieldMask_default_instance_;
-}  // namespace protobuf
-}  // namespace google
-namespace protobuf_google_2fprotobuf_2ffield_5fmask_2eproto {
-void InitDefaultsFieldMaskImpl() {
-  GOOGLE_PROTOBUF_VERIFY_VERSION;
-
-#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-  ::google::protobuf::internal::InitProtobufDefaultsForceUnique();
-#else
-  ::google::protobuf::internal::InitProtobufDefaults();
-#endif  // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS
-  {
-    void* ptr = &::google::protobuf::_FieldMask_default_instance_;
-    new (ptr) ::google::protobuf::FieldMask();
-    ::google::protobuf::internal::OnShutdownDestroyMessage(ptr);
-  }
-  ::google::protobuf::FieldMask::InitAsDefaultInstance();
-}
-
-void InitDefaultsFieldMask() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsFieldMaskImpl);
-}
-
-::google::protobuf::Metadata file_level_metadata[1];
-
-const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  ~0u,  // no _has_bits_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::FieldMask, _internal_metadata_),
-  ~0u,  // no _extensions_
-  ~0u,  // no _oneof_case_
-  ~0u,  // no _weak_field_map_
-  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::FieldMask, paths_),
-};
-static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-  { 0, -1, sizeof(::google::protobuf::FieldMask)},
-};
-
-static ::google::protobuf::Message const * const file_default_instances[] = {
-  reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_FieldMask_default_instance_),
-};
-
-void protobuf_AssignDescriptors() {
-  AddDescriptors();
-  ::google::protobuf::MessageFactory* factory = NULL;
-  AssignDescriptors(
-      "google/protobuf/field_mask.proto", schemas, file_default_instances, TableStruct::offsets, factory,
-      file_level_metadata, NULL, NULL);
-}
-
-void protobuf_AssignDescriptorsOnce() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &protobuf_AssignDescriptors);
-}
-
-void protobuf_RegisterTypes(const ::std::string&) GOOGLE_PROTOBUF_ATTRIBUTE_COLD;
-void protobuf_RegisterTypes(const ::std::string&) {
-  protobuf_AssignDescriptorsOnce();
-  ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1);
-}
-
-void AddDescriptorsImpl() {
-  InitDefaults();
-  static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = {
-      "\n google/protobuf/field_mask.proto\022\017goog"
-      "le.protobuf\"\032\n\tFieldMask\022\r\n\005paths\030\001 \003(\tB"
-      "\211\001\n\023com.google.protobufB\016FieldMaskProtoP"
-      "\001Z9google.golang.org/genproto/protobuf/f"
-      "ield_mask;field_mask\242\002\003GPB\252\002\036Google.Prot"
-      "obuf.WellKnownTypesb\006proto3"
-  };
-  ::google::protobuf::DescriptorPool::InternalAddGeneratedFile(
-      descriptor, 227);
-  ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
-    "google/protobuf/field_mask.proto", &protobuf_RegisterTypes);
-}
-
-void AddDescriptors() {
-  static GOOGLE_PROTOBUF_DECLARE_ONCE(once);
-  ::google::protobuf::GoogleOnceInit(&once, &AddDescriptorsImpl);
-}
-// Force AddDescriptors() to be called at dynamic initialization time.
-struct StaticDescriptorInitializer {
-  StaticDescriptorInitializer() {
-    AddDescriptors();
-  }
-} static_descriptor_initializer;
-}  // namespace protobuf_google_2fprotobuf_2ffield_5fmask_2eproto
-namespace google {
-namespace protobuf {
-
-// ===================================================================
-
-void FieldMask::InitAsDefaultInstance() {
-}
-#if !defined(_MSC_VER) || _MSC_VER >= 1900
-const int FieldMask::kPathsFieldNumber;
-#endif  // !defined(_MSC_VER) || _MSC_VER >= 1900
-
-FieldMask::FieldMask()
-  : ::google::protobuf::Message(), _internal_metadata_(NULL) {
-  if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) {
-    ::protobuf_google_2fprotobuf_2ffield_5fmask_2eproto::InitDefaultsFieldMask();
-  }
-  SharedCtor();
-  // @@protoc_insertion_point(constructor:google.protobuf.FieldMask)
-}
-FieldMask::FieldMask(const FieldMask& from)
-  : ::google::protobuf::Message(),
-      _internal_metadata_(NULL),
-      paths_(from.paths_),
-      _cached_size_(0) {
-  _internal_metadata_.MergeFrom(from._internal_metadata_);
-  // @@protoc_insertion_point(copy_constructor:google.protobuf.FieldMask)
-}
-
-void FieldMask::SharedCtor() {
-  _cached_size_ = 0;
-}
-
-FieldMask::~FieldMask() {
-  // @@protoc_insertion_point(destructor:google.protobuf.FieldMask)
-  SharedDtor();
-}
-
-void FieldMask::SharedDtor() {
-}
-
-void FieldMask::SetCachedSize(int size) const {
-  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
-  _cached_size_ = size;
-  GOOGLE_SAFE_CONCURRENT_WRITES_END();
-}
-const ::google::protobuf::Descriptor* FieldMask::descriptor() {
-  ::protobuf_google_2fprotobuf_2ffield_5fmask_2eproto::protobuf_AssignDescriptorsOnce();
-  return ::protobuf_google_2fprotobuf_2ffield_5fmask_2eproto::file_level_metadata[kIndexInFileMessages].descriptor;
-}
-
-const FieldMask& FieldMask::default_instance() {
-  ::protobuf_google_2fprotobuf_2ffield_5fmask_2eproto::InitDefaultsFieldMask();
-  return *internal_default_instance();
-}
-
-FieldMask* FieldMask::New(::google::protobuf::Arena* arena) const {
-  FieldMask* n = new FieldMask;
-  if (arena != NULL) {
-    arena->Own(n);
-  }
-  return n;
-}
-
-void FieldMask::Clear() {
-// @@protoc_insertion_point(message_clear_start:google.protobuf.FieldMask)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  // Prevent compiler warnings about cached_has_bits being unused
-  (void) cached_has_bits;
-
-  paths_.Clear();
-  _internal_metadata_.Clear();
-}
-
-bool FieldMask::MergePartialFromCodedStream(
-    ::google::protobuf::io::CodedInputStream* input) {
-#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure
-  ::google::protobuf::uint32 tag;
-  // @@protoc_insertion_point(parse_start:google.protobuf.FieldMask)
-  for (;;) {
-    ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u);
-    tag = p.first;
-    if (!p.second) goto handle_unusual;
-    switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
-      // repeated string paths = 1;
-      case 1: {
-        if (static_cast< ::google::protobuf::uint8>(tag) ==
-            static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadString(
-                input, this->add_paths()));
-          DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
-            this->paths(this->paths_size() - 1).data(),
-            static_cast<int>(this->paths(this->paths_size() - 1).length()),
-            ::google::protobuf::internal::WireFormatLite::PARSE,
-            "google.protobuf.FieldMask.paths"));
-        } else {
-          goto handle_unusual;
-        }
-        break;
-      }
-
-      default: {
-      handle_unusual:
-        if (tag == 0) {
-          goto success;
-        }
-        DO_(::google::protobuf::internal::WireFormat::SkipField(
-              input, tag, _internal_metadata_.mutable_unknown_fields()));
-        break;
-      }
-    }
-  }
-success:
-  // @@protoc_insertion_point(parse_success:google.protobuf.FieldMask)
-  return true;
-failure:
-  // @@protoc_insertion_point(parse_failure:google.protobuf.FieldMask)
-  return false;
-#undef DO_
-}
-
-void FieldMask::SerializeWithCachedSizes(
-    ::google::protobuf::io::CodedOutputStream* output) const {
-  // @@protoc_insertion_point(serialize_start:google.protobuf.FieldMask)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-  // repeated string paths = 1;
-  for (int i = 0, n = this->paths_size(); i < n; i++) {
-    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
-      this->paths(i).data(), static_cast<int>(this->paths(i).length()),
-      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
-      "google.protobuf.FieldMask.paths");
-    ::google::protobuf::internal::WireFormatLite::WriteString(
-      1, this->paths(i), output);
-  }
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    ::google::protobuf::internal::WireFormat::SerializeUnknownFields(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()), output);
-  }
-  // @@protoc_insertion_point(serialize_end:google.protobuf.FieldMask)
-}
-
-::google::protobuf::uint8* FieldMask::InternalSerializeWithCachedSizesToArray(
-    bool deterministic, ::google::protobuf::uint8* target) const {
-  (void)deterministic; // Unused
-  // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.FieldMask)
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-  // repeated string paths = 1;
-  for (int i = 0, n = this->paths_size(); i < n; i++) {
-    ::google::protobuf::internal::WireFormatLite::VerifyUtf8String(
-      this->paths(i).data(), static_cast<int>(this->paths(i).length()),
-      ::google::protobuf::internal::WireFormatLite::SERIALIZE,
-      "google.protobuf.FieldMask.paths");
-    target = ::google::protobuf::internal::WireFormatLite::
-      WriteStringToArray(1, this->paths(i), target);
-  }
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()), target);
-  }
-  // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.FieldMask)
-  return target;
-}
-
-size_t FieldMask::ByteSizeLong() const {
-// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FieldMask)
-  size_t total_size = 0;
-
-  if ((_internal_metadata_.have_unknown_fields() &&  ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) {
-    total_size +=
-      ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize(
-        (::google::protobuf::internal::GetProto3PreserveUnknownsDefault()   ? _internal_metadata_.unknown_fields()   : _internal_metadata_.default_instance()));
-  }
-  // repeated string paths = 1;
-  total_size += 1 *
-      ::google::protobuf::internal::FromIntSize(this->paths_size());
-  for (int i = 0, n = this->paths_size(); i < n; i++) {
-    total_size += ::google::protobuf::internal::WireFormatLite::StringSize(
-      this->paths(i));
-  }
-
-  int cached_size = ::google::protobuf::internal::ToCachedSize(total_size);
-  GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
-  _cached_size_ = cached_size;
-  GOOGLE_SAFE_CONCURRENT_WRITES_END();
-  return total_size;
-}
-
-void FieldMask::MergeFrom(const ::google::protobuf::Message& from) {
-// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.FieldMask)
-  GOOGLE_DCHECK_NE(&from, this);
-  const FieldMask* source =
-      ::google::protobuf::internal::DynamicCastToGenerated<const FieldMask>(
-          &from);
-  if (source == NULL) {
-  // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.FieldMask)
-    ::google::protobuf::internal::ReflectionOps::Merge(from, this);
-  } else {
-  // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.FieldMask)
-    MergeFrom(*source);
-  }
-}
-
-void FieldMask::MergeFrom(const FieldMask& from) {
-// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.FieldMask)
-  GOOGLE_DCHECK_NE(&from, this);
-  _internal_metadata_.MergeFrom(from._internal_metadata_);
-  ::google::protobuf::uint32 cached_has_bits = 0;
-  (void) cached_has_bits;
-
-  paths_.MergeFrom(from.paths_);
-}
-
-void FieldMask::CopyFrom(const ::google::protobuf::Message& from) {
-// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.FieldMask)
-  if (&from == this) return;
-  Clear();
-  MergeFrom(from);
-}
-
-void FieldMask::CopyFrom(const FieldMask& from) {
-// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.FieldMask)
-  if (&from == this) return;
-  Clear();
-  MergeFrom(from);
-}
-
-bool FieldMask::IsInitialized() const {
-  return true;
-}
-
-void FieldMask::Swap(FieldMask* other) {
-  if (other == this) return;
-  InternalSwap(other);
-}
-void FieldMask::InternalSwap(FieldMask* other) {
-  using std::swap;
-  paths_.InternalSwap(&other->paths_);
-  _internal_metadata_.Swap(&other->_internal_metadata_);
-  swap(_cached_size_, other->_cached_size_);
-}
-
-::google::protobuf::Metadata FieldMask::GetMetadata() const {
-  protobuf_google_2fprotobuf_2ffield_5fmask_2eproto::protobuf_AssignDescriptorsOnce();
-  return ::protobuf_google_2fprotobuf_2ffield_5fmask_2eproto::file_level_metadata[kIndexInFileMessages];
-}
-
-
-// @@protoc_insertion_point(namespace_scope)
-}  // namespace protobuf
-}  // namespace google
-
-// @@protoc_insertion_point(global_scope)
@@ -1,177 +1,228 @@
 // Generated by the protocol buffer compiler.  DO NOT EDIT!
 // source: google/protobuf/field_mask.proto

-#ifndef PROTOBUF_google_2fprotobuf_2ffield_5fmask_2eproto__INCLUDED
-#define PROTOBUF_google_2fprotobuf_2ffield_5fmask_2eproto__INCLUDED
+#ifndef GOOGLE_PROTOBUF_INCLUDED_google_2fprotobuf_2ffield_5fmask_2eproto
+#define GOOGLE_PROTOBUF_INCLUDED_google_2fprotobuf_2ffield_5fmask_2eproto

+#include <limits>
 #include <string>

-#include <google/protobuf/stubs/common.h>
-
-#if GOOGLE_PROTOBUF_VERSION < 3005000
+#include <google/protobuf/port_def.inc>
+#if PROTOBUF_VERSION < 3019000
 #error This file was generated by a newer version of protoc which is
-#error incompatible with your Protocol Buffer headers.  Please update
+#error incompatible with your Protocol Buffer headers. Please update
 #error your headers.
 #endif
-#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION
+#if 3019001 < PROTOBUF_MIN_PROTOC_VERSION
 #error This file was generated by an older version of protoc which is
-#error incompatible with your Protocol Buffer headers.  Please
+#error incompatible with your Protocol Buffer headers. Please
 #error regenerate this file with a newer version of protoc.
 #endif

+#include <google/protobuf/port_undef.inc>
 #include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/arena.h>
 #include <google/protobuf/arenastring.h>
 #include <google/protobuf/generated_message_table_driven.h>
 #include <google/protobuf/generated_message_util.h>
-#include <google/protobuf/metadata.h>
+#include <google/protobuf/metadata_lite.h>
+#include <google/protobuf/generated_message_reflection.h>
 #include <google/protobuf/message.h>
 #include <google/protobuf/repeated_field.h>  // IWYU pragma: export
 #include <google/protobuf/extension_set.h>  // IWYU pragma: export
 #include <google/protobuf/unknown_field_set.h>
 // @@protoc_insertion_point(includes)
+#include <google/protobuf/port_def.inc>
+#define PROTOBUF_INTERNAL_EXPORT_google_2fprotobuf_2ffield_5fmask_2eproto PROTOBUF_EXPORT
+PROTOBUF_NAMESPACE_OPEN
+namespace internal {
+class AnyMetadata;
+}  // namespace internal
+PROTOBUF_NAMESPACE_CLOSE

-namespace protobuf_google_2fprotobuf_2ffield_5fmask_2eproto {
 // Internal implementation detail -- do not use these members.
-struct LIBPROTOBUF_EXPORT TableStruct {
-  static const ::google::protobuf::internal::ParseTableField entries[];
-  static const ::google::protobuf::internal::AuxillaryParseTableField aux[];
-  static const ::google::protobuf::internal::ParseTable schema[1];
-  static const ::google::protobuf::internal::FieldMetadata field_metadata[];
-  static const ::google::protobuf::internal::SerializationTable serialization_table[];
-  static const ::google::protobuf::uint32 offsets[];
+struct PROTOBUF_EXPORT TableStruct_google_2fprotobuf_2ffield_5fmask_2eproto {
+  static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTableField entries[]
+    PROTOBUF_SECTION_VARIABLE(protodesc_cold);
+  static const ::PROTOBUF_NAMESPACE_ID::internal::AuxiliaryParseTableField aux[]
+    PROTOBUF_SECTION_VARIABLE(protodesc_cold);
+  static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[1]
+    PROTOBUF_SECTION_VARIABLE(protodesc_cold);
+  static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[];
+  static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[];
+  static const uint32_t offsets[];
 };
-void LIBPROTOBUF_EXPORT AddDescriptors();
-void LIBPROTOBUF_EXPORT InitDefaultsFieldMaskImpl();
-void LIBPROTOBUF_EXPORT InitDefaultsFieldMask();
-inline void LIBPROTOBUF_EXPORT InitDefaults() {
-  InitDefaultsFieldMask();
-}
-}  // namespace protobuf_google_2fprotobuf_2ffield_5fmask_2eproto
-namespace google {
-namespace protobuf {
+PROTOBUF_EXPORT extern const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_google_2fprotobuf_2ffield_5fmask_2eproto;
+PROTOBUF_NAMESPACE_OPEN
 class FieldMask;
-class FieldMaskDefaultTypeInternal;
-LIBPROTOBUF_EXPORT extern FieldMaskDefaultTypeInternal _FieldMask_default_instance_;
-}  // namespace protobuf
-}  // namespace google
-namespace google {
-namespace protobuf {
+struct FieldMaskDefaultTypeInternal;
+PROTOBUF_EXPORT extern FieldMaskDefaultTypeInternal _FieldMask_default_instance_;
+PROTOBUF_NAMESPACE_CLOSE
+PROTOBUF_NAMESPACE_OPEN
+template<> PROTOBUF_EXPORT ::PROTOBUF_NAMESPACE_ID::FieldMask* Arena::CreateMaybeMessage<::PROTOBUF_NAMESPACE_ID::FieldMask>(Arena*);
+PROTOBUF_NAMESPACE_CLOSE
+PROTOBUF_NAMESPACE_OPEN

 // ===================================================================

-class LIBPROTOBUF_EXPORT FieldMask : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.FieldMask) */ {
+class PROTOBUF_EXPORT FieldMask final :
+    public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:google.protobuf.FieldMask) */ {
 public:
-  FieldMask();
-  virtual ~FieldMask();
+  inline FieldMask() : FieldMask(nullptr) {}
+  ~FieldMask() override;
+  explicit constexpr FieldMask(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized);

  FieldMask(const FieldMask& from);
-
-  inline FieldMask& operator=(const FieldMask& from) {
-    CopyFrom(from);
-    return *this;
-  }
-  #if LANG_CXX11
  FieldMask(FieldMask&& from) noexcept
    : FieldMask() {
    *this = ::std::move(from);
  }

+  inline FieldMask& operator=(const FieldMask& from) {
+    CopyFrom(from);
+    return *this;
+  }
  inline FieldMask& operator=(FieldMask&& from) noexcept {
-    if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
-      if (this != &from) InternalSwap(&from);
+    if (this == &from) return *this;
+    if (GetOwningArena() == from.GetOwningArena()
+  #ifdef PROTOBUF_FORCE_COPY_IN_MOVE
+        && GetOwningArena() != nullptr
+  #endif  // !PROTOBUF_FORCE_COPY_IN_MOVE
+    ) {
+      InternalSwap(&from);
    } else {
      CopyFrom(from);
    }
    return *this;
  }
-  #endif
-  static const ::google::protobuf::Descriptor* descriptor();
-  static const FieldMask& default_instance();

-  static void InitAsDefaultInstance();  // FOR INTERNAL USE ONLY
+  static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
+    return GetDescriptor();
+  }
+  static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
+    return default_instance().GetMetadata().descriptor;
+  }
+  static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
+    return default_instance().GetMetadata().reflection;
+  }
+  static const FieldMask& default_instance() {
+    return *internal_default_instance();
+  }
  static inline const FieldMask* internal_default_instance() {
    return reinterpret_cast<const FieldMask*>(
               &_FieldMask_default_instance_);
  }
-  static PROTOBUF_CONSTEXPR int const kIndexInFileMessages =
+  static constexpr int kIndexInFileMessages =
    0;

-  void Swap(FieldMask* other);
  friend void swap(FieldMask& a, FieldMask& b) {
    a.Swap(&b);
  }
+  inline void Swap(FieldMask* other) {
+    if (other == this) return;
+  #ifdef PROTOBUF_FORCE_COPY_IN_SWAP
+    if (GetOwningArena() != nullptr &&
+        GetOwningArena() == other->GetOwningArena()) {
+   #else  // PROTOBUF_FORCE_COPY_IN_SWAP
+    if (GetOwningArena() == other->GetOwningArena()) {
+  #endif  // !PROTOBUF_FORCE_COPY_IN_SWAP
+      InternalSwap(other);
+    } else {
+      ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other);
+    }
+  }
+  void UnsafeArenaSwap(FieldMask* other) {
+    if (other == this) return;
+    GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena());
+    InternalSwap(other);
+  }

  // implements Message ----------------------------------------------

-  inline FieldMask* New() const PROTOBUF_FINAL { return New(NULL); }
-
-  FieldMask* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL;
-  void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
-  void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
+  FieldMask* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final {
+    return CreateMaybeMessage<FieldMask>(arena);
+  }
+  using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom;
  void CopyFrom(const FieldMask& from);
+  using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom;
  void MergeFrom(const FieldMask& from);
-  void Clear() PROTOBUF_FINAL;
-  bool IsInitialized() const PROTOBUF_FINAL;
+  private:
+  static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message* to, const ::PROTOBUF_NAMESPACE_ID::Message& from);
+  public:
+  PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
+  bool IsInitialized() const final;
+
+  size_t ByteSizeLong() const final;
+  const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
+  uint8_t* _InternalSerialize(
+      uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final;
+  int GetCachedSize() const final { return _cached_size_.Get(); }

-  size_t ByteSizeLong() const PROTOBUF_FINAL;
-  bool MergePartialFromCodedStream(
-      ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL;
-  void SerializeWithCachedSizes(
-      ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL;
-  ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(
-      bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL;
-  int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; }
  private:
  void SharedCtor();
  void SharedDtor();
-  void SetCachedSize(int size) const PROTOBUF_FINAL;
+  void SetCachedSize(int size) const final;
  void InternalSwap(FieldMask* other);
+
  private:
-  inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
-    return NULL;
-  }
-  inline void* MaybeArenaPtr() const {
-    return NULL;
+  friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
+  static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
+    return "google.protobuf.FieldMask";
  }
+  protected:
+  explicit FieldMask(::PROTOBUF_NAMESPACE_ID::Arena* arena,
+                       bool is_message_owned = false);
+  private:
+  static void ArenaDtor(void* object);
+  inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena);
  public:

-  ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL;
+  static const ClassData _class_data_;
+  const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final;
+
+  ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;

  // nested types ----------------------------------------------------

  // accessors -------------------------------------------------------

+  enum : int {
+    kPathsFieldNumber = 1,
+  };
  // repeated string paths = 1;
  int paths_size() const;
+  private:
+  int _internal_paths_size() const;
+  public:
  void clear_paths();
-  static const int kPathsFieldNumber = 1;
-  const ::std::string& paths(int index) const;
-  ::std::string* mutable_paths(int index);
-  void set_paths(int index, const ::std::string& value);
-  #if LANG_CXX11
-  void set_paths(int index, ::std::string&& value);
-  #endif
+  const std::string& paths(int index) const;
+  std::string* mutable_paths(int index);
+  void set_paths(int index, const std::string& value);
+  void set_paths(int index, std::string&& value);
  void set_paths(int index, const char* value);
  void set_paths(int index, const char* value, size_t size);
-  ::std::string* add_paths();
-  void add_paths(const ::std::string& value);
-  #if LANG_CXX11
-  void add_paths(::std::string&& value);
-  #endif
+  std::string* add_paths();
+  void add_paths(const std::string& value);
+  void add_paths(std::string&& value);
  void add_paths(const char* value);
  void add_paths(const char* value, size_t size);
-  const ::google::protobuf::RepeatedPtrField< ::std::string>& paths() const;
-  ::google::protobuf::RepeatedPtrField< ::std::string>* mutable_paths();
+  const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>& paths() const;
+  ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>* mutable_paths();
+  private:
+  const std::string& _internal_paths(int index) const;
+  std::string* _internal_add_paths();
+  public:

  // @@protoc_insertion_point(class_scope:google.protobuf.FieldMask)
 private:
+  class _Internal;

-  ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_;
-  ::google::protobuf::RepeatedPtrField< ::std::string> paths_;
-  mutable int _cached_size_;
-  friend struct ::protobuf_google_2fprotobuf_2ffield_5fmask_2eproto::TableStruct;
-  friend void ::protobuf_google_2fprotobuf_2ffield_5fmask_2eproto::InitDefaultsFieldMaskImpl();
+  template <typename T> friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper;
+  typedef void InternalArenaConstructable_;
+  typedef void DestructorSkippable_;
+  ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string> paths_;
+  mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
+  friend struct ::TableStruct_google_2fprotobuf_2ffield_5fmask_2eproto;
 };
 // ===================================================================

@@ -185,32 +236,41 @@ class LIBPROTOBUF_EXPORT FieldMask : public ::google::protobuf::Message /* @@pro
 // FieldMask

 // repeated string paths = 1;
-inline int FieldMask::paths_size() const {
+inline int FieldMask::_internal_paths_size() const {
  return paths_.size();
 }
+inline int FieldMask::paths_size() const {
+  return _internal_paths_size();
+}
 inline void FieldMask::clear_paths() {
  paths_.Clear();
 }
-inline const ::std::string& FieldMask::paths(int index) const {
-  // @@protoc_insertion_point(field_get:google.protobuf.FieldMask.paths)
+inline std::string* FieldMask::add_paths() {
+  std::string* _s = _internal_add_paths();
+  // @@protoc_insertion_point(field_add_mutable:google.protobuf.FieldMask.paths)
+  return _s;
+}
+inline const std::string& FieldMask::_internal_paths(int index) const {
  return paths_.Get(index);
 }
-inline ::std::string* FieldMask::mutable_paths(int index) {
+inline const std::string& FieldMask::paths(int index) const {
+  // @@protoc_insertion_point(field_get:google.protobuf.FieldMask.paths)
+  return _internal_paths(index);
+}
+inline std::string* FieldMask::mutable_paths(int index) {
  // @@protoc_insertion_point(field_mutable:google.protobuf.FieldMask.paths)
  return paths_.Mutable(index);
 }
-inline void FieldMask::set_paths(int index, const ::std::string& value) {
-  // @@protoc_insertion_point(field_set:google.protobuf.FieldMask.paths)
+inline void FieldMask::set_paths(int index, const std::string& value) {
  paths_.Mutable(index)->assign(value);
-}
-#if LANG_CXX11
-inline void FieldMask::set_paths(int index, ::std::string&& value) {
  // @@protoc_insertion_point(field_set:google.protobuf.FieldMask.paths)
-  paths_.Mutable(index)->assign(std::move(value));
 }
-#endif
+inline void FieldMask::set_paths(int index, std::string&& value) {
+  paths_.Mutable(index)->assign(std::move(value));
+  // @@protoc_insertion_point(field_set:google.protobuf.FieldMask.paths)
+}
 inline void FieldMask::set_paths(int index, const char* value) {
-  GOOGLE_DCHECK(value != NULL);
+  GOOGLE_DCHECK(value != nullptr);
  paths_.Mutable(index)->assign(value);
  // @@protoc_insertion_point(field_set_char:google.protobuf.FieldMask.paths)
 }
@@ -219,22 +279,19 @@ inline void FieldMask::set_paths(int index, const char* value, size_t size) {
    reinterpret_cast<const char*>(value), size);
  // @@protoc_insertion_point(field_set_pointer:google.protobuf.FieldMask.paths)
 }
-inline ::std::string* FieldMask::add_paths() {
-  // @@protoc_insertion_point(field_add_mutable:google.protobuf.FieldMask.paths)
+inline std::string* FieldMask::_internal_add_paths() {
  return paths_.Add();
 }
-inline void FieldMask::add_paths(const ::std::string& value) {
+inline void FieldMask::add_paths(const std::string& value) {
  paths_.Add()->assign(value);
  // @@protoc_insertion_point(field_add:google.protobuf.FieldMask.paths)
 }
-#if LANG_CXX11
-inline void FieldMask::add_paths(::std::string&& value) {
+inline void FieldMask::add_paths(std::string&& value) {
  paths_.Add(std::move(value));
  // @@protoc_insertion_point(field_add:google.protobuf.FieldMask.paths)
 }
-#endif
 inline void FieldMask::add_paths(const char* value) {
-  GOOGLE_DCHECK(value != NULL);
+  GOOGLE_DCHECK(value != nullptr);
  paths_.Add()->assign(value);
  // @@protoc_insertion_point(field_add_char:google.protobuf.FieldMask.paths)
 }
@@ -242,12 +299,12 @@ inline void FieldMask::add_paths(const char* value, size_t size) {
  paths_.Add()->assign(reinterpret_cast<const char*>(value), size);
  // @@protoc_insertion_point(field_add_pointer:google.protobuf.FieldMask.paths)
 }
-inline const ::google::protobuf::RepeatedPtrField< ::std::string>&
+inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>&
 FieldMask::paths() const {
  // @@protoc_insertion_point(field_list:google.protobuf.FieldMask.paths)
  return paths_;
 }
-inline ::google::protobuf::RepeatedPtrField< ::std::string>*
+inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>*
 FieldMask::mutable_paths() {
  // @@protoc_insertion_point(field_mutable_list:google.protobuf.FieldMask.paths)
  return &paths_;
@@ -259,9 +316,9 @@ FieldMask::mutable_paths() {

 // @@protoc_insertion_point(namespace_scope)

-}  // namespace protobuf
-}  // namespace google
+PROTOBUF_NAMESPACE_CLOSE

 // @@protoc_insertion_point(global_scope)

-#endif  // PROTOBUF_google_2fprotobuf_2ffield_5fmask_2eproto__INCLUDED
+#include <google/protobuf/port_undef.inc>
+#endif  // GOOGLE_PROTOBUF_INCLUDED_GOOGLE_PROTOBUF_INCLUDED_google_2fprotobuf_2ffield_5fmask_2eproto
@@ -41,14 +41,23 @@

 #include <string>

-#include <google/protobuf/stubs/template_util.h>
 #include <google/protobuf/generated_enum_util.h>
+#include <google/protobuf/port.h>
+#include <google/protobuf/stubs/strutil.h>
+
+#ifdef SWIG
+#error "You cannot SWIG proto headers"
+#endif
+
+#include <google/protobuf/port_def.inc>

 namespace google {
 namespace protobuf {
-  class EnumDescriptor;
+class EnumDescriptor;
 }  // namespace protobuf
+}  // namespace google

+namespace google {
 namespace protobuf {

 // Returns the EnumDescriptor for enum type E, which must be a
@@ -59,16 +68,14 @@ const EnumDescriptor* GetEnumDescriptor();

 namespace internal {

-// Helper for EnumType_Parse functions: try to parse the string 'name' as an
-// enum name of the given type, returning true and filling in value on success,
-// or returning false and leaving value unchanged on failure.
-LIBPROTOBUF_EXPORT bool ParseNamedEnum(const EnumDescriptor* descriptor,
-                    const string& name,
-                    int* value);
+// Helper for EnumType_Parse functions: try to parse the string 'name' as
+// an enum name of the given type, returning true and filling in value on
+// success, or returning false and leaving value unchanged on failure.
+PROTOBUF_EXPORT bool ParseNamedEnum(const EnumDescriptor* descriptor,
+                                    ConstStringParam name, int* value);

-template<typename EnumType>
-bool ParseNamedEnum(const EnumDescriptor* descriptor,
-                    const string& name,
+template <typename EnumType>
+bool ParseNamedEnum(const EnumDescriptor* descriptor, ConstStringParam name,
                    EnumType* value) {
  int tmp;
  if (!ParseNamedEnum(descriptor, name, &tmp)) return false;
@@ -79,10 +86,13 @@ bool ParseNamedEnum(const EnumDescriptor* descriptor,
 // Just a wrapper around printing the name of a value. The main point of this
 // function is not to be inlined, so that you can do this without including
 // descriptor.h.
-LIBPROTOBUF_EXPORT const string& NameOfEnum(const EnumDescriptor* descriptor, int value);
+PROTOBUF_EXPORT const std::string& NameOfEnum(const EnumDescriptor* descriptor,
+                                              int value);

 }  // namespace internal
 }  // namespace protobuf
-
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
 #endif  // GOOGLE_PROTOBUF_GENERATED_ENUM_REFLECTION_H__
@@ -31,16 +31,53 @@
 #ifndef GOOGLE_PROTOBUF_GENERATED_ENUM_UTIL_H__
 #define GOOGLE_PROTOBUF_GENERATED_ENUM_UTIL_H__

-#include <google/protobuf/stubs/template_util.h>
+#include <type_traits>
+
+#include <google/protobuf/message_lite.h>
+#include <google/protobuf/stubs/strutil.h>
+
+#include <google/protobuf/port_def.inc>
+
+#ifdef SWIG
+#error "You cannot SWIG proto headers"
+#endif

 namespace google {
 namespace protobuf {

 // This type trait can be used to cause templates to only match proto2 enum
 // types.
-template <typename T> struct is_proto_enum : ::google::protobuf::internal::false_type {};
+template <typename T>
+struct is_proto_enum : ::std::false_type {};

+namespace internal {
+
+// The table entry format for storing enum name-to-value mapping used with lite
+// protos. This struct and the following related functions should only be used
+// by protobuf generated code.
+struct EnumEntry {
+  StringPiece name;
+  int value;
+};
+
+// Looks up a numeric enum value given the string name.
+PROTOBUF_EXPORT bool LookUpEnumValue(const EnumEntry* enums, size_t size,
+                                     StringPiece name, int* value);
+
+// Looks up an enum name given the numeric value.
+PROTOBUF_EXPORT int LookUpEnumName(const EnumEntry* enums,
+                                   const int* sorted_indices, size_t size,
+                                   int value);
+
+// Initializes the list of enum names in std::string form.
+PROTOBUF_EXPORT bool InitializeEnumStrings(
+    const EnumEntry* enums, const int* sorted_indices, size_t size,
+    internal::ExplicitlyConstructed<std::string>* enum_strings);
+
+}  // namespace internal
 }  // namespace protobuf
-
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
 #endif  // GOOGLE_PROTOBUF_GENERATED_ENUM_UTIL_H__
@@ -0,0 +1,87 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// This file contains helpers for generated code.
+//
+//  Nothing in this file should be directly referenced by users of protobufs.
+
+#ifndef GOOGLE_PROTOBUF_GENERATED_MESSAGE_BASES_H__
+#define GOOGLE_PROTOBUF_GENERATED_MESSAGE_BASES_H__
+
+#include <google/protobuf/parse_context.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
+#include <google/protobuf/arena.h>
+#include <google/protobuf/generated_message_util.h>
+#include <google/protobuf/message.h>
+
+// Must come last:
+#include <google/protobuf/port_def.inc>
+
+namespace google {
+namespace protobuf {
+namespace internal {
+
+// To save code size, protos without any fields are derived from ZeroFieldsBase
+// rather than Message.
+class PROTOBUF_EXPORT ZeroFieldsBase : public Message {
+ public:
+  PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
+  bool IsInitialized() const final { return true; }
+  size_t ByteSizeLong() const final;
+  int GetCachedSize() const final { return _cached_size_.Get(); }
+  const char* _InternalParse(const char* ptr,
+                             internal::ParseContext* ctx) final;
+  ::uint8_t* _InternalSerialize(::uint8_t* target,
+                                io::EpsCopyOutputStream* stream) const final;
+
+ protected:
+  constexpr ZeroFieldsBase() {}
+  explicit ZeroFieldsBase(Arena* arena, bool is_message_owned)
+      : Message(arena, is_message_owned) {}
+  ZeroFieldsBase(const ZeroFieldsBase&) = delete;
+  ZeroFieldsBase& operator=(const ZeroFieldsBase&) = delete;
+  ~ZeroFieldsBase() override;
+
+  void SetCachedSize(int size) const final { _cached_size_.Set(size); }
+
+  static void MergeImpl(Message* to, const Message& from);
+  static void CopyImpl(Message* to, const Message& from);
+  void InternalSwap(ZeroFieldsBase* other);
+
+  mutable internal::CachedSize _cached_size_;
+};
+
+}  // namespace internal
+}  // namespace protobuf
+}  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
+#endif  // GOOGLE_PROTOBUF_GENERATED_MESSAGE_BASES_H__
@@ -42,45 +42,36 @@
 #include <vector>
 #include <google/protobuf/stubs/casts.h>
 #include <google/protobuf/stubs/common.h>
-// TODO(jasonh): Remove this once the compiler change to directly include this
-// is released to components.
+#include <google/protobuf/descriptor.h>
 #include <google/protobuf/generated_enum_reflection.h>
-#include <google/protobuf/message.h>
-#include <google/protobuf/metadata.h>
+#include <google/protobuf/stubs/once.h>
+#include <google/protobuf/port.h>
 #include <google/protobuf/unknown_field_set.h>


-namespace google {
-namespace upb {
-namespace google_opensource {
-class GMR_Handlers;
-}  // namespace google_opensource
-}  // namespace upb
+#include <google/protobuf/port_def.inc>

+#ifdef SWIG
+#error "You cannot SWIG proto headers"
+#endif
+
+namespace google {
 namespace protobuf {
-class DescriptorPool;
 class MapKey;
 class MapValueRef;
+class MessageLayoutInspector;
+class Message;
+struct Metadata;
 }  // namespace protobuf
+}  // namespace google

-
-namespace protobuf {
-namespace flat {
-class MetadataBuilder;
-}  // namespace flat
-}  // namespace protobuf
-
-
+namespace google {
 namespace protobuf {
 namespace internal {
 class DefaultEmptyOneof;
-
-// Defined in this file.
-class GeneratedMessageReflection;
-
 // Defined in other files.
-class ExtensionSet;             // extension_set.h
-class WeakFieldMap;             // weak_field_map.h
+class ExtensionSet;  // extension_set.h
+class WeakFieldMap;  // weak_field_map.h

 // This struct describes the internal layout of the message, hence this is
 // used to act on the message reflectively.
@@ -88,17 +79,17 @@ class WeakFieldMap;             // weak_field_map.h
 //                  used to obtain pointers to default instances of embedded
 //                  messages, which GetMessage() will return if the particular
 //                  sub-message has not been initialized yet.  (Thus, all
-//                  embedded message fields *must* have non-NULL pointers
+//                  embedded message fields *must* have non-null pointers
 //                  in the default instance.)
 //   offsets:       An array of ints giving the byte offsets.
 //                  For each oneof or weak field, the offset is relative to the
 //                  default_instance. These can be computed at compile time
 //                  using the
-//                  GOOGLE_PROTOBUF_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET()
+//                  PROTO2_GENERATED_DEFAULT_ONEOF_FIELD_OFFSET()
 //                  macro. For each none oneof field, the offset is related to
 //                  the start of the message object.  These can be computed at
 //                  compile time using the
-//                  GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET() macro.
+//                  PROTO2_GENERATED_MESSAGE_FIELD_OFFSET() macro.
 //                  Besides offsets for all fields, this array also contains
 //                  offsets for oneof unions. The offset of the i-th oneof union
 //                  is offsets[descriptor->field_count() + i].
@@ -116,7 +107,7 @@ class WeakFieldMap;             // weak_field_map.h
 //                  message, or -1 if the message type has no extension
 //                  ranges.
 //   oneof_case_offset:  Offset in the message of an array of uint32s of
-//                  size descriptor->oneof_decl_count().  Each uint32
+//                  size descriptor->oneof_decl_count().  Each uint32_t
 //                  indicates what field is set for each oneof.
 //   object_size:   The size of a message object of this type, as measured
 //                  by sizeof().
@@ -128,62 +119,88 @@ class WeakFieldMap;             // weak_field_map.h
 struct ReflectionSchema {
 public:
  // Size of a google::protobuf::Message object of this type.
-  uint32 GetObjectSize() const { return static_cast<uint32>(object_size_); }
+  uint32_t GetObjectSize() const { return static_cast<uint32_t>(object_size_); }
+
+  bool InRealOneof(const FieldDescriptor* field) const {
+    return field->containing_oneof() &&
+           !field->containing_oneof()->is_synthetic();
+  }

  // Offset of a non-oneof field.  Getting a field offset is slightly more
  // efficient when we know statically that it is not a oneof field.
-  uint32 GetFieldOffsetNonOneof(const FieldDescriptor* field) const {
-    GOOGLE_DCHECK(!field->containing_oneof());
-    return offsets_[field->index()];
+  uint32_t GetFieldOffsetNonOneof(const FieldDescriptor* field) const {
+    GOOGLE_DCHECK(!InRealOneof(field));
+    return OffsetValue(offsets_[field->index()], field->type());
  }

  // Offset of any field.
-  uint32 GetFieldOffset(const FieldDescriptor* field) const {
-    if (field->containing_oneof()) {
+  uint32_t GetFieldOffset(const FieldDescriptor* field) const {
+    if (InRealOneof(field)) {
      size_t offset =
          static_cast<size_t>(field->containing_type()->field_count() +
-          field->containing_oneof()->index());
-      return offsets_[offset];
+                              field->containing_oneof()->index());
+      return OffsetValue(offsets_[offset], field->type());
    } else {
      return GetFieldOffsetNonOneof(field);
    }
  }

-  uint32 GetOneofCaseOffset(const OneofDescriptor* oneof_descriptor) const {
-    return static_cast<uint32>(oneof_case_offset_) +
-           static_cast<uint32>(
-               static_cast<size_t>(oneof_descriptor->index()) * sizeof(uint32));
+  bool IsFieldInlined(const FieldDescriptor* field) const {
+    return Inlined(offsets_[field->index()], field->type());
+  }
+
+  uint32_t GetOneofCaseOffset(const OneofDescriptor* oneof_descriptor) const {
+    return static_cast<uint32_t>(oneof_case_offset_) +
+           static_cast<uint32_t>(
+               static_cast<size_t>(oneof_descriptor->index()) *
+               sizeof(uint32_t));
  }

  bool HasHasbits() const { return has_bits_offset_ != -1; }

  // Bit index within the bit array of hasbits.  Bit order is low-to-high.
-  uint32 HasBitIndex(const FieldDescriptor* field) const {
+  uint32_t HasBitIndex(const FieldDescriptor* field) const {
+    if (has_bits_offset_ == -1) return static_cast<uint32_t>(-1);
    GOOGLE_DCHECK(HasHasbits());
    return has_bit_indices_[field->index()];
  }

  // Byte offset of the hasbits array.
-  uint32 HasBitsOffset() const {
+  uint32_t HasBitsOffset() const {
    GOOGLE_DCHECK(HasHasbits());
-    return static_cast<uint32>(has_bits_offset_);
+    return static_cast<uint32_t>(has_bits_offset_);
+  }
+
+  bool HasInlinedString() const { return inlined_string_donated_offset_ != -1; }
+
+  // Bit index within the bit array of _inlined_string_donated_.  Bit order is
+  // low-to-high.
+  uint32_t InlinedStringIndex(const FieldDescriptor* field) const {
+    GOOGLE_DCHECK(HasInlinedString());
+    return inlined_string_indices_[field->index()];
+  }
+
+  // Byte offset of the _inlined_string_donated_ array.
+  uint32_t InlinedStringDonatedOffset() const {
+    GOOGLE_DCHECK(HasInlinedString());
+    return static_cast<uint32_t>(inlined_string_donated_offset_);
  }

  // The offset of the InternalMetadataWithArena member.
  // For Lite this will actually be an InternalMetadataWithArenaLite.
  // The schema doesn't contain enough information to distinguish between
  // these two cases.
-  uint32 GetMetadataOffset() const {
-    return static_cast<uint32>(metadata_offset_);
+  uint32_t GetMetadataOffset() const {
+    return static_cast<uint32_t>(metadata_offset_);
  }

  // Whether this message has an ExtensionSet.
  bool HasExtensionSet() const { return extensions_offset_ != -1; }

  // The offset of the ExtensionSet in this message.
-  uint32 GetExtensionSetOffset() const {
+  uint32_t GetExtensionSetOffset() const {
    GOOGLE_DCHECK(HasExtensionSet());
-    return static_cast<uint32>(extensions_offset_);
+    return static_cast<uint32_t>(extensions_offset_);
  }

  // The off set of WeakFieldMap when the message contains weak fields.
@@ -196,9 +213,33 @@ struct ReflectionSchema {

  // Returns a pointer to the default value for this field.  The size and type
  // of the underlying data depends on the field's type.
-  const void *GetFieldDefault(const FieldDescriptor* field) const {
-    return reinterpret_cast<const uint8*>(default_instance_) +
-                     offsets_[field->index()];
+  const void* GetFieldDefault(const FieldDescriptor* field) const {
+    return reinterpret_cast<const uint8_t*>(default_instance_) +
+           OffsetValue(offsets_[field->index()], field->type());
+  }
+
+  // Returns true if the field is implicitly backed by LazyField.
+  bool IsEagerlyVerifiedLazyField(const FieldDescriptor* field) const {
+    GOOGLE_DCHECK_EQ(field->type(), FieldDescriptor::TYPE_MESSAGE);
+    (void)field;
+    return false;
+  }
+
+  // Returns true if the field's accessor is called by any external code (aka,
+  // non proto library code).
+  bool IsFieldUsed(const FieldDescriptor* field) const {
+    (void)field;
+    return true;
+  }
+
+  bool IsFieldStripped(const FieldDescriptor* field) const {
+    (void)field;
+    return false;
+  }
+
+  bool IsMessageStripped(const Descriptor* descriptor) const {
+    (void)descriptor;
+    return false;
  }


@@ -209,16 +250,39 @@ struct ReflectionSchema {
  // them, ie.
  //
  //   ReflectionSchema schema = {a, b, c, d, e, ...};
- // private:
+  // private:
  const Message* default_instance_;
-  const uint32* offsets_;
-  const uint32* has_bit_indices_;
+  const uint32_t* offsets_;
+  const uint32_t* has_bit_indices_;
  int has_bits_offset_;
  int metadata_offset_;
  int extensions_offset_;
  int oneof_case_offset_;
  int object_size_;
  int weak_field_map_offset_;
+  const uint32_t* inlined_string_indices_;
+  int inlined_string_donated_offset_;
+
+  // We tag offset values to provide additional data about fields (such as
+  // "unused" or "lazy" or "inlined").
+  static uint32_t OffsetValue(uint32_t v, FieldDescriptor::Type type) {
+    if (type == FieldDescriptor::TYPE_MESSAGE ||
+        type == FieldDescriptor::TYPE_STRING ||
+        type == FieldDescriptor::TYPE_BYTES) {
+      return v & 0x7FFFFFFEu;
+    }
+    return v & 0x7FFFFFFFu;
+  }
+
+  static bool Inlined(uint32_t v, FieldDescriptor::Type type) {
+    if (type == FieldDescriptor::TYPE_STRING ||
+        type == FieldDescriptor::TYPE_BYTES) {
+      return (v & 1u) != 0u;
+    } else {
+      // Non string/byte fields are not inlined.
+      return false;
+    }
+  }
 };

 // Structs that the code generator emits directly to describe a message.
@@ -228,501 +292,71 @@ struct ReflectionSchema {
 // EXPERIMENTAL: these are changing rapidly, and may completely disappear
 // or merge with ReflectionSchema.
 struct MigrationSchema {
-  int32 offsets_index;
-  int32 has_bit_indices_index;
+  int32_t offsets_index;
+  int32_t has_bit_indices_index;
+  int32_t inlined_string_indices_index;
  int object_size;
 };

-// THIS CLASS IS NOT INTENDED FOR DIRECT USE.  It is intended for use
-// by generated code.  This class is just a big hack that reduces code
-// size.
-//
-// A GeneratedMessageReflection is an implementation of Reflection
-// which expects all fields to be backed by simple variables located in
-// memory.  The locations are given using a base pointer and a set of
-// offsets.
-//
-// It is required that the user represents fields of each type in a standard
-// way, so that GeneratedMessageReflection can cast the void* pointer to
-// the appropriate type.  For primitive fields and string fields, each field
-// should be represented using the obvious C++ primitive type.  Enums and
-// Messages are different:
-//  - Singular Message fields are stored as a pointer to a Message.  These
-//    should start out NULL, except for in the default instance where they
-//    should start out pointing to other default instances.
-//  - Enum fields are stored as an int.  This int must always contain
-//    a valid value, such that EnumDescriptor::FindValueByNumber() would
-//    not return NULL.
-//  - Repeated fields are stored as RepeatedFields or RepeatedPtrFields
-//    of whatever type the individual field would be.  Strings and
-//    Messages use RepeatedPtrFields while everything else uses
-//    RepeatedFields.
-class LIBPROTOBUF_EXPORT GeneratedMessageReflection PROTOBUF_FINAL : public Reflection {
- public:
-  // Constructs a GeneratedMessageReflection.
-  // Parameters:
-  //   descriptor:    The descriptor for the message type being implemented.
-  //   schema:        The description of the internal guts of the message.
-  //   pool:          DescriptorPool to search for extension definitions.  Only
-  //                  used by FindKnownExtensionByName() and
-  //                  FindKnownExtensionByNumber().
-  //   factory:       MessageFactory to use to construct extension messages.
-  GeneratedMessageReflection(const Descriptor* descriptor,
-                             const ReflectionSchema& schema,
-                             const DescriptorPool* pool,
-                             MessageFactory* factory);
-
-  ~GeneratedMessageReflection();
-
-  // implements Reflection -------------------------------------------
-
-  const UnknownFieldSet& GetUnknownFields(const Message& message) const;
-  UnknownFieldSet* MutableUnknownFields(Message* message) const;
-
-  size_t SpaceUsedLong(const Message& message) const;
-
-  bool HasField(const Message& message, const FieldDescriptor* field) const;
-  int FieldSize(const Message& message, const FieldDescriptor* field) const;
-  void ClearField(Message* message, const FieldDescriptor* field) const;
-  bool HasOneof(const Message& message,
-                const OneofDescriptor* oneof_descriptor) const;
-  void ClearOneof(Message* message, const OneofDescriptor* field) const;
-  void RemoveLast(Message* message, const FieldDescriptor* field) const;
-  Message* ReleaseLast(Message* message, const FieldDescriptor* field) const;
-  void Swap(Message* message1, Message* message2) const;
-  void SwapFields(Message* message1, Message* message2,
-                  const std::vector<const FieldDescriptor*>& fields) const;
-  void SwapElements(Message* message, const FieldDescriptor* field,
-                    int index1, int index2) const;
-  void ListFields(const Message& message,
-                  std::vector<const FieldDescriptor*>* output) const;
-
-  int32  GetInt32 (const Message& message,
-                   const FieldDescriptor* field) const;
-  int64  GetInt64 (const Message& message,
-                   const FieldDescriptor* field) const;
-  uint32 GetUInt32(const Message& message,
-                   const FieldDescriptor* field) const;
-  uint64 GetUInt64(const Message& message,
-                   const FieldDescriptor* field) const;
-  float  GetFloat (const Message& message,
-                   const FieldDescriptor* field) const;
-  double GetDouble(const Message& message,
-                   const FieldDescriptor* field) const;
-  bool   GetBool  (const Message& message,
-                   const FieldDescriptor* field) const;
-  string GetString(const Message& message,
-                   const FieldDescriptor* field) const;
-  const string& GetStringReference(const Message& message,
-                                   const FieldDescriptor* field,
-                                   string* scratch) const;
-  const EnumValueDescriptor* GetEnum(const Message& message,
-                                     const FieldDescriptor* field) const;
-  int GetEnumValue(const Message& message,
-                   const FieldDescriptor* field) const;
-  const Message& GetMessage(const Message& message,
-                            const FieldDescriptor* field,
-                            MessageFactory* factory = NULL) const;
-
-  const FieldDescriptor* GetOneofFieldDescriptor(
-      const Message& message,
-      const OneofDescriptor* oneof_descriptor) const;
-
- private:
-  bool ContainsMapKey(const Message& message,
-                      const FieldDescriptor* field,
-                      const MapKey& key) const;
-  bool InsertOrLookupMapValue(Message* message,
-                              const FieldDescriptor* field,
-                              const MapKey& key,
-                              MapValueRef* val) const;
-  bool DeleteMapValue(Message* message,
-                      const FieldDescriptor* field,
-                      const MapKey& key) const;
-  MapIterator MapBegin(
-      Message* message,
-      const FieldDescriptor* field) const;
-  MapIterator MapEnd(
-      Message* message,
-      const FieldDescriptor* field) const;
-  int MapSize(const Message& message, const FieldDescriptor* field) const;
-
- public:
-  void SetInt32 (Message* message,
-                 const FieldDescriptor* field, int32  value) const;
-  void SetInt64 (Message* message,
-                 const FieldDescriptor* field, int64  value) const;
-  void SetUInt32(Message* message,
-                 const FieldDescriptor* field, uint32 value) const;
-  void SetUInt64(Message* message,
-                 const FieldDescriptor* field, uint64 value) const;
-  void SetFloat (Message* message,
-                 const FieldDescriptor* field, float  value) const;
-  void SetDouble(Message* message,
-                 const FieldDescriptor* field, double value) const;
-  void SetBool  (Message* message,
-                 const FieldDescriptor* field, bool   value) const;
-  void SetString(Message* message,
-                 const FieldDescriptor* field,
-                 const string& value) const;
-  void SetEnum  (Message* message, const FieldDescriptor* field,
-                 const EnumValueDescriptor* value) const;
-  void SetEnumValue(Message* message, const FieldDescriptor* field,
-                    int value) const;
-  Message* MutableMessage(Message* message, const FieldDescriptor* field,
-                          MessageFactory* factory = NULL) const;
-  void SetAllocatedMessage(Message* message,
-                           Message* sub_message,
-                           const FieldDescriptor* field) const;
-  Message* ReleaseMessage(Message* message, const FieldDescriptor* field,
-                          MessageFactory* factory = NULL) const;
-
-  int32  GetRepeatedInt32 (const Message& message,
-                           const FieldDescriptor* field, int index) const;
-  int64  GetRepeatedInt64 (const Message& message,
-                           const FieldDescriptor* field, int index) const;
-  uint32 GetRepeatedUInt32(const Message& message,
-                           const FieldDescriptor* field, int index) const;
-  uint64 GetRepeatedUInt64(const Message& message,
-                           const FieldDescriptor* field, int index) const;
-  float  GetRepeatedFloat (const Message& message,
-                           const FieldDescriptor* field, int index) const;
-  double GetRepeatedDouble(const Message& message,
-                           const FieldDescriptor* field, int index) const;
-  bool   GetRepeatedBool  (const Message& message,
-                           const FieldDescriptor* field, int index) const;
-  string GetRepeatedString(const Message& message,
-                           const FieldDescriptor* field, int index) const;
-  const string& GetRepeatedStringReference(const Message& message,
-                                           const FieldDescriptor* field,
-                                           int index, string* scratch) const;
-  const EnumValueDescriptor* GetRepeatedEnum(const Message& message,
-                                             const FieldDescriptor* field,
-                                             int index) const;
-  int GetRepeatedEnumValue(const Message& message,
-                           const FieldDescriptor* field,
-                           int index) const;
-  const Message& GetRepeatedMessage(const Message& message,
-                                    const FieldDescriptor* field,
-                                    int index) const;
-
-  // Set the value of a field.
-  void SetRepeatedInt32 (Message* message,
-                         const FieldDescriptor* field, int index, int32  value) const;
-  void SetRepeatedInt64 (Message* message,
-                         const FieldDescriptor* field, int index, int64  value) const;
-  void SetRepeatedUInt32(Message* message,
-                         const FieldDescriptor* field, int index, uint32 value) const;
-  void SetRepeatedUInt64(Message* message,
-                         const FieldDescriptor* field, int index, uint64 value) const;
-  void SetRepeatedFloat (Message* message,
-                         const FieldDescriptor* field, int index, float  value) const;
-  void SetRepeatedDouble(Message* message,
-                         const FieldDescriptor* field, int index, double value) const;
-  void SetRepeatedBool  (Message* message,
-                         const FieldDescriptor* field, int index, bool   value) const;
-  void SetRepeatedString(Message* message,
-                         const FieldDescriptor* field, int index,
-                         const string& value) const;
-  void SetRepeatedEnum(Message* message, const FieldDescriptor* field,
-                       int index, const EnumValueDescriptor* value) const;
-  void SetRepeatedEnumValue(Message* message, const FieldDescriptor* field,
-                            int index, int value) const;
-  // Get a mutable pointer to a field with a message type.
-  Message* MutableRepeatedMessage(Message* message,
-                                  const FieldDescriptor* field,
-                                  int index) const;
-
-  void AddInt32 (Message* message,
-                 const FieldDescriptor* field, int32  value) const;
-  void AddInt64 (Message* message,
-                 const FieldDescriptor* field, int64  value) const;
-  void AddUInt32(Message* message,
-                 const FieldDescriptor* field, uint32 value) const;
-  void AddUInt64(Message* message,
-                 const FieldDescriptor* field, uint64 value) const;
-  void AddFloat (Message* message,
-                 const FieldDescriptor* field, float  value) const;
-  void AddDouble(Message* message,
-                 const FieldDescriptor* field, double value) const;
-  void AddBool  (Message* message,
-                 const FieldDescriptor* field, bool   value) const;
-  void AddString(Message* message,
-                 const FieldDescriptor* field, const string& value) const;
-  void AddEnum(Message* message,
-               const FieldDescriptor* field,
-               const EnumValueDescriptor* value) const;
-  void AddEnumValue(Message* message,
-                    const FieldDescriptor* field,
-                    int value) const;
-  Message* AddMessage(Message* message, const FieldDescriptor* field,
-                      MessageFactory* factory = NULL) const;
-  void AddAllocatedMessage(
-      Message* message, const FieldDescriptor* field,
-      Message* new_entry) const;
-
-  const FieldDescriptor* FindKnownExtensionByName(const string& name) const;
-  const FieldDescriptor* FindKnownExtensionByNumber(int number) const;
-
-  bool SupportsUnknownEnumValues() const;
-
-  // This value for arena_offset_ indicates that there is no arena pointer in
-  // this message (e.g., old generated code).
-  static const int kNoArenaPointer = -1;
-
-  // This value for unknown_field_offset_ indicates that there is no
-  // UnknownFieldSet in this message, and that instead, we are using the
-  // Zero-Overhead Arena Pointer trick. When this is the case, arena_offset_
-  // actually indexes to an InternalMetadataWithArena instance, which can return
-  // either an arena pointer or an UnknownFieldSet or both. It is never the case
-  // that unknown_field_offset_ == kUnknownFieldSetInMetadata && arena_offset_
-  // == kNoArenaPointer.
-  static const int kUnknownFieldSetInMetadata = -1;
-
- protected:
-  void* MutableRawRepeatedField(
-      Message* message, const FieldDescriptor* field, FieldDescriptor::CppType,
-      int ctype, const Descriptor* desc) const;
-
-  const void* GetRawRepeatedField(
-      const Message& message, const FieldDescriptor* field,
-      FieldDescriptor::CppType, int ctype,
-      const Descriptor* desc) const;
-
-  virtual MessageFactory* GetMessageFactory() const;
-
-  virtual void* RepeatedFieldData(
-      Message* message, const FieldDescriptor* field,
-      FieldDescriptor::CppType cpp_type,
-      const Descriptor* message_type) const;
-
- private:
-  friend class google::protobuf::flat::MetadataBuilder;
-  friend class upb::google_opensource::GMR_Handlers;
-
-  const Descriptor* const descriptor_;
-  const ReflectionSchema schema_;
-  const DescriptorPool* const descriptor_pool_;
-  MessageFactory* const message_factory_;
-
-  // Last non weak field index. This is an optimization when most weak fields
-  // are at the end of the containing message. If a message proto doesn't
-  // contain weak fields, then this field equals descriptor_->field_count().
-  int last_non_weak_field_index_;
-
-  template <class T>
-  const T& GetRawNonOneof(const Message& message,
-                          const FieldDescriptor* field) const;
-  template <class T>
-  T* MutableRawNonOneof(Message* message, const FieldDescriptor* field) const;
-
-  template <typename Type>
-  const Type& GetRaw(const Message& message,
-                            const FieldDescriptor* field) const;
-  template <typename Type>
-  inline Type* MutableRaw(Message* message,
-                          const FieldDescriptor* field) const;
-  template <typename Type>
-  inline const Type& DefaultRaw(const FieldDescriptor* field) const;
-
-  inline const uint32* GetHasBits(const Message& message) const;
-  inline uint32* MutableHasBits(Message* message) const;
-  inline uint32 GetOneofCase(
-      const Message& message,
-      const OneofDescriptor* oneof_descriptor) const;
-  inline uint32* MutableOneofCase(
-      Message* message,
-      const OneofDescriptor* oneof_descriptor) const;
-  inline const ExtensionSet& GetExtensionSet(const Message& message) const;
-  inline ExtensionSet* MutableExtensionSet(Message* message) const;
-  inline Arena* GetArena(Message* message) const;
-
-  inline const InternalMetadataWithArena& GetInternalMetadataWithArena(
-      const Message& message) const;
-
-  inline InternalMetadataWithArena*
-      MutableInternalMetadataWithArena(Message* message) const;
-
-  inline bool HasBit(const Message& message,
-                     const FieldDescriptor* field) const;
-  inline void SetBit(Message* message,
-                     const FieldDescriptor* field) const;
-  inline void ClearBit(Message* message,
-                       const FieldDescriptor* field) const;
-  inline void SwapBit(Message* message1,
-                      Message* message2,
-                      const FieldDescriptor* field) const;
-
-  // This function only swaps the field. Should swap corresponding has_bit
-  // before or after using this function.
-  void SwapField(Message* message1,
-                 Message* message2,
-                 const FieldDescriptor* field) const;
-
-  void SwapOneofField(Message* message1,
-                      Message* message2,
-                      const OneofDescriptor* oneof_descriptor) const;
-
-  inline bool HasOneofField(const Message& message,
-                            const FieldDescriptor* field) const;
-  inline void SetOneofCase(Message* message,
-                           const FieldDescriptor* field) const;
-  inline void ClearOneofField(Message* message,
-                              const FieldDescriptor* field) const;
-
-  template <typename Type>
-  inline const Type& GetField(const Message& message,
-                              const FieldDescriptor* field) const;
-  template <typename Type>
-  inline void SetField(Message* message,
-                       const FieldDescriptor* field, const Type& value) const;
-  template <typename Type>
-  inline Type* MutableField(Message* message,
-                            const FieldDescriptor* field) const;
-  template <typename Type>
-  inline const Type& GetRepeatedField(const Message& message,
-                                      const FieldDescriptor* field,
-                                      int index) const;
-  template <typename Type>
-  inline const Type& GetRepeatedPtrField(const Message& message,
-                                         const FieldDescriptor* field,
-                                         int index) const;
-  template <typename Type>
-  inline void SetRepeatedField(Message* message,
-                               const FieldDescriptor* field, int index,
-                               Type value) const;
-  template <typename Type>
-  inline Type* MutableRepeatedField(Message* message,
-                                    const FieldDescriptor* field,
-                                    int index) const;
-  template <typename Type>
-  inline void AddField(Message* message,
-                       const FieldDescriptor* field, const Type& value) const;
-  template <typename Type>
-  inline Type* AddField(Message* message,
-                        const FieldDescriptor* field) const;
-
-  int GetExtensionNumberOrDie(const Descriptor* type) const;
-
-  // Internal versions of EnumValue API perform no checking. Called after checks
-  // by public methods.
-  void SetEnumValueInternal(Message* message,
-                            const FieldDescriptor* field,
-                            int value) const;
-  void SetRepeatedEnumValueInternal(Message* message,
-                                    const FieldDescriptor* field,
-                                    int index,
-                                    int value) const;
-  void AddEnumValueInternal(Message* message,
-                            const FieldDescriptor* field,
-                            int value) const;
-
-
-  Message* UnsafeArenaReleaseMessage(Message* message,
-                                     const FieldDescriptor* field,
-                                     MessageFactory* factory = NULL) const;
-
-  void UnsafeArenaSetAllocatedMessage(Message* message,
-                                      Message* sub_message,
-                                      const FieldDescriptor* field) const;
-
-  internal::MapFieldBase* MapData(
-      Message* message, const FieldDescriptor* field) const;
-
-  friend inline  // inline so nobody can call this function.
-      void
-      RegisterAllTypesInternal(const Metadata* file_level_metadata, int size);
-  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(GeneratedMessageReflection);
+// This struct tries to reduce unnecessary padding.
+// The num_xxx might not be close to their respective pointer, but this saves
+// padding.
+struct PROTOBUF_EXPORT DescriptorTable {
+  mutable bool is_initialized;
+  bool is_eager;
+  int size;  // of serialized descriptor
+  const char* descriptor;
+  const char* filename;
+  once_flag* once;
+  const DescriptorTable* const* deps;
+  int num_deps;
+  int num_messages;
+  const MigrationSchema* schemas;
+  const Message* const* default_instances;
+  const uint32_t* offsets;
+  // update the following descriptor arrays.
+  Metadata* file_level_metadata;
+  const EnumDescriptor** file_level_enum_descriptors;
+  const ServiceDescriptor** file_level_service_descriptors;
 };

-// There are some places in proto2 where dynamic_cast would be useful as an
-// optimization.  For example, take Message::MergeFrom(const Message& other).
-// For a given generated message FooMessage, we generate these two methods:
-//   void MergeFrom(const FooMessage& other);
-//   void MergeFrom(const Message& other);
-// The former method can be implemented directly in terms of FooMessage's
-// inline accessors, but the latter method must work with the reflection
-// interface.  However, if the parameter to the latter method is actually of
-// type FooMessage, then we'd like to be able to just call the other method
-// as an optimization.  So, we use dynamic_cast to check this.
-//
-// That said, dynamic_cast requires RTTI, which many people like to disable
-// for performance and code size reasons.  When RTTI is not available, we
-// still need to produce correct results.  So, in this case we have to fall
-// back to using reflection, which is what we would have done anyway if the
-// objects were not of the exact same class.
-//
-// dynamic_cast_if_available() implements this logic.  If RTTI is
-// enabled, it does a dynamic_cast.  If RTTI is disabled, it just returns
-// NULL.
-//
-// If you need to compile without RTTI, simply #define GOOGLE_PROTOBUF_NO_RTTI.
-// On MSVC, this should be detected automatically.
-template<typename To, typename From>
-inline To dynamic_cast_if_available(From from) {
-#if defined(GOOGLE_PROTOBUF_NO_RTTI) || (defined(_MSC_VER)&&!defined(_CPPRTTI))
-  // Avoid the compiler warning about unused variables.
-  (void)from;
-  return NULL;
-#else
-  return dynamic_cast<To>(from);
-#endif
-}
+enum {
+  // Tag used on offsets for fields that don't have a real offset.
+  // For example, weak message fields go into the WeakFieldMap and not in an
+  // actual field.
+  kInvalidFieldOffsetTag = 0x40000000u,
+};

-// Tries to downcast this message to a generated message type.
-// Returns NULL if this class is not an instance of T.
-//
-// This is like dynamic_cast_if_available, except it works even when
-// dynamic_cast is not available by using Reflection.  However it only works
-// with Message objects.
-//
-// TODO(haberman): can we remove dynamic_cast_if_available in favor of this?
-template <typename T>
-T* DynamicCastToGenerated(const Message* from) {
-  // Compile-time assert that T is a generated type that has a
-  // default_instance() accessor, but avoid actually calling it.
-  const T&(*get_default_instance)() = &T::default_instance;
-  (void)get_default_instance;
+// AssignDescriptors() pulls the compiled FileDescriptor from the DescriptorPool
+// and uses it to populate all of the global variables which store pointers to
+// the descriptor objects.  It also constructs the reflection objects.  It is
+// called the first time anyone calls descriptor() or GetReflection() on one of
+// the types defined in the file.  AssignDescriptors() is thread-safe.
+void PROTOBUF_EXPORT AssignDescriptors(const DescriptorTable* table,
+                                       bool eager = false);

-  // Compile-time assert that T is a subclass of google::protobuf::Message.
-  const Message* unused = static_cast<T*>(NULL);
-  (void)unused;
-
-#if defined(GOOGLE_PROTOBUF_NO_RTTI) || \
-  (defined(_MSC_VER) && !defined(_CPPRTTI))
-  bool ok = &T::default_instance() ==
-            from->GetReflection()->GetMessageFactory()->GetPrototype(
-                from->GetDescriptor());
-  return ok ? down_cast<T*>(from) : NULL;
-#else
-  return dynamic_cast<T*>(from);
-#endif
-}
-
-template <typename T>
-T* DynamicCastToGenerated(Message* from) {
-  const Message* message_const = from;
-  return const_cast<T*>(DynamicCastToGenerated<const T>(message_const));
-}
-
-LIBPROTOBUF_EXPORT void AssignDescriptors(
-    const string& filename, const MigrationSchema* schemas,
-    const Message* const* default_instances_, const uint32* offsets,
-    MessageFactory* factory,
-    // update the following descriptor arrays.
-    Metadata* file_level_metadata,
-    const EnumDescriptor** file_level_enum_descriptors,
-    const ServiceDescriptor** file_level_service_descriptors);
-
-LIBPROTOBUF_EXPORT void RegisterAllTypes(const Metadata* file_level_metadata, int size);
+// Overload used to implement GetMetadataStatic in the generated code.
+// See comments in compiler/cpp/internal/file.cc as to why.
+// It takes a `Metadata` and returns it to allow for tail calls and reduce
+// binary size.
+Metadata PROTOBUF_EXPORT AssignDescriptors(const DescriptorTable* (*table)(),
+                                           internal::once_flag* once,
+                                           const Metadata& metadata);

 // These cannot be in lite so we put them in the reflection.
-LIBPROTOBUF_EXPORT void UnknownFieldSetSerializer(const uint8* base, uint32 offset, uint32 tag,
-                               uint32 has_offset,
-                               ::google::protobuf::io::CodedOutputStream* output);
+PROTOBUF_EXPORT void UnknownFieldSetSerializer(const uint8_t* base,
+                                               uint32_t offset, uint32_t tag,
+                                               uint32_t has_offset,
+                                               io::CodedOutputStream* output);
+
+struct PROTOBUF_EXPORT AddDescriptorsRunner {
+  explicit AddDescriptorsRunner(const DescriptorTable* table);
+};

 }  // namespace internal
 }  // namespace protobuf
-
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
 #endif  // GOOGLE_PROTOBUF_GENERATED_MESSAGE_REFLECTION_H__
@@ -1,103 +0,0 @@
-// Protocol Buffers - Google's data interchange format
-// Copyright 2008 Google Inc.  All rights reserved.
-// https://developers.google.com/protocol-buffers/
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include <google/protobuf/generated_message_table_driven.h>
-
-#include <google/protobuf/stubs/type_traits.h>
-
-#include <google/protobuf/generated_message_table_driven_lite.h>
-#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
-#include <google/protobuf/metadata.h>
-#include <google/protobuf/repeated_field.h>
-#include <google/protobuf/wire_format.h>
-#include <google/protobuf/wire_format_lite.h>
-#include <google/protobuf/wire_format_lite_inl.h>
-
-namespace google {
-namespace protobuf {
-namespace internal {
-
-namespace {
-
-UnknownFieldSet* MutableUnknownFields(MessageLite* msg, int64 arena_offset) {
-  return Raw<InternalMetadataWithArena>(msg, arena_offset)
-      ->mutable_unknown_fields();
-}
-
-struct UnknownFieldHandler {
-  static bool Skip(MessageLite* msg, const ParseTable& table,
-                   io::CodedInputStream* input,
-                   int tag) {
-    GOOGLE_DCHECK(table.unknown_field_set);
-
-    return WireFormat::SkipField(input, tag,
-        MutableUnknownFields(msg, table.arena_offset));
-  }
-
-  static void Varint(MessageLite* msg, const ParseTable& table,
-                     int tag, int value) {
-    GOOGLE_DCHECK(table.unknown_field_set);
-
-    MutableUnknownFields(msg, table.arena_offset)->AddVarint(
-        WireFormatLite::GetTagFieldNumber(tag), value);
-  }
-
-  static bool ParseExtension(
-      MessageLite* msg, const ParseTable& table,
-      io::CodedInputStream* input, int tag) {
-    ExtensionSet* extensions = GetExtensionSet(msg, table.extension_offset);
-    if (extensions == NULL) {
-      return false;
-    }
-
-    const Message* prototype = down_cast<const Message*>(
-        table.default_instance());
-
-    GOOGLE_DCHECK(prototype != NULL);
-    GOOGLE_DCHECK(table.unknown_field_set);
-    UnknownFieldSet* unknown_fields =
-        MutableUnknownFields(msg, table.arena_offset);
-
-    return extensions->ParseField(tag, input, prototype, unknown_fields);
-  }
-};
-
-}  // namespace
-
-bool MergePartialFromCodedStream(
-    MessageLite* msg, const ParseTable& table, io::CodedInputStream* input) {
-  return MergePartialFromCodedStreamImpl<UnknownFieldHandler,
-                                         InternalMetadataWithArena>(msg, table,
-                                                                    input);
-}
-
-}  // namespace internal
-}  // namespace protobuf
-}  // namespace google
@@ -36,10 +36,6 @@
 #include <google/protobuf/map_field_lite.h>
 #include <google/protobuf/message_lite.h>
 #include <google/protobuf/wire_format_lite.h>
-#include <google/protobuf/wire_format_lite_inl.h>
-
-#if LANG_CXX11
-#define PROTOBUF_CONSTEXPR constexpr

 // We require C++11 and Clang to use constexpr for variables, as GCC 4.8
 // requires constexpr to be consistent between declarations of variables
@@ -51,37 +47,72 @@
 #define PROTOBUF_CONSTEXPR_VAR
 #endif  // !_clang

-#else
-#define PROTOBUF_CONSTEXPR
-#define PROTOBUF_CONSTEXPR_VAR
+#ifdef SWIG
+#error "You cannot SWIG proto headers"
 #endif

+#include <google/protobuf/port_def.inc>
+
 namespace google {
 namespace protobuf {
 namespace internal {

 // Processing-type masks.
-static PROTOBUF_CONSTEXPR const unsigned char kOneofMask = 0x40;
-static PROTOBUF_CONSTEXPR const unsigned char kRepeatedMask = 0x20;
+static constexpr const unsigned char kOneofMask = 0x40;
+static constexpr const unsigned char kRepeatedMask = 0x20;
 // Mask for the raw type: either a WireFormatLite::FieldType or one of the
 // ProcessingTypes below, without the oneof or repeated flag.
-static PROTOBUF_CONSTEXPR const unsigned char kTypeMask = 0x1f;
+static constexpr const unsigned char kTypeMask = 0x1f;

 // Wire type masks.
-static PROTOBUF_CONSTEXPR const unsigned char kNotPackedMask = 0x10;
-static PROTOBUF_CONSTEXPR const unsigned char kInvalidMask = 0x20;
+static constexpr const unsigned char kNotPackedMask = 0x10;
+static constexpr const unsigned char kInvalidMask = 0x20;

 enum ProcessingTypes {
  TYPE_STRING_CORD = 19,
  TYPE_STRING_STRING_PIECE = 20,
  TYPE_BYTES_CORD = 21,
  TYPE_BYTES_STRING_PIECE = 22,
-  TYPE_MAP = 23,
+  TYPE_STRING_INLINED = 23,
+  TYPE_BYTES_INLINED = 24,
+  TYPE_MAP = 25,
 };

-#if LANG_CXX11
 static_assert(TYPE_MAP < kRepeatedMask, "Invalid enum");
-#endif
+
+struct PROTOBUF_EXPORT FieldMetadata {
+  uint32_t offset;  // offset of this field in the struct
+  uint32_t tag;     // field * 8 + wire_type
+  // byte offset * 8 + bit_offset;
+  // if the high bit is set then this is the byte offset of the oneof_case
+  // for this field.
+  uint32_t has_offset;
+  uint32_t type;    // the type of this field.
+  const void* ptr;  // auxiliary data
+
+  // From the serializer point of view each fundamental type can occur in
+  // 4 different ways. For simplicity we treat all combinations as a cartesion
+  // product although not all combinations are allowed.
+  enum FieldTypeClass {
+    kPresence,
+    kNoPresence,
+    kRepeated,
+    kPacked,
+    kOneOf,
+    kNumTypeClasses  // must be last enum
+  };
+  // C++ protobuf has 20 fundamental types, were we added Cord and StringPiece
+  // and also distinguish the same types if they have different wire format.
+  enum {
+    kCordType = 19,
+    kStringPieceType = 20,
+    kInlinedType = 21,
+    kNumTypes = 21,
+    kSpecial = kNumTypes * kNumTypeClasses,
+  };
+
+  static int CalculateType(int fundamental_type, FieldTypeClass type_class);
+};

 // TODO(ckennelly):  Add a static assertion to ensure that these masks do not
 // conflict with wiretypes.
@@ -89,12 +120,12 @@ static_assert(TYPE_MAP < kRepeatedMask, "Invalid enum");
 // ParseTableField is kept small to help simplify instructions for computing
 // offsets, as we will always need this information to parse a field.
 // Additional data, needed for some types, is stored in
-// AuxillaryParseTableField.
+// AuxiliaryParseTableField.
 struct ParseTableField {
-  uint32 offset;
+  uint32_t offset;
  // The presence_index ordinarily represents a has_bit index, but for fields
  // inside a oneof it represents the index in _oneof_case_.
-  uint32 presence_index;
+  uint32_t presence_index;
  unsigned char normal_wiretype;
  unsigned char packed_wiretype;

@@ -107,7 +138,7 @@ struct ParseTableField {

 struct ParseTable;

-union AuxillaryParseTableField {
+union AuxiliaryParseTableField {
  typedef bool (*EnumValidator)(int);

  // Enums
@@ -124,7 +155,6 @@ union AuxillaryParseTableField {
    const MessageLite* default_message() const {
      return static_cast<const MessageLite*>(default_message_void);
    }
-    const ParseTable* parse_table;
  };
  message_aux messages;
  // Strings
@@ -139,33 +169,28 @@ union AuxillaryParseTableField {
  };
  map_aux maps;

-#if LANG_CXX11
-  AuxillaryParseTableField() = default;
-#else
-  AuxillaryParseTableField() { }
-#endif
-  PROTOBUF_CONSTEXPR AuxillaryParseTableField(
-      AuxillaryParseTableField::enum_aux e) : enums(e) {}
-  PROTOBUF_CONSTEXPR AuxillaryParseTableField(
-      AuxillaryParseTableField::message_aux m) : messages(m) {}
-  PROTOBUF_CONSTEXPR AuxillaryParseTableField(
-      AuxillaryParseTableField::string_aux s) : strings(s) {}
-  PROTOBUF_CONSTEXPR AuxillaryParseTableField(
-      AuxillaryParseTableField::map_aux m)
+  AuxiliaryParseTableField() = default;
+  constexpr AuxiliaryParseTableField(AuxiliaryParseTableField::enum_aux e)
+      : enums(e) {}
+  constexpr AuxiliaryParseTableField(AuxiliaryParseTableField::message_aux m)
+      : messages(m) {}
+  constexpr AuxiliaryParseTableField(AuxiliaryParseTableField::string_aux s)
+      : strings(s) {}
+  constexpr AuxiliaryParseTableField(AuxiliaryParseTableField::map_aux m)
      : maps(m) {}
 };

 struct ParseTable {
  const ParseTableField* fields;
-  const AuxillaryParseTableField* aux;
+  const AuxiliaryParseTableField* aux;
  int max_field_number;
  // TODO(ckennelly): Do something with this padding.

  // TODO(ckennelly): Vet these for sign extension.
-  int64 has_bits_offset;
-  int64 oneof_case_offset;
-  int64 extension_offset;
-  int64 arena_offset;
+  int64_t has_bits_offset;
+  int64_t oneof_case_offset;
+  int64_t extension_offset;
+  int64_t arena_offset;

  // ExplicitlyInitialized<T> -> T requires a reinterpret_cast, which prevents
  // the tables from being constructed as a constexpr.  We use void to avoid
@@ -178,42 +203,149 @@ struct ParseTable {
  bool unknown_field_set;
 };

-// TODO(jhen): Remove the __NVCC__ check when we get a version of nvcc that
-// supports these checks.
-#if LANG_CXX11 && !defined(__NVCC__)
 static_assert(sizeof(ParseTableField) <= 16, "ParseTableField is too large");
 // The tables must be composed of POD components to ensure link-time
 // initialization.
-static_assert(std::is_pod<ParseTableField>::value, "");
-static_assert(std::is_pod<AuxillaryParseTableField>::value, "");
-static_assert(std::is_pod<AuxillaryParseTableField::enum_aux>::value, "");
-static_assert(std::is_pod<AuxillaryParseTableField::message_aux>::value, "");
-static_assert(std::is_pod<AuxillaryParseTableField::string_aux>::value, "");
-static_assert(std::is_pod<ParseTable>::value, "");
-#endif
+static_assert(std::is_standard_layout<ParseTableField>::value, "");
+static_assert(std::is_trivial<ParseTableField>::value, "");
+static_assert(std::is_standard_layout<AuxiliaryParseTableField>::value, "");
+static_assert(std::is_trivial<AuxiliaryParseTableField>::value, "");
+static_assert(
+    std::is_standard_layout<AuxiliaryParseTableField::enum_aux>::value, "");
+static_assert(std::is_trivial<AuxiliaryParseTableField::enum_aux>::value, "");
+static_assert(
+    std::is_standard_layout<AuxiliaryParseTableField::message_aux>::value, "");
+static_assert(std::is_trivial<AuxiliaryParseTableField::message_aux>::value,
+              "");
+static_assert(
+    std::is_standard_layout<AuxiliaryParseTableField::string_aux>::value, "");
+static_assert(std::is_trivial<AuxiliaryParseTableField::string_aux>::value, "");
+static_assert(std::is_standard_layout<ParseTable>::value, "");
+static_assert(std::is_trivial<ParseTable>::value, "");

 // TODO(ckennelly): Consolidate these implementations into a single one, using
 // dynamic dispatch to the appropriate unknown field handler.
 bool MergePartialFromCodedStream(MessageLite* msg, const ParseTable& table,
                                 io::CodedInputStream* input);
 bool MergePartialFromCodedStreamLite(MessageLite* msg, const ParseTable& table,
-                                 io::CodedInputStream* input);
+                                     io::CodedInputStream* input);

 template <typename Entry>
 bool ParseMap(io::CodedInputStream* input, void* map_field) {
  typedef typename MapEntryToMapField<Entry>::MapFieldType MapFieldType;
-  typedef google::protobuf::Map<typename Entry::EntryKeyType,
-                      typename Entry::EntryValueType>
+  typedef Map<typename Entry::EntryKeyType, typename Entry::EntryValueType>
      MapType;
  typedef typename Entry::template Parser<MapFieldType, MapType> ParserType;

  ParserType parser(static_cast<MapFieldType*>(map_field));
-  return ::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(input,
-                                                                  &parser);
+  return WireFormatLite::ReadMessageNoVirtual(input, &parser);
+}
+
+struct SerializationTable {
+  int num_fields;
+  const FieldMetadata* field_table;
+};
+
+PROTOBUF_EXPORT void SerializeInternal(const uint8_t* base,
+                                       const FieldMetadata* table,
+                                       int32_t num_fields,
+                                       io::CodedOutputStream* output);
+
+inline void TableSerialize(const MessageLite& msg,
+                           const SerializationTable* table,
+                           io::CodedOutputStream* output) {
+  const FieldMetadata* field_table = table->field_table;
+  int num_fields = table->num_fields - 1;
+  const uint8_t* base = reinterpret_cast<const uint8_t*>(&msg);
+  // TODO(gerbens) This skips the first test if we could use the fast
+  // array serialization path, we should make this
+  // int cached_size =
+  //    *reinterpret_cast<const int32_t*>(base + field_table->offset);
+  // SerializeWithCachedSize(msg, field_table + 1, num_fields, cached_size, ...)
+  // But we keep conformance with the old way for now.
+  SerializeInternal(base, field_table + 1, num_fields, output);
+}
+
+PROTOBUF_EXPORT uint8_t* SerializeInternalToArray(const uint8_t* base,
+                                                  const FieldMetadata* table,
+                                                  int32_t num_fields,
+                                                  bool is_deterministic,
+                                                  uint8_t* buffer);
+
+inline uint8_t* TableSerializeToArray(const MessageLite& msg,
+                                      const SerializationTable* table,
+                                      bool is_deterministic, uint8_t* buffer) {
+  const uint8_t* base = reinterpret_cast<const uint8_t*>(&msg);
+  const FieldMetadata* field_table = table->field_table + 1;
+  int num_fields = table->num_fields - 1;
+  return SerializeInternalToArray(base, field_table, num_fields,
+                                  is_deterministic, buffer);
+}
+
+template <typename T>
+struct CompareHelper {
+  bool operator()(const T& a, const T& b) const { return a < b; }
+};
+
+template <>
+struct CompareHelper<ArenaStringPtr> {
+  bool operator()(const ArenaStringPtr& a, const ArenaStringPtr& b) const {
+    return a.Get() < b.Get();
+  }
+};
+
+struct CompareMapKey {
+  template <typename T>
+  bool operator()(const MapEntryHelper<T>& a,
+                  const MapEntryHelper<T>& b) const {
+    return Compare(a.key_, b.key_);
+  }
+  template <typename T>
+  bool Compare(const T& a, const T& b) const {
+    return CompareHelper<T>()(a, b);
+  }
+};
+
+template <typename MapFieldType, const SerializationTable* table>
+void MapFieldSerializer(const uint8_t* base, uint32_t offset, uint32_t tag,
+                        uint32_t has_offset, io::CodedOutputStream* output) {
+  typedef MapEntryHelper<typename MapFieldType::EntryTypeTrait> Entry;
+  typedef typename MapFieldType::MapType::const_iterator Iter;
+
+  const MapFieldType& map_field =
+      *reinterpret_cast<const MapFieldType*>(base + offset);
+  const SerializationTable* t =
+      table +
+      has_offset;  // has_offset is overloaded for maps to mean table offset
+  if (!output->IsSerializationDeterministic()) {
+    for (Iter it = map_field.GetMap().begin(); it != map_field.GetMap().end();
+         ++it) {
+      Entry map_entry(*it);
+      output->WriteVarint32(tag);
+      output->WriteVarint32(map_entry._cached_size_);
+      SerializeInternal(reinterpret_cast<const uint8_t*>(&map_entry),
+                        t->field_table, t->num_fields, output);
+    }
+  } else {
+    std::vector<Entry> v;
+    for (Iter it = map_field.GetMap().begin(); it != map_field.GetMap().end();
+         ++it) {
+      v.push_back(Entry(*it));
+    }
+    std::sort(v.begin(), v.end(), CompareMapKey());
+    for (int i = 0; i < v.size(); i++) {
+      output->WriteVarint32(tag);
+      output->WriteVarint32(v[i]._cached_size_);
+      SerializeInternal(reinterpret_cast<const uint8_t*>(&v[i]), t->field_table,
+                        t->num_fields, output);
+    }
+  }
 }

 }  // namespace internal
 }  // namespace protobuf
-
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>
+
 #endif  // GOOGLE_PROTOBUF_GENERATED_MESSAGE_TABLE_DRIVEN_H__
@@ -1,109 +0,0 @@
-// Protocol Buffers - Google's data interchange format
-// Copyright 2008 Google Inc.  All rights reserved.
-// https://developers.google.com/protocol-buffers/
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include <google/protobuf/generated_message_table_driven_lite.h>
-
-#include <google/protobuf/stubs/type_traits.h>
-
-#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
-#include <google/protobuf/metadata_lite.h>
-#include <google/protobuf/repeated_field.h>
-#include <google/protobuf/wire_format_lite.h>
-#include <google/protobuf/wire_format_lite_inl.h>
-
-namespace google {
-namespace protobuf {
-namespace internal {
-
-namespace {
-
-string* MutableUnknownFields(MessageLite* msg, int64 arena_offset) {
-  return Raw<InternalMetadataWithArenaLite>(msg, arena_offset)
-      ->mutable_unknown_fields();
-}
-
-struct UnknownFieldHandlerLite {
-  static bool Skip(MessageLite* msg, const ParseTable& table,
-                   io::CodedInputStream* input,
-                   int tag) {
-    GOOGLE_DCHECK(!table.unknown_field_set);
-    ::google::protobuf::io::StringOutputStream unknown_fields_string(
-        MutableUnknownFields(msg, table.arena_offset));
-    ::google::protobuf::io::CodedOutputStream unknown_fields_stream(
-        &unknown_fields_string, false);
-
-    return ::google::protobuf::internal::WireFormatLite::SkipField(
-        input, tag, &unknown_fields_stream);
-  }
-
-  static void Varint(MessageLite* msg, const ParseTable& table,
-                     int tag, int value) {
-    GOOGLE_DCHECK(!table.unknown_field_set);
-
-    ::google::protobuf::io::StringOutputStream unknown_fields_string(
-        MutableUnknownFields(msg, table.arena_offset));
-    ::google::protobuf::io::CodedOutputStream unknown_fields_stream(
-        &unknown_fields_string, false);
-    unknown_fields_stream.WriteVarint32(tag);
-    unknown_fields_stream.WriteVarint32(value);
-  }
-
-  static bool ParseExtension(
-      MessageLite* msg, const ParseTable& table,
-      io::CodedInputStream* input, int tag) {
-    ExtensionSet* extensions = GetExtensionSet(msg, table.extension_offset);
-    if (extensions == NULL) {
-      return false;
-    }
-
-    const MessageLite* prototype = table.default_instance();
-
-    GOOGLE_DCHECK(!table.unknown_field_set);
-    ::google::protobuf::io::StringOutputStream unknown_fields_string(
-        MutableUnknownFields(msg, table.arena_offset));
-    ::google::protobuf::io::CodedOutputStream unknown_fields_stream(
-        &unknown_fields_string, false);
-    return extensions->ParseField(
-        tag, input, prototype, &unknown_fields_stream);
-  }
-};
-
-}  // namespace
-
-bool MergePartialFromCodedStreamLite(
-    MessageLite* msg, const ParseTable& table, io::CodedInputStream* input) {
-  return MergePartialFromCodedStreamImpl<UnknownFieldHandlerLite,
-                                         InternalMetadataWithArenaLite>(
-      msg, table, input);
-}
-
-}  // namespace internal
-}  // namespace protobuf
-}  // namespace google
--- a/Show More
+++ b/Show More