opencv/3rdparty/harfbuzz/hb_extract.py

#!/usr/bin/env python3
"""
hb_extract.py -- copy the minimal subset of HarfBuzz sources used by OpenCV.

OpenCV vendors HarfBuzz as a normal static library: every .cc is a separate
translation unit (no unity / amalgamation build). This script reproduces the
exact src/ tree that OpenCV ships, from a pristine HarfBuzz checkout.

It does three things beyond a naive copy, all to keep the vendored tree small
and buildable with OpenCV's configuration (HB_TINY + HB_HAS_RASTER, see
CMakeLists.txt):

  1. Copies the translation units that upstream `harfbuzz-world.cc` would
     compile for the requested HB_HAS_* sections -- EXCEPT the ones that build
     to an empty object under our config (see EMPTY_TUS). It never emits
     `harfbuzz-world.cc` itself (we compile each .cc directly), and never
     copies non-TU .cc files found under src/OT|graph (only real TUs from the
     parsed world.cc lists are copied), so CMake's file(GLOB_RECURSE src/*.cc)
     picks up exactly the right set.

  2. Prunes headers down to the set actually reachable by #include from the
     copied TUs (+ the public headers OpenCV includes). This automatically
     drops the large amount of HarfBuzz src/ that we never compile -- GPU and
     WASM backends, the subset/repacker graph, platform backends (CoreText/
     DirectWrite/GDI/Uniscribe/Graphite2/GObject/Cairo), the vector-paint
     backend, and headers used only by the skipped empty TUs -- without a
     hardcoded blocklist.

  3. Generates `hb-opencv-config.hh`, the config-override header CMakeLists.txt
     points HB_CONFIG_OVERRIDE_H at. It re-enables thread-safety (HB_NO_MT) and
     variable fonts (HB_NO_VAR) that HB_TINY would otherwise switch off. (It
     does NOT generate upstream's hb-features.h: nothing we compile includes
     it.)

Usage (to refresh this very directory, just point it at a harfbuzz checkout):
  python hb_extract.py path/to/harfbuzz

  The defaults already produce the OpenCV subset: output '.', features
  HB_HAS_RASTER, and it also copies README.md + COPYING. Override only if needed:
    -o DIR              output root (default '.')
    -f FLAGS            comma-separated HB_HAS_* (default HB_HAS_RASTER)
    -a FILES            extra files (default README.md,COPYING)
    --list-flags        print available HB_HAS_* flags and exit

-a paths are relative to the harfbuzz repo root (parent of src/):
  -a README.md            -> copied to   <out>/README.md
  -a COPYING              -> copied to   <out>/COPYING
  -a src/hb-raster.h      -> copied to   <out>/src/hb-raster.h
"""

import argparse
import posixpath
import re
import shutil
import sys
from pathlib import Path


# Translation units that compile to an EMPTY object (libtool: "has no symbols")
# under OpenCV's HarfBuzz configuration (HB_TINY + HB_HAS_RASTER => AAT, legacy
# fallback shaping, math, meta, name, buffer (de)serialize/verify and the style
# API are all compiled out). They contribute nothing, so we do not vendor them.
# Re-derive this list (from the libtool warnings of a clean libharfbuzz build)
# if the HarfBuzz configuration in CMakeLists.txt changes.
EMPTY_TUS = {
    "hb-aat-layout.cc",
    "hb-aat-map.cc",
    "hb-buffer-serialize.cc",
    "hb-buffer-verify.cc",
    "hb-fallback-shape.cc",
    "hb-ot-math.cc",
    "hb-ot-meta.cc",
    "hb-ot-name.cc",
    "hb-raster.cc",
    "hb-style.cc",
}

# Public headers OpenCV's text engine (drawing_text.cpp) includes directly.
# Used together with the copied TUs as the roots of the header-reachability
# prune.
SEED_PUBLIC_HEADERS = ("hb.h", "hb-ot.h", "hb-raster.h")

# Generated config-override header. CMakeLists.txt builds HarfBuzz with HB_TINY
# and HB_CONFIG_OVERRIDE_H="hb-opencv-config.hh"; this file is included by
# hb-config.hh after HB_TINY/HB_LEAN expand (so the #undef takes effect) but
# before the option-closure derives dependent macros.
OPENCV_CONFIG_HH = '''\
/*
 * OpenCV-specific HarfBuzz configuration override.  GENERATED by hb_extract.py.
 *
 * HarfBuzz is built with HB_TINY (see 3rdparty/harfbuzz/CMakeLists.txt) for the
 * smallest possible footprint. HB_TINY pulls in HB_LEAN + HB_MINI, which would
 * disable two features that OpenCV's text engine relies on. We restore them
 * here.
 *
 * This file is included by hb-config.hh via HB_CONFIG_OVERRIDE_H, i.e. AFTER
 * HB_TINY/HB_LEAN/HB_MINI have expanded their macros (so the #undef below takes
 * effect) but BEFORE the "closure of options" derives dependent macros (e.g.
 * HB_NO_VAR_COMPOSITES from HB_NO_VAR). That ordering is what makes a plain
 * #undef sufficient.
 *
 *   HB_NO_MT  - keep thread-safety. hb_font_t instances live inside cv::FontFace
 *               objects, NOT in the thread_local FontRenderEngine, so a single
 *               FontFace (and its hb_font_t) can be shared across threads; its
 *               reference counting must stay atomic.
 *
 *   HB_NO_VAR - keep variable-font support: the 'wght' axis used for synthetic
 *               weights and named-instance/axis queries (hb_font_set_variations,
 *               hb_ot_var_*). Variable fonts are a primary reason OpenCV adopted
 *               HarfBuzz, so this must remain enabled.
 */

#undef HB_NO_MT
#undef HB_NO_VAR
'''

OPENCV_CONFIG_NAME = "hb-opencv-config.hh"


def parse_world_cc(path):
    """Return (core_files, sections) parsed from harfbuzz-world.cc.

    core_files -- list of .cc paths (relative to src/) in the unconditional
                  "Core library" section.
    sections   -- dict HB_HAS_XXX -> list of .cc paths from that #ifdef block.
    """
    lines = path.read_text(encoding="utf-8").splitlines()
    include_re = re.compile(r'^\s*#include\s+"([^"]+\.cc)"')

    core_files = []
    sections = {}

    STATE_PREAMBLE, STATE_CORE, STATE_IFDEF = "preamble", "core", "ifdef"
    state = STATE_PREAMBLE
    current_flag = None
    ifdef_depth = 0

    for line in lines:
        if state == STATE_PREAMBLE:
            if "/* Core library." in line:
                state = STATE_CORE

        elif state == STATE_CORE:
            m = include_re.match(line)
            if m:
                core_files.append(m.group(1))
            else:
                m = re.match(r"^\s*#ifdef\s+(HB_HAS_\w+)", line)
                if m:
                    current_flag = m.group(1)
                    sections[current_flag] = []
                    state = STATE_IFDEF
                    ifdef_depth = 1

        elif state == STATE_IFDEF:
            if re.match(r"^\s*#if", line):
                ifdef_depth += 1
            elif re.match(r"^\s*#endif", line):
                ifdef_depth -= 1
                if ifdef_depth == 0:
                    state = STATE_CORE
                    current_flag = None
            else:
                m = include_re.match(line)
                if m and current_flag is not None:
                    sections[current_flag].append(m.group(1))

    return core_files, sections


def copy_files(file_list, src_dir, out_src_dir, label=""):
    skipped = []
    for rel in file_list:
        if Path(rel).name in EMPTY_TUS:
            skipped.append(rel)
            continue
        src = src_dir / rel
        dst = out_src_dir / rel
        dst.parent.mkdir(parents=True, exist_ok=True)
        if not src.exists():
            print(f"  [WARN] not found: {src}", file=sys.stderr)
            continue
        shutil.copy2(src, dst)
        print(f"  [{label or 'core'}] src/{rel}")
    for rel in skipped:
        print(f"  [skip-empty] src/{rel}")


def copy_headers(src_dir, out_src_dir, public_h=True):
    """Copy all candidate headers: src/*.hh, src/*.h (public API), and *.h/*.hh
    from src/OT/** and src/graph/**. The unreferenced ones are removed later by
    prune_unreferenced_headers().

    NOTE: .cc files under OT/ and graph/ are intentionally NOT copied here --
    the only ones we need are real translation units, copied via the parsed
    world.cc lists. Copying e.g. src/graph/test-classdef-graph.cc would make
    CMake's file(GLOB_RECURSE src/*.cc) try to compile a non-TU file.
    """
    count = 0

    def _copy(p):
        nonlocal count
        dst = out_src_dir / p.relative_to(src_dir)
        dst.parent.mkdir(parents=True, exist_ok=True)
        shutil.copy2(p, dst)
        count += 1

    for hdr in src_dir.glob("*.hh"):
        _copy(hdr)

    for subdir in ("OT", "graph"):
        d = src_dir / subdir
        if not d.exists():
            continue
        for hdr in d.rglob("*"):
            if hdr.is_file() and hdr.suffix in (".h", ".hh"):
                _copy(hdr)

    if public_h:
        for hdr in src_dir.glob("*.h"):
            _copy(hdr)

    return count


def prune_unreferenced_headers(out_src_dir):
    """Delete every header not reachable by #include from the copied TUs and the
    public seed headers, then drop any directory left empty.

    HarfBuzz's amalgamated src/ ships many headers we never compile (GPU/WASM
    backends, subset/repacker graph, platform backends, vector-paint, and the
    headers used only by the empty TUs we skip). Reachability removes them all
    without a hardcoded list. The edge regex also treats any quoted "...h"/
    "...hh" literal as an include, which covers the `#include HB_STRING_ARRAY_LIST`
    macro-indirection trick (e.g. hb-ot-cff1-std-str.hh, hb-ot-post-macroman.hh).
    """
    files = {p.relative_to(out_src_dir).as_posix()
             for p in out_src_dir.rglob("*") if p.is_file()}
    headers = {f for f in files if f.endswith((".h", ".hh"))}
    edge_re = re.compile(r'#\s*include\s+["<]([^">]+)[">]|"([^"]+\.hh?)"')

    def resolve(inc, including):
        for cand in (posixpath.normpath(posixpath.join(posixpath.dirname(including), inc)),
                     posixpath.normpath(inc)):
            if cand in files:
                return cand
        return None

    seeds = {f for f in files if f.endswith(".cc")}
    seeds |= {h for h in SEED_PUBLIC_HEADERS if h in files}

    seen, stack = set(), list(seeds)
    while stack:
        cur = stack.pop()
        if cur in seen:
            continue
        seen.add(cur)
        try:
            txt = (out_src_dir / cur).read_text(encoding="utf-8", errors="ignore")
        except OSError:
            continue
        for m in edge_re.finditer(txt):
            inc = m.group(1) or m.group(2)
            r = resolve(inc, cur)
            if r and r not in seen:
                stack.append(r)

    needed = {f for f in seen if f.endswith((".h", ".hh"))}
    for h in sorted(headers - needed):
        (out_src_dir / h).unlink()
        print(f"  [prune] src/{h}")

    # remove directories left empty by pruning (deepest first)
    for d in sorted((p for p in out_src_dir.rglob("*") if p.is_dir()),
                    key=lambda p: len(p.parts), reverse=True):
        try:
            d.rmdir()
        except OSError:
            pass

    return len(headers - needed), len(needed)


def write_opencv_config_h(out_src_dir):
    """Generate hb-opencv-config.hh (the HB_CONFIG_OVERRIDE_H header)."""
    (out_src_dir / OPENCV_CONFIG_NAME).write_text(OPENCV_CONFIG_HH, encoding="utf-8")
    print(f"  [gen] src/{OPENCV_CONFIG_NAME}")


def main():
    parser = argparse.ArgumentParser(
        description="Copy the OpenCV subset of HarfBuzz sources.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
examples:
  python hb_extract.py ~/work/harfbuzz -o . -f HB_HAS_RASTER -a README.md,COPYING
  python hb_extract.py ~/work/harfbuzz --list-flags
        """,
    )
    parser.add_argument("harfbuzz_dir", metavar="harfbuzz-dir",
                        help="path to the harfbuzz repo root (contains src/)")
    parser.add_argument("-o", "--output", metavar="DIR", default=".",
                        help="output root directory; sources go into DIR/src/ "
                             "(default: current directory)")
    parser.add_argument("-f", "--features", metavar="FLAGS", default="HB_HAS_RASTER",
                        help="comma-separated HB_HAS_* flags (default: HB_HAS_RASTER)")
    parser.add_argument("-a", "--add", metavar="FILES", default="README.md,COPYING",
                        help="comma-separated files relative to the harfbuzz repo "
                             "root (default: README.md,COPYING)")
    parser.add_argument("--no-public-headers", action="store_true",
                        help="skip copying public src/*.h API headers "
                             "(OT/, graph/, *.hh are always copied)")
    parser.add_argument("--list-flags", action="store_true",
                        help="list available HB_HAS_* flags and exit")
    args = parser.parse_args()

    repo_dir = Path(args.harfbuzz_dir).resolve()
    world_cc_path = repo_dir / "src" / "harfbuzz-world.cc"
    if not world_cc_path.exists():
        print(f"error: {world_cc_path} not found", file=sys.stderr)
        sys.exit(1)

    src_dir = world_cc_path.parent
    core_files, sections = parse_world_cc(world_cc_path)

    if args.list_flags:
        for flag, files in sorted(sections.items()):
            print(f"  {flag:<22}  ({len(files)} .cc files)")
        return

    enabled_flags = [f.strip() for f in args.features.split(",") if f.strip()]
    extra_files = [f.strip() for f in args.add.split(",") if f.strip()]

    out_dir = Path(args.output).resolve()
    out_src_dir = out_dir / "src"

    # Start from a clean src/ so removed-upstream files do not linger.
    if out_src_dir.exists():
        shutil.rmtree(out_src_dir)
    out_src_dir.mkdir(parents=True, exist_ok=True)

    print(f"\nsource : {repo_dir}")
    print(f"output : {out_dir}")
    print(f"flags  : {enabled_flags}\n")

    print("core:")
    copy_files(core_files, src_dir, out_src_dir)

    for flag in enabled_flags:
        flag_files = sections.get(flag)
        if not flag_files:
            print(f"\n[{flag}]: no .cc files (flag not found in harfbuzz-world.cc)")
            continue
        print(f"\n[{flag}]:")
        copy_files(flag_files, src_dir, out_src_dir, label=flag)

    if extra_files:
        print("\nextra:")
        for rel in extra_files:
            src = repo_dir / rel
            dst = out_dir / rel
            dst.parent.mkdir(parents=True, exist_ok=True)
            if not src.exists():
                print(f"  [WARN] not found: {src}", file=sys.stderr)
                continue
            shutil.copy2(src, dst)
            print(f"  [add] {rel}")

    n = copy_headers(src_dir, out_src_dir,
                     public_h=not args.no_public_headers)
    print(f"\nheaders: {n} files copied")

    print("\nprune (unreferenced headers):")
    removed, kept = prune_unreferenced_headers(out_src_dir)
    print(f"  removed {removed}, kept {kept}")

    print("\ngenerated:")
    write_opencv_config_h(out_src_dir)

    all_cc = sorted(out_src_dir.rglob("*.cc"))
    all_h = list(out_src_dir.rglob("*.h")) + list(out_src_dir.rglob("*.hh"))
    print(f"\ndone: {len(all_cc)} .cc  |  {len(all_h)} headers  ->  {out_dir}")


if __name__ == "__main__":
    main()