#!/bin/sh
# Benchmark the full LumoSQL matrix in parallel on one large node merging results into a
# shared SQLite database.

set -eu

# ---- environment -------------------------------------------------------
# YOU NEED TO SET PREFIX, CACHE_DIR and BUILD_DIR. Faster and bigger is better.
export CACHE_DIR="$HOME/my/dir/cache"
export BUILD_DIR="$HOME/my/dir/build"

export PREFIX="$HOME/.local"
# Make tcl tooling, libs, headers and pkgconfig findable first.
export PATH="$PREFIX/bin:$PATH"
export LD_LIBRARY_PATH="$PREFIX/lib:${LD_LIBRARY_PATH:-}"
export LIBRARY_PATH="$PREFIX/lib:${LIBRARY_PATH:-}"
export C_INCLUDE_PATH="$PREFIX/include:${C_INCLUDE_PATH:-}"
export CPATH="$PREFIX/include:${CPATH:-}"
export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
export TCLLIBPATH="$PREFIX/lib"
# Pick a tcl interpreter that exists and we think works. This is in addition to
# Makefile's own tclsh/tclsh8.7/tclsh8.6 autodetection.
for _t in "$PREFIX/bin/tclsh8.6" "$PREFIX/bin/tclsh8.7" "$PREFIX/bin/tclsh" \
          tclsh tclsh8.7 tclsh8.6; do
    if command -v "$_t" >/dev/null 2>&1; then
        export TCL="$_t"
        break
    fi
done
unset _t

# ---- tunable thingies --------------------------------------------------
SQLITE_RANGE='3.30.0 3.35.0 3.40.1 3.45.0 3.50.0 3.53.2'
# Native (stock btree) runs are cheap and need no backend, so sweep more
# SQLite versions for them than the backend cross-products use. This list
# should be a superset of SQLITE_RANGE; section (c) builds these stock-only.
SQLITE_NATIVE_RANGE=${SQLITE_NATIVE_RANGE:-"3.30.0 3.31.1 3.33.0 3.35.0 3.37.2 3.40.1 3.42.0 3.43.0 3.45.0 3.47.0 3.50.0 3.53.2"}
LMDB_RANGE='0.9.25 0.9.31 0.9.35'
LMDBV1_RANGE='1.0'
ROWSUM_VARIANTS='off on'
ENCRYPT_VARIANTS='off on'   # only the lmdbv1 backend honours this
CHECKSUM_VARIANTS='off on'  # only the lmdbv1 backend honours this
# Transaction model benchmarks are for a single datasize. The transaction model
# is a runtime option (build=no). Doesn't apply to native SQLite, just LMDB.
TRANSACTION_VARIANTS=${TRANSACTION_VARIANTS:-"optimistic serialise"}
DATASIZE_OPTIONS=${DATASIZE_OPTIONS:-"10"} 

# NOTE As of June 2026 there is an annoying bug in the build system. To pin
# to everything down to the versions specified above you currently also need to edit
# not-fork.d/lmdb/benchmark/versions (and the same for lmdbv1) and specify the versions
# there with "=" as well.

# How many cores do we have? Fall back to 1 if we cannot tell.
if command -v nproc >/dev/null 2>&1; then
    NCPU=$(nproc)
else
    NCPU=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)
fi
case $NCPU in ''|*[!0-9]*) NCPU=1 ;; esac

# WORKERS is overridable from the environment. Default to the core count
# More workers than cores thrashes memory and I/O, is no faster and adds timing noise.
WORKERS=${WORKERS:-$NCPU}
case $WORKERS in ''|*[!0-9]*|0) WORKERS=1 ;; esac
if [ "$WORKERS" -gt "$NCPU" ]; then
    printf 'WARNING: WORKERS=%s exceeds detected cores (%s); capping to %s.\n' \
        "$WORKERS" "$NCPU" "$NCPU" >&2
    printf '         Set WORKERS explicitly to override this cap.\n' >&2
    WORKERS=$NCPU
fi

# Resolve to an absolute path so every backgrounded worker writes the same
# file regardless of cwd.
DATABASE_NAME=${DATABASE_NAME:-$PWD/benchmarks.sqlite}
case $DATABASE_NAME in /*) ;; *) DATABASE_NAME=$PWD/$DATABASE_NAME ;; esac

# running WORKERS benchmarks concurrently maximises throughput but adds noise
WORKDIR=$(mktemp -d "${TMPDIR:-/tmp}/lumo-bench.XXXXXX")
trap 'rm -rf "$WORKDIR"' EXIT

# ---- warm the not-fork cache once -----------------------------------
echo "=== warming not-fork cache (one-off) ==="
make what SQLITE_VERSIONS="$SQLITE_NATIVE_RANGE $SQLITE_RANGE" LMDB_VERSIONS="$LMDB_RANGE" >/dev/null

# Create serially, so no races on the file-exists/CREATE TABLE window.
echo "=== creating results database $DATABASE_NAME ==="
make database \
    DATABASE_NAME="$DATABASE_NAME" \
    SQLITE_VERSIONS="$SQLITE_RANGE" \
    LMDB_VERSIONS="$LMDB_RANGE" \
    >/dev/null

# Extract the target list from a captured `make targets` output file. 
extract_targets() {
    awk '/^TARGETS=/ { getline; print; exit }' "$1"
}

# Run `make targets ...` as its own statement and append its targets to $all. Capturing to
# a temp file lets us check make's real exit status here.
emit_targets() {
    raw="$WORKDIR/targets-raw"
    if ! make targets "$@" > "$raw" 2>"$WORKDIR/targets-err"; then
        echo "ERROR: 'make targets $*' failed:" >&2
        cat "$WORKDIR/targets-err" >&2
        exit 1
    fi
    extract_targets "$raw" | tr ' ' '\n' | sed '/^$/d' >> "$all"
}

# USE_LMDB=no to suppress 0.9 cross-products, and encrypt only applies to
# the lmdbv1 backend.
all="$WORKDIR/targets-all"
: > "$all"

# Native btree across the backend-paired SQLite range. 
for rowsum in $ROWSUM_VARIANTS; do
    for datasize in $DATASIZE_OPTIONS; do
        emit_targets \
            SQLITE_VERSIONS="$SQLITE_RANGE" \
            USE_LMDB=no USE_LMDBV1=no \
            OPTION_ROWSUM="$rowsum" \
            OPTION_DATASIZE="$datasize"
    done
done

# lmdb 0.9.x backend, all selected transaction models
for rowsum in $ROWSUM_VARIANTS; do
    for txn in $TRANSACTION_VARIANTS; do
        for datasize in $DATASIZE_OPTIONS; do
            emit_targets \
                SQLITE_VERSIONS="$SQLITE_RANGE" \
                LMDB_VERSIONS="$LMDB_RANGE" \
                USE_SQLITE=no USE_LMDBV1=no USE_LMDB=yes \
                OPTION_ROWSUM="$rowsum" \
                OPTION_LMDB_TRANSACTION="$txn" \
                OPTION_DATASIZE="$datasize"
        done
    done
done

# lmdbv1 backend with transaction model, encrypt and checksum cross-product
for rowsum in $ROWSUM_VARIANTS; do
    for txn in $TRANSACTION_VARIANTS; do
        for encrypt in $ENCRYPT_VARIANTS; do
            for checksum in $CHECKSUM_VARIANTS; do
                for datasize in $DATASIZE_OPTIONS; do
                    emit_targets \
                        SQLITE_VERSIONS="$SQLITE_RANGE" \
                        LMDBV1_VERSIONS="$LMDBV1_RANGE" \
                        USE_SQLITE=no USE_LMDB=no USE_LMDBV1=yes \
                        OPTION_ROWSUM="$rowsum" \
                        OPTION_LMDBV1_TRANSACTION="$txn" \
                        OPTION_LMDBV1_ENCRYPT="$encrypt" \
                        OPTION_LMDBV1_CHECKSUM="$checksum" \
                        OPTION_DATASIZE="$datasize"
                done
            done
        done
    done
done

# Native btree only, aross the wider SQLITE_NATIVE_RANGE. These need
# no backend, so build many more SQLite versions here. 
for rowsum in $ROWSUM_VARIANTS; do
    for datasize in $DATASIZE_OPTIONS; do
        emit_targets \
            SQLITE_VERSIONS="$SQLITE_NATIVE_RANGE" \
            USE_LMDB=no USE_LMDBV1=no \
            OPTION_ROWSUM="$rowsum" \
            OPTION_DATASIZE="$datasize"
    done
done

sort -u "$all" -o "$all"

if [ ! -s "$all" ]; then
    echo "ERROR: 'make targets' produced no targets" >&2
    echo "       (not-fork cache empty/corrupt, or enumeration failed)" >&2
    exit 1
fi

n=$(wc -l < "$all")
printf '\n=== %d targets over %d workers ===\n' "$n" "$WORKERS"

# This is more manual control than build.tcl needs, however
# a build failure halts the whole script before any benchmark runs, keeping
# $DATABASE_NAME free of partial results. Also avoids thrashing the not-fork cache.
build_log="$HOME/bench-build.log"

# $all holds BENCHMARK targets (with +datasize-N). datasize is a runtime-only
# option (build=no), so many benchmark targets share one build directory.
# Derive the BUILD targets by stripping the runtime-only suffix. 
strip_runtime() {
    sed -E \
      -e 's/\+datasize-[0-9]+(,[0-9]+)?//g' \
      -e 's/\+lmdb_transaction-[A-Za-z0-9]+//g' \
      -e 's/\+lmdbv1_transaction-[A-Za-z0-9]+//g' \
      -e 's/\+sqlite3_journal-[A-Za-z0-9]+//g' \
      -e 's/\+rowsum_algorithm-[A-Za-z0-9]+//g' \
      -e 's/\+discard_output-[A-Za-z0-9]+//g' \
      -e 's/\+$//'
}
build_all="$WORKDIR/builds-all"
strip_runtime < "$all" | sort -u > "$build_all"

nb=$(wc -l < "$build_all")
echo "=== building $nb binaries for $n benchmark targets (log: $build_log) ==="
build_targets=$(tr '\n' ' ' < "$build_all")
if ! make build \
        TARGETS="$build_targets" \
        > "$build_log" 2>&1; then
    echo "ERROR: build failed; see $build_log" >&2
    tail -40 "$build_log" >&2
    exit 1
fi
echo "build complete"

# The pre-build above is the single serial step to populate cache and build.
# Workers below must not repeat this, not-fork caching is not safe for concurrency.
# This tries to avoid an accidental storm of parallel rebuilds.
missing=""
while IFS= read -r t; do
    [ -n "$t" ] || continue
    if [ ! -x "$BUILD_DIR/$t/sqlite3" ]; then
        missing="$missing $t"
    fi
done < "$build_all"
if [ -n "$missing" ]; then
    echo "ERROR: pre-build did not produce binaries for:$missing" >&2
    echo "       refusing to launch workers because they would race not-fork" >&2
    echo "       See $build_log." >&2
    exit 1
fi
echo "verified: all $nb binaries present under $BUILD_DIR ($n benchmark targets)"

w=0
while [ "$w" -lt "$WORKERS" ]; do
    # Round-robin slice w-of-N: spreads the expensive trunk+LMDB sweep
    # (emitted last by build.tcl) across all workers, not just the final one.
    awk -v n="$WORKERS" -v k="$w" 'NR % n == k' "$all" \
        > "$WORKDIR/slice-$w"
    c=$(wc -l < "$WORKDIR/slice-$w")
    printf '  worker %d: %d targets\n' "$w" "$c"
    w=$((w + 1))
done

printf '\nDATASIZE_OPTIONS=%s DATABASE_NAME=%s\n' "$DATASIZE_OPTIONS" "$DATABASE_NAME"

# Record a cutoff so the post-run accounting counts only the rows this
# invocation writes. The results database accumulates across runs; every
# benchmark row carries a when-run value, so benchmark-filter -since
# "$run_started" scopes the checks to this run no matter how large the
# database grows.
run_started=$(date +%s)
echo "batch cutoff (when-run >=): $run_started"

# This could be running for many hours ...

# No OPTION_ROWSUM or OPTION_DATASIZE: each target's +rowsum-* and +datasize-*
# suffix sets these per target.
# ALWAYS_REBUILD=0 keeps workers from recompiling: the serial pre-build above
# is authoritative, and build.tcl will skip up-to-date targets immediately.
pids=""
w=0
while [ "$w" -lt "$WORKERS" ]; do
    slice="$WORKDIR/slice-$w"
    if [ -s "$slice" ]; then
        chunk=$(tr '\n' ' ' < "$slice")
        log="$HOME/bench-w$w.log"
        nohup make benchmark \
            TARGETS="$chunk" \
            DATABASE_NAME="$DATABASE_NAME" \
            ALWAYS_REBUILD=0 \
            > "$log" 2>&1 &
        pid=$!
        pids="$pids $pid"
        # Record the worker index and log path for this pid in a file keyed by
        # the pid itself. Avoids fragile string matching against a flat list
        # (and the PID-reuse hazard) when reporting failures later.
        printf '%s %s\n' "$w" "$log" > "$WORKDIR/pid-$pid"
        echo "started: worker=$w pid=$pid log=$log"
    fi
    w=$((w + 1))
done

set -- $pids
echo "=== waiting for $# workers ==="
rc=0
for pid in $pids; do
    if ! wait "$pid"; then
        if [ -f "$WORKDIR/pid-$pid" ]; then
            read -r wi wl < "$WORKDIR/pid-$pid"
        else
            wi="?"; wl="(log unknown)"
        fi
        echo "FAILED: worker=$wi (pid $pid) log=$wl" >&2
        rc=1
    fi
done

if [ "$rc" -eq 0 ]; then
    echo "=== all workers finished OK ==="
else
    echo "=== some workers FAILED; inspect \$HOME/bench-w*.log ===" >&2
fi
echo "Logs in \$HOME/bench-w*.log; results merged into $DATABASE_NAME"

# ---- sanity-check this run's results --------------------------------
# Every check is scoped to this run with benchmark-filter.tcl's -since flag
# (rows whose when-run >= the cutoff recorded above), so the numbers stay
# correct regardless how many earlier runs the database holds.
bf() { tclsh tool/benchmark-filter.tcl -db "$DATABASE_NAME" -since "$run_started" "$@" 2>/dev/null; }

# Each target carries its own +datasize-N suffix, so one target is one run.
EXPECTED=$n

this_total=$(bf -count)
completed=$(bf -count -completed)
failed=$(bf -count -failed)
crashed=$(bf -count -crashed)
interrupted=$(bf -count -interrupted)

echo
echo "=== this run's benchmark accounting (benchmark-filter -since $run_started) ==="
printf 'expected (targets)                   : %d\n' "$EXPECTED"
printf 'recorded by this run                 : %d\n' "${this_total:-0}"
printf '  completed OK                        : %d\n' "${completed:-0}"
printf '  failed (a benchmark test failed)    : %d\n' "${failed:-0}"
printf '  crashed (never finished)            : %d\n' "${crashed:-0}"
printf '  interrupted                         : %d\n' "${interrupted:-0}"

if [ "${completed:-0}" -eq "$EXPECTED" ] && [ "${this_total:-0}" -eq "$EXPECTED" ]; then
    echo "VERDICT: complete -- all $EXPECTED runs from this matrix recorded and completed."
else
    short=$(( EXPECTED - ${completed:-0} ))
    echo "VERDICT: INCOMPLETE -- $short of $EXPECTED expected runs not completed (see \$HOME/bench-w*.log)." >&2
    rc=1
fi

echo
echo "=== this run's per-datasize coverage ==="
for ds in $DATASIZE_OPTIONS; do
    want=$(grep -cE "\+datasize-$ds(\+|\$)" "$all")
    got=$(bf -count -completed -datasize "$ds")
    if [ "${got:-0}" -eq "$want" ]; then
        printf '  datasize %-3s : %d/%d  ok\n'         "$ds" "${got:-0}" "$want"
    else
        printf '  datasize %-3s : %d/%d  MISSING %d\n' "$ds" "${got:-0}" "$want" "$(( want - ${got:-0} ))"
        rc=1
    fi
done

echo
echo "=== targets benchmarked more than once (should be empty: slices are disjoint) ==="
sqlite3 "$DATABASE_NAME" "
    SELECT value AS target, COUNT(*) AS runs
    FROM run_data WHERE key='target'
    GROUP BY value HAVING COUNT(*) > 1
    ORDER BY runs DESC, target;
"

echo
[ "$rc" -eq 0 ] && echo "=== SANITY: all checks passed for this run ===" \
                || echo "=== SANITY: problems found (rc=$rc) -- see messages above ===" >&2

exit "$rc"