From 15474315cabf0c19384c94a1d8b31adbca929771 Mon Sep 17 00:00:00 2001 From: Isaac Garzon Date: Wed, 14 Sep 2022 16:02:24 +0300 Subject: [PATCH] build: add a version build-tag for non-release builds (#156) The build tag can be set during the build (using either the Makefile or the CMake). If it's not provided, it will be calculated using the state of the git tree since the last release tag (for example, for this PR the build tag will be calculated as `(main+17)-(156-build-add-a-build-tag-into-the-version-for-non-release-builds+1)`. If the branch name can not be determined, a question mark will be used instead. --- CMakeLists.txt | 18 +++ Makefile | 12 +- build_tools/get_build_tag.py | 305 +++++++++++++++++++++++++++++++++++ util/build_version.cc.in | 24 ++- 4 files changed, 354 insertions(+), 5 deletions(-) create mode 100755 build_tools/get_build_tag.py diff --git a/CMakeLists.txt b/CMakeLists.txt index b8d6ec5828..3bdca76663 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1067,6 +1067,24 @@ endif() string(REGEX REPLACE "[^0-9a-fA-F]+" "" GIT_SHA "${GIT_SHA}") string(REGEX REPLACE "[^0-9: /-]+" "" GIT_DATE "${GIT_DATE}") +option(SPDB_RELEASE_BUILD "Create a release build of Speedb" OFF) +set(SPDB_BUILD_TAG "" CACHE STRING "Set a specific build tag for this Speedb build") + +if(NOT SPDB_RELEASE_BUILD AND "${SPDB_BUILD_TAG}" STREQUAL "") + include(FindPython) + find_package(Python COMPONENTS Interpreter) + if(NOT Python_Interpreter_FOUND) + set(SPDB_BUILD_TAG "?") + else() + execute_process( + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE SPDB_BUILD_TAG + COMMAND "${Python_EXECUTABLE}" build_tools/get_build_tag.py OUTPUT_STRIP_TRAILING_WHITESPACE) + if ("${SPDB_BUILD_TAG}" STREQUAL "") + set(SPDB_BUILD_TAG "?") + endif() + endif() +endif() + set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc) configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY) diff --git a/Makefile b/Makefile index 2b738db5d0..a2cb4289cb 100644 --- a/Makefile +++ b/Makefile @@ -747,7 +747,17 @@ else git_mod := $(shell git diff-index HEAD --quiet 2>/dev/null; echo $$?) git_date := $(shell git log -1 --date=iso --format="%ad" 2>/dev/null | awk '{print $1 " " $2}' 2>/dev/null) endif -gen_build_version = sed -e s/@GIT_SHA@/$(git_sha)/ -e s:@GIT_TAG@:"$(git_tag)": -e s/@GIT_MOD@/"$(git_mod)"/ -e s/@BUILD_DATE@/"$(build_date)"/ -e s/@GIT_DATE@/"$(git_date)"/ -e s/@ROCKSDB_PLUGIN_BUILTINS@/'$(ROCKSDB_PLUGIN_BUILTINS)'/ -e s/@ROCKSDB_PLUGIN_EXTERNS@/"$(ROCKSDB_PLUGIN_EXTERNS)"/ util/build_version.cc.in + +SPDB_BUILD_TAG ?= +ifneq (${SPDB_RELEASE_BUILD},1) + ifeq ($(strip ${SPDB_BUILD_TAG}),) + SPDB_BUILD_TAG := $(shell $(PYTHON) "$(CURDIR)/build_tools/get_build_tag.py") + endif + ifeq ($(strip ${SPDB_BUILD_TAG}),) + SPDB_BUILD_TAG := ? + endif +endif +gen_build_version = sed -e s/@GIT_SHA@/$(git_sha)/ -e s:@GIT_TAG@:"$(git_tag)": -e s/@GIT_MOD@/"$(git_mod)"/ -e s/@BUILD_DATE@/"$(build_date)"/ -e s/@GIT_DATE@/"$(git_date)"/ -e s!@SPDB_BUILD_TAG@!"$(SPDB_BUILD_TAG)"! -e s/@ROCKSDB_PLUGIN_BUILTINS@/'$(ROCKSDB_PLUGIN_BUILTINS)'/ -e s/@ROCKSDB_PLUGIN_EXTERNS@/"$(ROCKSDB_PLUGIN_EXTERNS)"/ util/build_version.cc.in # Record the version of the source that we are compiling. # We keep a record of the git revision in this file. It is then built diff --git a/build_tools/get_build_tag.py b/build_tools/get_build_tag.py new file mode 100755 index 0000000000..629635f9dd --- /dev/null +++ b/build_tools/get_build_tag.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python +from __future__ import print_function +import argparse +import os +import re +import subprocess +import sys + + +SPEEDB_URL_PATTERN = re.compile(r".*[/:]speedb-io/speedb.*") +TAG_VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") + + +def split_nonempty_lines(s): + for line in s.splitlines(): + line = line.rstrip() + if line: + yield line + + +def check_output(call, with_stderr=True): + stderr = None if with_stderr else subprocess.DEVNULL + return subprocess.check_output(call, stderr=stderr).rstrip(b"\n").decode("utf-8") + + +def get_suitable_remote(): + for remote in split_nonempty_lines(check_output(["git", "remote", "show"])): + remote = remote.strip() + url = check_output(["git", "remote", "get-url", remote]) + if SPEEDB_URL_PATTERN.match(url): + return remote + + +def get_branch_name(remote, ref, hint=None): + remote_candidates = [] + results = split_nonempty_lines( + check_output( + [ + "git", + "branch", + "-r", + "--contains", + ref, + "--format=%(refname:lstrip=3)", + "{}/*".format(remote), + ] + ) + ) + for result in results: + if result == "main": + return (False, result) + + remote_candidates.append(result) + + local_candidates = [] + results = split_nonempty_lines( + check_output( + ["git", "branch", "--contains", ref, "--format=%(refname:lstrip=2)"] + ) + ) + for result in results: + if result == "main": + return (True, result) + + local_candidates.append(result) + + # Find the most fitting branch by giving more weight to branches that are + # ancestors to the most branches + # + # This will choose A by lexigoraphic order in the following case (the ref + # that we are checking is bracketed): + # BASE * - * - (*) - * - * A + # \ + # * - * B + # This is not a wrong choice, even if originally A was branched from B, + # because without looking at the reflog (which we can't do on build machines) + # there is no way to tell which branch was the "original". Moreover, if B + # is later rebased, A indeed will be the sole branch containing the checked + # commit. + # + # `hint` is used to guide the choice in that case to the branch that we've + # chosen in a previous commit. + all_candidates = [] + for target in remote_candidates: + boost = -0.5 if hint == (False, target) else 0.0 + all_candidates.append( + ( + boost + + sum( + -1.0 + for c in remote_candidates + if is_ancestor_of( + "{}/{}".format(remote, c), "{}/{}".format(remote, target) + ) + ), + (False, target), + ) + ) + for target in local_candidates: + boost = -0.5 if hint == (True, target) else 0.0 + all_candidates.append( + ( + boost + + sum(-1.0 for c in local_candidates if is_ancestor_of(c, target)), + (True, target), + ) + ) + all_candidates.sort() + + if all_candidates: + return all_candidates[0][1] + + # Not on any branch (detached on a commit that isn't referenced by a branch) + return (True, "?") + + +def is_ancestor_of(ancestor, ref): + try: + subprocess.check_output(["git", "merge-base", "--is-ancestor", ancestor, ref]) + except subprocess.CalledProcessError: + return False + else: + return True + + +def get_remote_tags_for_ref(remote, from_ref): + tag_ref_prefix = "refs/tags/" + tags = {} + for line in split_nonempty_lines( + check_output(["git", "ls-remote", "--tags", "--refs", remote]) + ): + h, tag = line.split(None, 1) + if not tag.startswith(tag_ref_prefix) or not is_ancestor_of(h, from_ref): + continue + tags[h] = tag[len(tag_ref_prefix) :] + return tags + + +def get_branches_for_revlist(remote, base_ref, head_ref): + refs_since = tuple( + split_nonempty_lines( + check_output(["git", "rev-list", "{}..{}".format(base_ref, head_ref)]) + ) + ) + + branches = [] + last_branch, last_count = None, 0 + for i, cur_ref in enumerate(refs_since): + cur_branch = get_branch_name(remote, cur_ref, last_branch) + + if cur_branch != last_branch: + if last_count > 0: + branches.append((last_branch, last_count)) + + # All versions are rooted in main, so there's no point to continue + # iterating after hitting it + if cur_branch == ("", "main"): + last_branch, last_count = cur_branch, len(refs_since) - i + break + else: + last_branch, last_count = cur_branch, 1 + else: + last_count += 1 + + if last_count > 0: + branches.append((last_branch, last_count)) + + return branches + + +def is_dirty_worktree(): + try: + subprocess.check_call(["git", "diff-index", "--quiet", "HEAD", "--"]) + except subprocess.CalledProcessError: + return True + else: + return False + + +def get_latest_release_ref(ref, tags): + for line in split_nonempty_lines( + check_output( + ["git", "rev-list", "--no-walk", "--topo-order"] + list(tags.keys()) + ) + ): + line = line.strip() + return (line, tags[line]) + + +def get_current_speedb_version(): + base_path = check_output(["git", "rev-parse", "--show-toplevel"]) + with open(os.path.join(base_path, "speedb", "version.h"), "rb") as f: + data = f.read() + + components = [] + for component in (b"MAJOR", b"MINOR", b"PATCH"): + v = re.search(rb"\s*#\s*define\s+SPEEDB_%b\s+(\d+)" % component, data).group(1) + components.append(int(v.decode("utf-8"))) + + return tuple(components) + + +def which(cmd): + exts = os.environ.get("PATHEXT", "").split(os.pathsep) + for p in os.environ["PATH"].split(os.pathsep): + if not p: + continue + + full_path = os.path.join(p, cmd) + if os.access(full_path, os.X_OK): + return full_path + + for ext in exts: + if not ext: + continue + + check_path = "{}.{}".format(full_path, ext) + if os.access(check_path, os.X_OK): + return check_path + + return None + + +verbose_output = False + + +def info(s): + if s and verbose_output: + print("info: {}".format(s), file=sys.stderr) + + +def exit_unknown(s, additional_components=[]): + print("-".join(["?"] + additional_components)) + info(s) + raise SystemExit(2) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-v", "--verbose", action="store_true", help="print information to stderr" + ) + args = parser.parse_args() + + global verbose_output + verbose_output = bool(args.verbose) + + if not which("git"): + exit_unknown("git wasn't found on your system") + + try: + git_dir = check_output(["git", "rev-parse", "--git-dir"], False) + except subprocess.CalledProcessError: + exit_unknown("not a git repository") + + components = [] + if is_dirty_worktree(): + components.append("*") + + if os.path.isfile(os.path.join(git_dir, "shallow")): + exit_unknown("can't calculate build tag in shallow repository", components) + + remote = get_suitable_remote() + if not remote: + exit_unknown("no suitable remote found", components) + + head_ref = check_output(["git", "rev-parse", "HEAD"]).strip() + remote_tags = { + h: n + for h, n in get_remote_tags_for_ref(remote, head_ref).items() + if TAG_VERSION_PATTERN.match(n) + } + + if not remote_tags: + exit_unknown("no tags found for remote {}".format(remote)) + + base_ref, release_name = get_latest_release_ref(head_ref, remote_tags) + current_ver = ".".join(str(v) for v in get_current_speedb_version()) + if current_ver != release_name[1:]: + info( + "current version doesn't match latest release tag (current={}, tag={})".format( + current_ver, release_name[1:] + ) + ) + components.append("(tag:{})".format(release_name)) + else: + info("latest release is {} ({})".format(release_name, base_ref)) + info("current Speedb version is {}".format(current_ver)) + + branches = get_branches_for_revlist(remote, base_ref, head_ref) + + for (is_local, name), commits in reversed(branches): + components.append( + "({}{}+{})".format( + "#" if is_local else "", + re.sub(r"([#()+\"])", r"\\\1", name.replace("\\", "\\\\")), + commits, + ) + ) + + print("-".join(components)) + + +if __name__ == "__main__": + main() diff --git a/util/build_version.cc.in b/util/build_version.cc.in index 6fd2f430a9..ccea4f4a00 100644 --- a/util/build_version.cc.in +++ b/util/build_version.cc.in @@ -22,6 +22,9 @@ static const std::string speedb_build_date = "speedb_build_date:@GIT_DATE@"; static const std::string speedb_build_date = "speedb_build_date:@BUILD_DATE@"; #endif +#define SPDB_BUILD_TAG "@SPDB_BUILD_TAG@" +static const std::string speedb_build_tag = "speedb_build_tag:" SPDB_BUILD_TAG; + #ifndef ROCKSDB_LITE extern "C" { @ROCKSDB_PLUGIN_EXTERNS@ @@ -50,6 +53,11 @@ static std::unordered_map* LoadPropertiesSet() { AddProperty(properties, speedb_build_git_sha); AddProperty(properties, speedb_build_git_tag); AddProperty(properties, speedb_build_date); + if (SPDB_BUILD_TAG[0] == '@') { + AddProperty(properties, "?"); + } else { + AddProperty(properties, speedb_build_tag); + } return properties; } @@ -61,10 +69,18 @@ const std::unordered_map& GetRocksBuildProperties() { std::string GetRocksVersionAsString(bool with_patch) { std::string version = ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR); if (with_patch) { - return version + "." + ToString(ROCKSDB_PATCH); - } else { - return version; - } + version += "." + ToString(SPEEDB_PATCH); + // Only add a build tag if it was specified (e.g. not a release build) + if (SPDB_BUILD_TAG[0] != '\0') { + if (SPDB_BUILD_TAG[0] == '@') { + // In case build tag substitution at build time failed, add a question mark + version += "-?"; + } else { + version += "-" + std::string(SPDB_BUILD_TAG); + } + } + } + return version; } std::string GetSpeedbVersionAsString(bool with_patch) {