diff --git a/.gitignore b/.gitignore index 36cd4439..eef13e55 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ bin/ codegen/.generated/ codegen/debug*.json obj/ +.mypy_cache/ diff --git a/Makefile b/Makefile index 6f0d7bd2..9b266c96 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,12 @@ format: format-black .PHONY: format-black format-black: - python -m black --line-length=120 --exclude submodules -v . + python -m black --line-length=120 --extend-exclude submodules -v . + +# Type check our own scripts, SDKs in submodules have their own checks +.PHONY: mypy +mypy: + python -m mypy --config-file $(CURDIR)/mypy.ini . .PHONY: update update: diff --git a/codegen/Tools/split-cs-file.py b/codegen/Tools/split-cs-file.py index a8182932..683b6c03 100644 --- a/codegen/Tools/split-cs-file.py +++ b/codegen/Tools/split-cs-file.py @@ -4,6 +4,7 @@ import errno import os import re +import typing SPLIT_RE = re.compile( r'//\s+[-]+\n//\s+', @@ -11,7 +12,7 @@ ) -def main(src_file, dst_dir): +def main(src_file: typing.IO[str], dst_dir: str) -> None: remaining = src_file.read() try: os.makedirs(dst_dir) @@ -20,7 +21,10 @@ def main(src_file, dst_dir): if e.errno != errno.EEXIST: raise - for match in reversed(list(SPLIT_RE.finditer(remaining))[1:]): + found = list(reversed(list(SPLIT_RE.finditer(remaining))[1:])) + if not found: + raise SystemExit("No parts matching regex '%s' found in %s" % (SPLIT_RE.pattern, src_file.name)) + for match in found: start_pos = match.span()[0] with open(os.path.join(dst_dir, match.groupdict()["file"]), "wt") as out_f: @@ -34,14 +38,14 @@ def main(src_file, dst_dir): src_file.close() -def parse_args(): +def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("src_file", type=argparse.FileType("rt+")) parser.add_argument("dst_dir", type=str) - args = parser.parse_args() - return vars(args) + return parser.parse_args() if __name__ == "__main__": - main(**parse_args()) + parsed_args = parse_args() + main(parsed_args.src_file, parsed_args.dst_dir) diff --git a/codegen/Tools/split-java-file.py b/codegen/Tools/split-java-file.py index 439cd5fc..e1b53d9e 100644 --- a/codegen/Tools/split-java-file.py +++ b/codegen/Tools/split-java-file.py @@ -4,6 +4,7 @@ import errno import os import re +import typing SPLIT_RE = re.compile( r'//\s+[-]+\n//\s+', @@ -11,7 +12,7 @@ ) -def main(src_file, dst_dir): +def main(src_file: typing.IO[str], dst_dir: str) -> None: remaining = src_file.read() try: os.makedirs(dst_dir) @@ -20,7 +21,10 @@ def main(src_file, dst_dir): if e.errno != errno.EEXIST: raise - for match in reversed(list(SPLIT_RE.finditer(remaining))[1:]): + found = list(reversed(list(SPLIT_RE.finditer(remaining))[1:])) + if not found: + raise SystemExit("No parts matching regex '%s' found in %s" % (SPLIT_RE.pattern, src_file.name)) + for match in found: start_pos = match.span()[0] with open(os.path.join(dst_dir, match.groupdict()["file"]), "wt") as out_f: @@ -34,14 +38,14 @@ def main(src_file, dst_dir): src_file.close() -def parse_args(): +def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("src_file", type=argparse.FileType("rt+")) parser.add_argument("dst_dir", type=str) - args = parser.parse_args() - return vars(args) + return parser.parse_args() if __name__ == "__main__": - main(**parse_args()) + parsed_args = parse_args() + main(parsed_args.src_file, parsed_args.dst_dir) diff --git a/codegen/Tools/split-php-file.py b/codegen/Tools/split-php-file.py index fc8aee22..07db204b 100644 --- a/codegen/Tools/split-php-file.py +++ b/codegen/Tools/split-php-file.py @@ -4,6 +4,7 @@ import errno import os import re +import typing SPLIT_RE = re.compile( r'/[\*]+\s*\n\s\*\s+[-]+\n\s*\*\s+', @@ -11,7 +12,7 @@ ) -def main(src_file, dst_dir): +def main(src_file: typing.IO[str], dst_dir: str) -> None: remaining = src_file.read() try: os.makedirs(dst_dir) @@ -21,7 +22,8 @@ def main(src_file, dst_dir): raise found = list(reversed(list(SPLIT_RE.finditer(remaining))[1:])) - assert found, "No parts matching regex '%s' found" % SPLIT_RE.pattern + if not found: + raise SystemExit("No parts matching regex '%s' found in %s" % (SPLIT_RE.pattern, src_file.name)) for match in found: filename = match.groupdict()["file"] classname = os.path.splitext(filename)[0] @@ -44,14 +46,14 @@ def main(src_file, dst_dir): src_file.close() -def parse_args(): +def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("src_file", type=argparse.FileType("rt+")) parser.add_argument("dst_dir", type=str) - args = parser.parse_args() - return vars(args) + return parser.parse_args() if __name__ == "__main__": - main(**parse_args()) + parsed_args = parse_args() + main(parsed_args.src_file, parsed_args.dst_dir) diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..cee75887 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +python_version = 3.12 +strict = True +# SDKs are generated code with their own type checks in their own repos +exclude = (^|/)submodules/ diff --git a/scripts/new-version.py b/scripts/new-version.py index 9ae8bdd2..69ca4d1e 100755 --- a/scripts/new-version.py +++ b/scripts/new-version.py @@ -7,6 +7,7 @@ import json import os import sys +import typing from datetime import datetime SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -14,102 +15,95 @@ SUBMODULES_DIR = os.path.join(SCRIPT_DIR, "..", "submodules") GO_VERSION_FORMAT = "4.{0}{1:02d}.{2}" +CHANGELOG_TITLE = "# CHANGELOG\n\n" +# Normalized release version (year, month, patch), e.g. 26.7.0 +# The patch is optional on the command line and defaults to 0, see `doc/versioning.md` +Version = tuple[int, int, int] +Config = collections.OrderedDict[str, object] -def get_dart_pub_version(new_version): + +def get_plain_version(new_version: Version) -> str: + return str.join(".", map(str, new_version)) + + +def get_dart_pub_version(new_version: Version) -> str: pub_version = str.join(".", map(str, (4,) + new_version[:2])) if new_version[2] > 0: pub_version += "+" + str(new_version[2]) return pub_version -def set_android_version(new_version, filename=os.path.join(BASE_CONFIG_DIR, "config-android.json")): - config = read_config(filename) - config["artifactVersion"] = str.join(".", map(str, new_version)) - save_config(config, filename) - - -def set_go_version(new_version, filename=os.path.join(BASE_CONFIG_DIR, "config-go.json")): - go_version = GO_VERSION_FORMAT.format(*new_version) - config = read_config(filename) - config["packageVersion"] = go_version - save_config(config, filename) - - -def set_dart_version(new_version, filename=os.path.join(BASE_CONFIG_DIR, "config-dart.json")): - config = read_config(filename) - config["pubVersion"] = get_dart_pub_version(new_version) - save_config(config, filename) - - -def update_dart_changelog(new_version, filename=os.path.join(SUBMODULES_DIR, "dart", "CHANGELOG.md")): - pub_version = get_dart_pub_version(new_version) - entry_header = "## {}".format(pub_version) - - with open(filename, "r", encoding="utf-8") as rf: - changelog = rf.read() - - if entry_header in changelog: - return - - release_month = datetime(2000 + new_version[0], new_version[1], 1).strftime("%B %Y") - entry = "{}\n\n* {} Release\n\n".format(entry_header, release_month) - title = "# CHANGELOG\n\n" - - if changelog.startswith(title): - changelog = title + entry + changelog[len(title):] - else: - changelog = entry + changelog +def get_go_version(new_version: Version) -> str: + return GO_VERSION_FORMAT.format(*new_version) - with open(filename, "w", encoding="utf-8") as wf: - wf.write(changelog) +def get_release_month(new_version: Version) -> str: + return datetime(2000 + new_version[0], new_version[1], 1).strftime("%B %Y") -def set_java_version(new_version, filename=os.path.join(BASE_CONFIG_DIR, "config-java.json")): - config = read_config(filename) - config["artifactVersion"] = str.join(".", map(str, new_version)) - save_config(config, filename) +class Sdk(typing.NamedTuple): + """ + One SDK a release bumps. -def set_net_version(new_version, filename=os.path.join(BASE_CONFIG_DIR, "config-dotnet.json")): - config = read_config(filename) - config["packageVersion"] = str.join(".", map(str, new_version)) - save_config(config, filename) + The config is always `codegen/config-.json` and the changelog, when there + is one, is always `submodules//CHANGELOG.md`, so the name drives both. + See `doc/versioning.md` for the per-language version formats. + """ + name: str + version_field: str + get_version: typing.Callable[[Version], str] = get_plain_version + has_changelog: bool = False + # Swift tags its releases vYY.M.P, so its changelog headers carry a prefix + changelog_prefix: str = "" -def set_node_version(new_version, filename=os.path.join(BASE_CONFIG_DIR, "config-node.json")): - config = read_config(filename) - config["npmVersion"] = str.join(".", map(str, new_version)) - save_config(config, filename) + @property + def config_filename(self) -> str: + return os.path.join(BASE_CONFIG_DIR, "config-{}.json".format(self.name)) + @property + def changelog_filename(self) -> str: + return os.path.join(SUBMODULES_DIR, self.name, "CHANGELOG.md") -def set_php_version(new_version, filename=os.path.join(BASE_CONFIG_DIR, "config-php.json")): - config = read_config(filename) - config["artifactVersion"] = str.join(".", map(str, new_version)) - save_config(config, filename) +# Adding an SDK is one line here +SDKS = ( + Sdk("android", "artifactVersion"), + Sdk("dart", "pubVersion", get_dart_pub_version, has_changelog=True), + Sdk("dotnet", "packageVersion"), + Sdk("go", "packageVersion", get_go_version), + Sdk("java", "artifactVersion"), + Sdk("node", "npmVersion"), + Sdk("php", "artifactVersion"), + Sdk("python", "packageVersion"), + Sdk("swift", "packageVersion", has_changelog=True, changelog_prefix="v"), +) -def set_python_version(new_version, filename=os.path.join(BASE_CONFIG_DIR, "config-python.json")): - config = read_config(filename) - config["packageVersion"] = str.join(".", map(str, new_version)) - save_config(config, filename) +# Every file a release rewrites, checked up front so a bad run cannot bump only some SDKs +RELEASE_FILES = tuple(sdk.config_filename for sdk in SDKS) + tuple( + sdk.changelog_filename for sdk in SDKS if sdk.has_changelog +) -def set_swift_version(new_version, filename=os.path.join(BASE_CONFIG_DIR, "config-swift.json")): - config = read_config(filename) - config["packageVersion"] = str.join(".", map(str, new_version)) - save_config(config, filename) +def check_release_files(filenames: typing.Iterable[str] = RELEASE_FILES) -> None: + # Fail before the first write, otherwise a missing file leaves the SDKs on mixed versions + missing = sorted(name for name in filenames if not os.path.isfile(name)) + if missing: + raise SystemExit("Cannot bump version, missing files:\n " + "\n ".join(missing)) -def read_config(filename): +def read_config(filename: str) -> Config: # Preserve the on-disk key order so a version bump round-trips to a minimal, # stable diff instead of reshuffling hand-ordered keys. with open(filename, "rb") as rf: config = json.load(rf, object_pairs_hook=collections.OrderedDict) + if not isinstance(config, collections.OrderedDict): + raise SystemExit("Expected a JSON object in {}, got {}".format(filename, type(config).__name__)) return config -def save_config(config, filename): +def save_config(config: Config, filename: str) -> None: # Keep key order (no sort_keys) and always end with a single trailing newline # so releases produce stable, POSIX-friendly JSON. with open(filename, "wb") as wf: @@ -117,28 +111,61 @@ def save_config(config, filename): wf.write((string.replace("\r", "") + "\n").encode("utf-8")) -def main(new_versions): - assert 2 <= len(new_versions) <= 3, "Version format should be: 23 7 or 23 7 1" - new_version = tuple(new_versions + [0] * (3 - len(new_versions))) +def set_version(sdk: Sdk, new_version: Version) -> None: + config = read_config(sdk.config_filename) + config[sdk.version_field] = sdk.get_version(new_version) + save_config(config, sdk.config_filename) + + +def update_changelog(sdk: Sdk, new_version: Version) -> None: + entry_header = "## {}{}".format(sdk.changelog_prefix, sdk.get_version(new_version)) + filename = sdk.changelog_filename + + with open(filename, "r", encoding="utf-8") as rf: + changelog = rf.read() + + # Match whole lines only: "## 4.26.4" is a substring of both "## 4.26.4+1" and "## 4.26.41" + if any(line.rstrip() == entry_header for line in changelog.splitlines()): + return + + entry = "{}\n\n* {} Release\n\n".format(entry_header, get_release_month(new_version)) + + if changelog.startswith(CHANGELOG_TITLE): + changelog = CHANGELOG_TITLE + entry + changelog[len(CHANGELOG_TITLE) :] + else: + changelog = entry + changelog + + with open(filename, "w", encoding="utf-8", newline="\n") as wf: + wf.write(changelog) - set_android_version(new_version) - set_dart_version(new_version) - update_dart_changelog(new_version) - set_go_version(new_version) - set_java_version(new_version) - set_net_version(new_version) - set_node_version(new_version) - set_php_version(new_version) - set_python_version(new_version) - set_swift_version(new_version) +def main(new_version: Version) -> None: + check_release_files() -def parse_args(): + for sdk in SDKS: + set_version(sdk, new_version) + if sdk.has_changelog: + update_changelog(sdk, new_version) + + +def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(usage="%s %s" % (sys.argv[0], datetime.today().strftime("%y %m"))) - parser.add_argument("new_versions", type=int, nargs="+", help="Use separate int values like: 21 6 1") + parser.add_argument("year", type=int, help="Two digit release year, like 26") + parser.add_argument("month", type=int, help="Release month, 1-12") + parser.add_argument("patch", type=int, nargs="?", default=0, help="Optional patch number, defaults to 0") args = parser.parse_args() - return vars(args) + + # Validate before anything is written, a bad month used to blow up mid-release + if not 0 <= args.year <= 99: + parser.error("year must be a two digit number in 0..99, got {}".format(args.year)) + if not 1 <= args.month <= 12: + parser.error("month must be in 1..12, got {}".format(args.month)) + if args.patch < 0: + parser.error("patch must not be negative, got {}".format(args.patch)) + + return args if __name__ == "__main__": - main(**parse_args()) + parsed_args = parse_args() + main((parsed_args.year, parsed_args.month, parsed_args.patch)) diff --git a/scripts/open_ai_code_convertion.py b/scripts/open_ai_code_convertion.py deleted file mode 100644 index 9cd0d14e..00000000 --- a/scripts/open_ai_code_convertion.py +++ /dev/null @@ -1,100 +0,0 @@ -import os -from openai import OpenAI -from pathlib import Path -import argparse -import re - -# Environment Variables for OpenAI API Authentication -client = OpenAI( - api_key=os.getenv("OPEN_AI_API_KEY"), - organization=os.getenv("OPEN_AI_ORG_ID"), - project=os.getenv("OPEN_AI_PROJECT_ID"), -) - - -# Helper function to convert file content -def convert_code(content, source_lang, target_lang, target_snippet): - """ - Convert code from one language to another using OpenAI API. - """ - prompt = f""" - Convert the following {source_lang} code to {target_lang}. - Please don't add any comments in code. Don't replace constant class properties with it string values. - Add use statements like in provided snippets in {source_lang}. - Use the provided {target_lang} code snippet as the basis for the conversion: - - {content} - - Here is a code snippet in {target_lang} to use as a reference: - - {target_snippet} - """ - try: - print("Start request to OpenAI") - response = client.chat.completions.create( - model="gpt-4o-mini", - messages=[ - {"role": "system", "content": "You are a helpful code conversion assistant."}, - {"role": "user", "content": prompt}, - ], - ) - print("Getted response") - return response.choices[0].message.content - except Exception as e: - print(f"Error during API call: {e}") - return None - - -# Main function to process files -def process_files(folder, source_ext, target_ext, source_lang, target_lang, target_snippet): - # Pattern to find code snippet in response message - pattern = f"```{target_lang}((.|\s)*?)```" - """ - Walk through the folder and convert files with the specified extension. - """ - for root, _, files in os.walk(folder): - for file in files: - # Process only files with the specified source extension - if file.endswith(source_ext): - file_path = Path(root) / file - print(f"Processing file: {file_path}") - - with open(file_path, "r") as f: - content = f.read() - - converted_content = convert_code(content, source_lang, target_lang, target_snippet) - if converted_content: - matches = re.findall(pattern, converted_content) - if matches: - converted_content = matches[0][0].strip() - target_file_path = file_path.with_suffix(target_ext) - with open(target_file_path, "w") as f: - f.write(converted_content) - print(f"Converted file saved as: {target_file_path}") - else: - print(f"Failed to convert file: {file_path}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Convert source code files between programming languages.") - parser.add_argument("folder", type=str, help="Path to the folder containing source files.") - parser.add_argument("source_ext", type=str, help="Source file extension (e.g., .js).") - parser.add_argument("target_ext", type=str, help="Target file extension (e.g., .py).") - parser.add_argument("source_lang", type=str, help="Source language (e.g., JavaScript).") - parser.add_argument("target_lang", type=str, help="Target language (e.g., Python).") - parser.add_argument("target_snippet_file", type=str, help="File containing a code snippet in the target language.") - - args = parser.parse_args() - - # Read the target snippet from the provided file - try: - with open(args.target_snippet_file, "r") as snippet_file: - target_snippet = snippet_file.read() - except Exception as e: - print(f"Error reading target snippet file: {e}") - exit(1) - - print("Starting code conversion...") - process_files(args.folder, args.source_ext, args.target_ext, args.source_lang, args.target_lang, target_snippet) - - print("Conversion process completed.")