diff --git a/CHANGELOG.md b/CHANGELOG.md index fa41ace..693840f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,113 @@ # Changelog +## 1.0.0 + +### New + +- **Inheritance**: `struct Dog : Animal { ... }` gets Animal's fields and + methods; methods can be replaced and call the replaced version with + `super.name()`. A Dog can be used wherever an Animal is expected, and + `is_a(value, Animal)` checks it. +- **Operator overloading** with methods named `operator+`, `operator-`, + `operator*`, `operator/`, `operator%`, `operator**`, `operator==`, + `operator<` (which also gives `>`, `<=`, `>=`, `sort`, `min` and `max`) and + `operator[]`. +- **Special methods**: `to_string()` (used by `print`, `str`, f-strings and + string concatenation), `iter()` (for loops and comprehensions), `len()` and + `contains()` (`in`). +- **Typed lists and maps**: `list`, `map`, + `list>`. Elements are checked and converted, and the list or + map keeps checking what's added to it later. +- **Tracebacks**: uncaught errors list the calls they passed through. +- **Deep recursion**: programs run on a thread with a large stack, so + recursion can go 100,000 calls deep (it was about 2,000). +- **`ascend test`** runs every `*_test.asc` / `test_*.asc` file and reports + which passed. +- **Running programs**: `run(["git", "status"])` and `run("ls | wc -l")` + capture the output and exit code. +- **HTTP**: `http_get`, `http_post` and `http_request` (using curl). +- **Files and folders**: `walk_dir`, `remove_dir`, `copy_file`, `move_file`, + `file_size`, `path_join`, `basename`, `dirname`, `extension`, `abs_path`, + `cwd`, `chdir`, `temp_dir`. +- **Data**: `csv_parse`, `csv_stringify`, `base64_encode`, `base64_decode`, + `sha256`, `uuid`. +- **More builtins**: `now`, `format_time`, `set_env`, `platform`, + `capitalize`, `title`, `trim_start`, `trim_end`, `last_index_of`, `lcm`, + `factorial`, `trunc`, `degrees`, `radians`, `union`, `intersect`, + `difference`, `is_a`. + +### Changed + +- Comparing values that can't be compared says what they are: "can't compare + a string with an int". + +## 0.6.0 + +### New + +- **Struct methods**: functions written inside a struct, called as + `value.method()`, get the value as `this`. An `init()` method runs whenever + a value is made. +- **Lambdas**: `x => x * 2`, `(a, b) => a + b`, `(i32 n) => { ... }`. +- **Comprehensions**: `[x * x for x in xs if x > 0]`, with several `for` and + `if` clauses, and map comprehensions `{k: v for k, v in m}`. +- **Destructuring**: `a, b := 1, 2`, `a, b = b, a`, `low, high := minmax(xs)` + (with `return a, b`), and `x, y := point` for structs. +- **Ranges**: `0..10` and `1..=10` (also for chars: `'a'..='z'`), usable in + `for` loops and `in` tests without building a list, and as `switch` cases + and `match` patterns: `case 90..=100:`. +- **Null-safe operators**: `a ?? b` and `a?.b` / `a?.method()`. +- **Named arguments**: `greet("Ada", mark: "?")`, `Box(height: 2)`. +- **Variadic functions**: `i32 sum(i32 ...numbers)`, and `...list` to spread + a list into a call's arguments or another list, or a map into a map. +- **`finally`** blocks, which run however a `try` ends; `try` can also be used + with only `finally`. +- **Raw strings** `r"\d+"` and **multi-line strings** `"""..."""`. +- New builtins: `group_by`, `count_by`, `chunk`, `flat_map`, `min_by`, + `max_by`, `mean`, `median`. + +## 0.5.0 + +### New + +- **C-style syntax**: + - `for (init; condition; step)` loops, with several comma-separated + statements in the first and last parts (`for (i32 i = 0, j = n; i < j; + i++, j--)`), empty parts (`for (;;)`), and `for { }` to loop until + `break`. Closures created in the body see that iteration's variables. + - `++` and `--`, before or after a variable, element or field. + - `switch` / `case` / `default` with C's fall-through and `break`; a case + can list several values (`case 1, 2:`). + - `do { ... } while (condition)`. + - Bodies without braces for `if`, `else`, `while`, `for` and `do`: + `if (x < 0) x = -x`, on the same line or the next. + - Casts: `(f64) total / count`, `(i32) 3.9`, `(char) 65`. + - Several declarations in one statement: `i32 a = 1, b = 2`. + - `double`, `long` and `short` as names for `f64`, `i64` and `i16`. + - `auto x = value` fixes the variable's type to its first value's type. + - A top-level `main()` runs automatically after the rest of the program + (unless the program calls it), with `args` or `(argc, argv)` if it takes + them; an integer result is the exit code. + - `printf` and `sprintf` with C format strings. +- Typed loop variables, and `:` instead of `in`: `for (string s : names)`. +- **Import statements**: `import geometry`, `import lib.shapes as shapes`, + `import "path/file.asc" as name`, `from geometry import area, Point as P` + and `from geometry import *`. The `import(...)` function still works. +- **Structs**: `struct Point { f64 x; f64 y = 0 }` declares a type with typed + fields and a constructor, `Point(1, 2)`. Struct names work as types in + declarations, parameters and return values; fields keep their types, and + unknown fields are errors. Struct values print as `Point{x: 1.0, y: 2.0}`. +- **Enums**: `enum Color { Red, Green = 5, Blue }` makes a constant map of + names to ints. +- Regular expressions: `regex_match`, `regex_search`, `regex_find_all`, + `regex_replace`, `regex_split`. +- Folders: `is_dir`, `make_dir`, `list_dir`. + +### Fixed + +- Floats with few significant digits printed in exponent form: `20.0` + printed as `2e+01`. + ## 0.4.0 ### New diff --git a/CMakeLists.txt b/CMakeLists.txt index ee66444..253bc4a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.10) -project(ascend VERSION 0.4.0 LANGUAGES CXX) +project(ascend VERSION 1.0.0 LANGUAGES CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -23,12 +23,14 @@ add_library(ascend_core STATIC src/numeric.cpp src/typecheck.cpp src/stack_limit.cpp + src/big_stack.cpp src/lexer.cpp src/parser.cpp src/semantic_analyzer.cpp src/value.cpp src/interpreter.cpp src/stdlib.cpp + src/stdlib_extra.cpp src/format.cpp src/json.cpp src/runner.cpp @@ -39,6 +41,10 @@ add_library(ascend_core STATIC ) target_include_directories(ascend_core PUBLIC src/include) +# The interpreter runs programs on a thread with a large stack. +find_package(Threads REQUIRED) +target_link_libraries(ascend_core PUBLIC Threads::Threads) + # GCC 8 keeps std::filesystem in a separate library. if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9) target_link_libraries(ascend_core PUBLIC stdc++fs) @@ -78,6 +84,7 @@ if(ASCEND_BUILD_TESTS) tests/lexer_tests.cpp tests/parser_tests.cpp tests/interpreter_tests.cpp + tests/language_tests.cpp tests/package_tests.cpp ) target_link_libraries(ascend_tests PRIVATE ascend_core) @@ -92,4 +99,7 @@ if(ASCEND_BUILD_TESTS) get_filename_component(example_name ${example} NAME_WE) add_test(NAME example_${example_name} COMMAND ascend ${example}) endforeach() + + # `ascend test` finds and runs the test files among the examples. + add_test(NAME ascend_test_command COMMAND ascend test ${CMAKE_CURRENT_SOURCE_DIR}/examples) endif() diff --git a/README.md b/README.md index f7a2bc6..df4da3d 100644 --- a/README.md +++ b/README.md @@ -2,52 +2,56 @@ [![Build](https://github.com/csubbdev/Ascend/actions/workflows/build.yml/badge.svg)](https://github.com/csubbdev/Ascend/actions/workflows/build.yml) -Ascend is a small, friendly scripting language with optional types, written in -C++17. +Ascend is a friendly, general-purpose scripting language with optional C-style +types. Write quick scripts without types, or add them where you want the +checks: typed variables, functions, structs and collections are enforced, and +many mistakes are reported before the program even runs. It comes with a +package manager, a test runner, a REPL and a large standard library, and runs +on Linux, macOS and Windows. ``` -string fizzbuzz(i32 n) { - if n % 15 == 0 { return "FizzBuzz" } - if n % 3 == 0 { return "Fizz" } - if n % 5 == 0 { return "Buzz" } - return str(n) +struct Animal { + string name + string sound() { return "..." } + string to_string() { return f"{this.name} says {this.sound()}" } } -for i in range(1, 16) { - print(fizzbuzz(i)) +struct Dog : Animal { + string sound() { return "Woof" } } -u8 level = 2 -match level { - 1 => print("easy") - 2, 3 => print("normal") - else => print("hard") -} +list pets = [Dog("Rex"), Animal("Generic")] +for (Animal pet : pets) print(pet) -scores := {ada: 95, linus: 88} -for name, score in scores { - print(f"{name:<6} {score:>3}") -} +scores := {ada: 95, linus: 88, grace: 91} +best := max_by(keys(scores), name => scores[name]) +printf("best: %s with %d\n", best, scores[best]) + +squares := [n * n for n in 1..=10 if n % 2 == 0] +response := http_get("https://api.github.com/repos/csubbdev/Ascend") +if (response.ok) print(json_parse(response.body).full_name) ``` ## Features -- **Optional C-style types**: `u8 age = 30`, `i32 add(i32 a, i32 b) { }` with - `i8`–`i64`, `u8`–`u64`, `byte`, `f32`, `f64`, `bool`, `char`, `string`, - `list`, `map`, `func`, `void` and nullable types like `string?`. Typed - variables, parameters and return values are checked, and integer overflow is - an error instead of a silent wrap. -- Constants, `match`, `if` / `else if`, `while`, `for ... in` over ranges, - lists, strings and maps, `break` / `continue` -- Functions with default parameters, recursion, closures and anonymous functions -- f-strings with format specs: `f"{price:>8.2f}"` -- Hex, binary and octal literals, type suffixes (`255u8`) and bitwise operators -- Unicode strings and a `char` type -- Modules with `import("file")`, and **`apkg`, a package manager** like pip: - `apkg install colors`, then `import("colors")` -- `try` / `catch` / `throw` error handling -- 100+ builtins for strings, lists, maps, math, files, JSON and the system -- Clear error messages that point at the problem, many caught before running: +**The language** +- **Optional C-style types**: `u8 age = 30`, `f64 area(f64 w, f64 h) { }`, with + `i8`–`i64`, `u8`–`u64`, `f32`, `f64`, `bool`, `char`, `string`, nullable + types like `string?`, `auto`, and typed collections like `list` and + `map`. Integer overflow is an error instead of a silent wrap. +- **Structs** with typed fields, methods, inheritance (`struct Dog : Animal`), + operator overloading (`operator+`, `operator==`, `operator<`, ...) and + special methods (`to_string`, `iter`, `len`); **enums**. +- **Functions**: default and named arguments, `...rest` parameters, closures, + lambdas (`x => x * 2`), multiple return values (`return lo, hi`). +- **Expressive code**: comprehensions (`[x * x for x in xs if x > 0]`), ranges + (`0..n`), destructuring (`a, b = b, a`), `??` and `?.`, `match`, f-strings + with format specs, raw and multi-line strings. +- **C-style code works too**: `for (i32 i = 0; i < n; i++)`, `i++`, + `switch`/`case` with fall-through, `do`/`while`, bodies without braces, + casts, `printf`, and a `main()` that runs automatically. +- **Errors**: `try`/`catch`/`finally`/`throw`, clear messages that point at + the problem, and tracebacks through function calls: ``` script.asc:3:10: error: 'age' is declared as u8, so it can't hold 300 (u8 is 0 to 255) @@ -55,22 +59,39 @@ for name, score in scores { | ^ ``` -- An interactive REPL, a `--check` mode, and syntax highlighting for VS Code +**The tools** +- **`apkg`**, a package manager like pip: `apkg install colors`, then + `import colors`. Packages come from the registry, GitHub, any git + repository or a folder. +- **`ascend test`** runs your `*_test.asc` files. +- An interactive REPL, `ascend --check` to find errors without running, and + syntax highlighting for VS Code. + +**The standard library** (170+ builtin functions) +- Strings, lists, maps and sets, math, regular expressions, JSON and CSV +- Files, folders and paths; running programs and capturing their output +- HTTP requests, dates and times, base64, SHA-256, UUIDs ## Quick start ```sh cmake -S . -B build cmake --build build -./build/ascend examples/types.asc -./build/ascend # interactive REPL +./build/ascend examples/shapes.asc # run a program +./build/ascend # interactive REPL +./build/ascend test examples # run the test files in a folder ``` +See [examples/](examples/) for more: [functional.asc](examples/functional.asc), +[methods.asc](examples/methods.asc), [scripting.asc](examples/scripting.asc), +[c_style.asc](examples/c_style.asc) and others. + Install a package and use it: ```sh ./build/apkg install colors -./build/ascend -e 'print(import("colors").green("Hello!"))' +./build/ascend -e 'import colors +print(colors.green("Hello!"))' ``` ## Documentation @@ -90,8 +111,9 @@ Install a package and use it: | Checks before running (types of literals, `break` outside loops, ...) | `src/semantic_analyzer.cpp`, `src/typecheck.cpp` | | Tree-walking interpreter | `src/interpreter.cpp` | | Values, numeric types and checked arithmetic | `src/value.cpp`, `src/numeric.cpp`, `src/types.cpp` | -| Builtin functions, formatting and JSON | `src/stdlib.cpp`, `src/format.cpp`, `src/json.cpp` | -| Command line and REPL | `src/main.cpp`, `src/repl.cpp`, `src/runner.cpp` | +| Builtin functions, formatting and JSON | `src/stdlib.cpp`, `src/stdlib_extra.cpp`, `src/format.cpp`, `src/json.cpp` | +| Running other programs | `src/process.cpp` | +| Command line, REPL, `ascend test` | `src/main.cpp`, `src/repl.cpp`, `src/runner.cpp`, `src/big_stack.cpp` | | Package manager (`apkg`) and finding packages for `import` | `src/apkg_main.cpp`, `src/package_manager.cpp`, `src/packages.cpp` | ## License diff --git a/docs/installation.md b/docs/installation.md index 1ab97bc..81bac7b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -5,9 +5,10 @@ - A C++17 compiler: GCC 8+, Clang 7+ or Visual Studio 2019+ - CMake 3.10 or newer -Ascend has no other dependencies. The package manager, `apkg`, uses -[git](https://git-scm.com) to download packages, so install git too if you -want to use packages from the internet. +Ascend has no other dependencies. A few things use programs you probably +have already: the package manager, `apkg`, uses [git](https://git-scm.com) to +download packages, and `http_get` and the other HTTP functions use +[curl](https://curl.se) (included with macOS and Windows 10 and later). ## Build @@ -27,6 +28,7 @@ Visual Studio). Builds default to the optimized `Release` configuration. ./build/ascend # start the interactive REPL ./build/ascend -e 'print(2 ** 10)' # run a one-liner ./build/ascend --check my_program.asc # find errors without running +./build/ascend test # run the *_test.asc files in this folder ``` To run `ascend` and `apkg` from anywhere, copy them from `build/` into a @@ -35,7 +37,8 @@ directory on your `PATH`, or run `cmake --install build` (add ```sh apkg install colors # see docs/packages.md -ascend -e 'print(import("colors").bold("ready"))' +ascend -e 'import colors +print(colors.bold("ready"))' ``` ## Test diff --git a/docs/packages.md b/docs/packages.md index 3a2987c..cbecb46 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -12,7 +12,7 @@ apkg uninstall colors # remove it again Programs use installed packages with `import`: ``` -colors := import("colors") +import colors print(colors.green("it works!")) ``` @@ -94,7 +94,7 @@ somewhere else. ## How import finds packages -`import("name")` first looks for a file `name.asc` next to the importing file, +`import name` first looks for a file `name.asc` next to the importing file, as always. If there isn't one, it looks for an installed package called `name`: @@ -103,7 +103,8 @@ as always. If there isn't one, it looks for an installed package called 2. in your own packages (`~/.ascend/packages/name`). It runs the package's main file (`main` in its `ascend.json`). To import a -different file from a package, add its path: `import("strings/strings")`. +different file from a package, add its path: `import strings.strings` (or +`import("strings/strings")`). A package that depends on another one imports it the same way, so packages can be used together without knowing where they're installed. @@ -131,7 +132,7 @@ A package is a folder with an `ascend.json`: | `name` | Required. Lowercase letters, digits, `-` and `_`. This is the name used to `import` it. | | `version` | Required. Use `major.minor.patch` numbers. | | `description` | A one-line summary, shown by `apkg show` and `apkg search`. | -| `main` | The file `import("name")` runs. Defaults to `main.asc`. | +| `main` | The file `import name` runs. Defaults to `main.asc`. | | `dependencies` | Other packages it needs: `"*"` for any version from the registry, a version number like `"1.2.0"`, or any source `apkg install` accepts, like `"github:user/repo@v1.0"` or `"file:../other-folder"`. | Everything at the top level of the main file that doesn't start with `_` diff --git a/docs/standard_library.md b/docs/standard_library.md index 80cfab7..4b9ff54 100644 --- a/docs/standard_library.md +++ b/docs/standard_library.md @@ -15,6 +15,17 @@ change their argument. | `print(values...)` | Prints the values separated by spaces, then a newline. `print()` prints an empty line. | | `input(prompt?)` | Prints `prompt` (if given) and reads a line from standard input. Returns `null` at end of input. | | `format(template, values...)` | Fills each `{}` in the template with the next value; `{0}` picks a value by position and `{:spec}` formats it like an f-string: `format("{:>6.2f}", x)`. | +| `printf(format, values...)` | Prints values with a C format string, without adding a newline: `printf("%-10s %5.2f\n", name, price)`. See below. | +| `sprintf(format, values...)` | The same, but returns the text instead of printing it. | + +`printf` and `sprintf` understand C's conversions: `%d` `%i` `%u` (integers; +a char gives its code), `%f` `%e` `%g` (and `%F` `%E` `%G`), `%x` `%X` `%o` +and `%b` (hex, octal, binary), `%c` (a char, or a character code), `%s` (any +value, as `print` shows it) and `%%`, with the flags `-` (left-align), `+`, +space, `0` (pad with zeros) and `#` (`0x` prefix), a width and a +`.precision`, either of which can be `*` to take it from the values. Length +modifiers like `l`, `ll` and `z` are accepted and ignored. Giving more or +fewer values than the format uses is an error. ## Types and conversion @@ -22,15 +33,16 @@ Each type name converts a value to that type, and fails if it doesn't fit. | Function | Description | |----------|-------------| -| `type(x)` | The type's name: `"int"`, `"u8"`, `"f32"`, `"char"`, `"string"`, `"list"`, `"map"`, `"func"`, `"bool"`, `"null"`, ... | -| `int(x)`, `i8(x)`, `i16(x)`, `i32(x)`, `i64(x)` | Convert to a signed integer type. Floats drop their fraction; chars give their code; strings are parsed (decimal, or `0x`/`0b`/`0o`). | +| `type(x)` | The type's name: `"int"`, `"u8"`, `"f32"`, `"char"`, `"string"`, `"list"`, `"map"`, `"func"`, `"bool"`, `"null"`, ..., or a struct's name like `"Point"`. | +| `int(x)`, `i8(x)`, `i16(x)`, `i32(x)`, `i64(x)`, `short(x)`, `long(x)` | Convert to a signed integer type. Floats drop their fraction; chars give their code; strings are parsed (decimal, or `0x`/`0b`/`0o`). | | `u8(x)`, `u16(x)`, `u32(x)`, `u64(x)`, `byte(x)` | The same for unsigned types. `u8(-1)` and `u8(256)` are errors. | -| `float(x)`, `f32(x)`, `f64(x)` | Convert a number, bool or numeric string to a float type. | +| `float(x)`, `f32(x)`, `f64(x)`, `double(x)` | Convert a number, bool or numeric string to a float type. | | `char(x)` | A char from a character code (`char(65)` is `'A'`) or a one-character string. | | `str(x)`, `string(x)` | `x` as a string, the way `print` shows it. | | `repr(x)` | `x` as source-like text (strings and chars are quoted and escaped). | | `bool(x)` | `true` if `x` is truthy. | | `hex(n)`, `bin(n)` | An integer in hex or binary: `hex(255)` is `"0xff"`. | +| `is_a(x, type)` | `true` if `x` is a value of the struct `type` (or of a struct that extends it), or if `type` is a type name string matching `type(x)`: `is_a(d, Animal)`, `is_a(5, "int")`. | ## Lists @@ -59,7 +71,12 @@ Each type name converts a value to that type, and fails if it doesn't fit. | `reduce(list, f, initial?)` | Combines the elements: `reduce([1, 2, 3], func(a, b) { return a + b })` is `6`. | | `find(list, f)` | The first element for which `f(element)` is truthy, or `null`. | | `any(list, f?)`, `all(list, f?)` | Whether any / all elements (or `f(element)`) are truthy. | -| `copy(x)` | A shallow copy of a list or map. | +| `flat_map(list, f)` | Calls `f` on each element and joins the lists it returns: `flat_map([1, 2], x => [x, x])` is `[1, 1, 2, 2]`. | +| `group_by(list, f)` | A map from each `f(element)` to the list of elements that gave it: `group_by(words, w => len(w))`. | +| `count_by(list, f)` | A map from each `f(element)` to how many elements gave it. | +| `chunk(list, size)` | The list cut into lists of `size` elements (the last may be shorter). | +| `min_by(list, f)`, `max_by(list, f)` | The element with the smallest / largest `f(element)`: `max_by(people, p => p.age)`. | +| `copy(x)` | A shallow copy of a list, map or struct. | ## Maps @@ -89,6 +106,10 @@ Lengths and positions count characters, not bytes. | `join(list, separator?)` | See Lists. | | `upper(s)`, `lower(s)` | Upper/lower case (ASCII letters). Also work on a char. | | `trim(s)` | Removes leading and trailing whitespace. | +| `trim_start(s)`, `trim_end(s)` | Removes whitespace from one end only. | +| `capitalize(s)` | Upper-cases the first letter: `"hello world"` → `"Hello world"`. | +| `title(s)` | Upper-cases the first letter of every word: `"hello world"` → `"Hello World"`. | +| `last_index_of(s, part)` | Position of the last occurrence of `part` in a string (or of a value in a list), or `-1`. | | `replace(s, old, new)` | Replaces every occurrence of `old`. | | `starts_with(s, prefix)`, `ends_with(s, suffix)` | Prefix/suffix tests. | | `contains(s, part)` | `true` if `part` occurs in `s` (same as `part in s`). | @@ -111,8 +132,13 @@ Lengths and positions count characters, not bytes. | `max(a, b, ...)`, `max(list)` | Largest value. | | `clamp(x, low, high)` | `x` limited to the range `low`..`high`. | | `sum(list)` | Sum of a list of numbers. Small integer types are widened, so the sum of `u8` values is an `int`. | +| `mean(list)`, `median(list)` | The average / middle value of a list of numbers, as a float. | | `sign(x)` | `-1`, `0` or `1`. | -| `gcd(a, b)` | Greatest common divisor. | +| `gcd(a, b)`, `lcm(a, b)` | Greatest common divisor and least common multiple. | +| `factorial(n)` | `n!` for `n` from 0 to 20. | +| `trunc(x)` | Drops the fraction, giving an int (toward zero: `trunc(-3.7)` is `-3`). | +| `degrees(x)`, `radians(x)` | Converts radians to degrees and back. | +| `union(a, b)`, `intersect(a, b)`, `difference(a, b)` | Set operations on lists: the results have no duplicates and keep the order of `a`. | | `sqrt(x)`, `cbrt(x)` | Square and cube root. | | `pow(a, b)` | Same as `a ** b`. | | `exp(x)`, `log(x, base?)`, `log2(x)`, `log10(x)` | Exponential and logarithms (`log` is natural without a base). | @@ -128,7 +154,7 @@ Lengths and positions count characters, not bytes. | `shuffle(list)` | The elements in random order. | | `seed(n)` | Seeds the random number generator for repeatable results. | -## Files and JSON +## Files, folders and JSON | Function | Description | |----------|-------------| @@ -138,25 +164,83 @@ Lengths and positions count characters, not bytes. | `append_file(path, text)` | Adds `text` to the end of the file. | | `file_exists(path)` | `true` if the file can be opened. | | `delete_file(path)` | Deletes the file; returns `true` if it was deleted. | +| `is_dir(path)` | `true` if `path` is a folder. | +| `make_dir(path)` | Creates a folder (and the folders above it if needed). Returns `true` if it was created, `false` if it already existed. | +| `list_dir(path?)` | The names of the files and folders in a folder (default: the current one), sorted. | +| `walk_dir(path?)` | Every file below a folder (default: the current one), as sorted paths relative to it. | +| `remove_dir(path)` | Deletes a folder and everything in it. Returns `false` if there was no such folder. | +| `copy_file(from, to)`, `move_file(from, to)` | Copies or moves (renames) a file, replacing `to` if it exists. | +| `file_size(path)` | The size of a file in bytes. | +| `path_join(parts...)` | Joins path parts with `/`: `path_join("data", "2024", "log.txt")`. | +| `basename(path)`, `dirname(path)`, `extension(path)` | The last part of a path, everything before it, and its extension (`".txt"`). | +| `abs_path(path)` | The full path. | +| `cwd()`, `chdir(path)` | The current folder, and changing it. | +| `temp_dir()` | The folder for temporary files. | +| `csv_parse(text, separator?)` | Parses CSV into a list of rows, each a list of strings. Handles quoted fields with commas, quotes (`""`) and newlines. | +| `csv_stringify(rows, separator?)` | The opposite: rows (lists of values) as CSV text, quoting fields where needed. | | `json_parse(text)` | Converts JSON to Ascend values (objects become maps, arrays become lists). | | `json_stringify(value, indent?)` | Converts a value to JSON; with `indent`, pretty-prints with that many spaces. | +## Regular expressions + +Patterns use JavaScript (ECMAScript) syntax. Raw strings keep backslashes as +written, so they're the easiest way to write patterns: `r"\d+"` (the same as +`"\\d+"`). + +| Function | Description | +|----------|-------------| +| `regex_match(text, pattern)` | `true` if the whole text matches the pattern. | +| `regex_search(text, pattern)` | The first match as a list: the matched text, then each group (`null` for groups that didn't match). `null` if there's no match. | +| `regex_find_all(text, pattern)` | A list of every matching piece of text. | +| `regex_replace(text, pattern, replacement)` | Replaces every match. In the replacement, `$1` is the first group, `$&` the whole match. | +| `regex_split(text, pattern)` | Splits the text wherever the pattern matches. | + ## Modules and program control | Function | Description | |----------|-------------| -| `import(path)` | Runs another Ascend file once and returns its top-level names as a map. Relative paths start from the importing file; `.asc` is optional. If there's no such file, imports the installed package with that name (see [packages.md](packages.md)). | +| `import(path)` | Runs another Ascend file once and returns its top-level names as a map. Relative paths start from the importing file; `.asc` is optional. If there's no such file, imports the installed package with that name (see [packages.md](packages.md)). The `import name` statement does the same and stores the map in a variable (see [syntax.md](syntax.md#modules)). | | `assert(condition, message?)` | Throws an error if `condition` is falsy. | | `exit(code?)` | Ends the program with exit code `code` (default 0). | | `args` | List of the extra command line arguments given after the script name. | +## Running programs and HTTP + +| Function | Description | +|----------|-------------| +| `run(command, input?)` | Runs a program and waits for it. `command` is a list (`["git", "status"]`, run directly) or a string (run by the system shell, so pipes work: `"ls | wc -l"`). `input` is sent to its standard input. Returns `{code, stdout, stderr, ok}`, where `ok` is `code == 0`. | +| `system(command)` | Runs a shell command with its output going straight to the terminal; returns the exit code. | +| `http_get(url, headers?)` | Downloads a URL. Returns `{status, ok, body}` (`ok` for statuses 200-299). `headers` is a map: `{"Authorization": "Bearer ..."}`. | +| `http_post(url, body, headers?)` | Sends `body`: a string as is, or a map or list as JSON (with a JSON content type). Returns the same as `http_get`. | +| `http_request(method, url, body?, headers?)` | Any other method: `http_request("DELETE", url)`. | + +The HTTP functions use the `curl` program (installed on macOS, Windows 10 +and later, and most Linux systems). A connection problem is an error; an +HTTP error status like 404 isn't, so check `ok` or `status`. + +``` +response := http_get("https://api.github.com/repos/csubbdev/Ascend") +if (response.ok) print(json_parse(response.body).description) +``` + +## Encodings and ids + +| Function | Description | +|----------|-------------| +| `base64_encode(text)`, `base64_decode(text)` | Base64. | +| `sha256(text)` | The SHA-256 hash, as 64 hex digits. | +| `uuid()` | A random UUID, like `"3f2a8c1e-5b7d-4e9a-8c1f-2d6b9e0a4c3b"`. | + ## Time and system | Function | Description | |----------|-------------| +| `now()` | The local date and time as a map: `year`, `month`, `day`, `hour`, `minute`, `second`, `weekday` (0 is Sunday), `yearday` and `timestamp`. | +| `format_time(seconds, format?)` | A timestamp (like `time()`) as local time, formatted with `strftime` codes (default `"%Y-%m-%d %H:%M:%S"`). | | `time()` | Seconds since 1970-01-01 as a float. | | `clock()` | Seconds from a steady clock; subtract two readings to time code. | | `sleep(seconds)` | Pauses the program. | | `date(format?)` | The local date and time, formatted with `strftime` codes (default `"%Y-%m-%d %H:%M:%S"`). | | `env(name)` | The value of an environment variable, or `null`. | -| `system(command)` | Runs a shell command and returns its exit code. | +| `set_env(name, value)` | Sets an environment variable (for this program and the programs it runs). | +| `platform()` | `"linux"`, `"macos"` or `"windows"`. | diff --git a/docs/syntax.md b/docs/syntax.md index 17e7735..289c2db 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -43,8 +43,25 @@ string? nickname // the ? allows null; without a value this starts as null i64 total // without a value it starts at zero ``` -`=` changes an existing variable (`count = 5`, `count += 1`). Assigning to a -name that was never declared is an error, which catches typos. +`=` changes an existing variable (`count = 5`, `count += 1`, `count++`). +Assigning to a name that was never declared is an error, which catches typos. + +Several variables of one type can be declared together, like in C: +`i32 low = 0, high = 10`. + +`auto` declares a variable whose type is taken from its first value, like +C++: after `auto total = 0`, `total` only holds ints. + +### Several at once + +``` +a, b := 1, 2 // declare several variables +a, b = b, a // swap: all the values are worked out first +low, high := minmax(xs) // unpack a list (a function can `return a, b`) +x, y := point // unpack a struct's fields, in order +``` + +The number of names must match the number of values. A typed variable only ever holds values of its type. Numbers convert automatically when nothing is lost; anything else is an error: @@ -68,17 +85,21 @@ starts running. | `i8` `i16` `i32` `i64` | signed integers of 8, 16, 32 and 64 bits | | `u8` `u16` `u32` `u64` | unsigned integers; `byte` is another name for `u8` | | `int` | the type of integer literals like `42`: a 64-bit signed integer | -| `f32` `f64` | 32- and 64-bit floating point numbers | +| `short` `long` | other names for `i16` and `i64`, as in C | +| `f32` `f64` | 32- and 64-bit floating point numbers; `double` is another name for `f64` | | `float` | the type of float literals like `1.5`: a 64-bit float | | `bool` | `true`, `false` | | `char` | one Unicode character: `'a'`, `'\n'`, `'é'` | | `string` | text: `"hello"` | | `list` | `[1, "two", 3.0]` | +| `list` | a list whose elements are all `T`: `list`, `list>` | | `map` | `{name: "Ada", "a key": 1}` | +| `map` | a map with keys of type `K` and values of type `V`: `map` | | `func` | a function | | `any` | anything (the same as writing no type) | | `void` | only as a return type: the function returns nothing | | `T?` | `T` or `null`, e.g. `string?` | +| a struct's name | a value made by that struct, e.g. `Point` (see [Structs](#structs)) | `type(value)` returns the type's name: `type(5)` is `"int"`, `type(u8(5))` is `"u8"`, `type('a')` is `"char"`. @@ -101,6 +122,10 @@ the fraction), `f32(1)`, `char(65)`, `string(12)`, `byte('A')`, `int("0xFF")`. A conversion fails if the value doesn't fit: `u8(300)` and `i8(-129)` are errors. +C-style casts do the same: `(f64) total / count` converts `total` before +dividing, `(i32) 3.9` is `3`, `(char) 65` is `'A'`. A cast binds tighter than +the arithmetic operators. + ### Truthiness `false`, `null`, `0`, `0.0`, `'\0'`, `""`, `[]` and `{}` are falsy. Everything @@ -140,20 +165,22 @@ From lowest to highest precedence: | Operators | Meaning | |---------------------------------|---------| | `a ? b : c` | conditional expression | +| `a ?? b` | `a`, or `b` if `a` is null | | `or`, `\|\|` | logical or (short-circuit) | | `and`, `&&` | logical and (short-circuit) | | `not`, `!` | logical not | | `==`, `!=` | equality (deep for lists and maps; `1 == 1.0`) | -| `<`, `<=`, `>`, `>=`, `in` | comparison; `x in list`, `key in map`, `"sub" in string` | +| `<`, `<=`, `>`, `>=`, `in` | comparison; `x in list`, `key in map`, `"sub" in string`, `x in 1..10` | +| `a..b`, `a..=b` | range: the numbers (or chars) from `a` up to `b`, without or with `b` | | `\|` | bitwise or | | `^` | bitwise xor | | `&` | bitwise and | | `<<`, `>>` | shifts | | `+`, `-` | add, subtract, concatenate | | `*`, `/`, `%` | multiply, divide, remainder | -| `-x`, `~x` | negation, bitwise not | +| `-x`, `~x`, `++x`, `--x`, `(type) x` | negation, bitwise not, increment, decrement, cast | | `**` | power (right associative, `-2 ** 2 == -4`) | -| `f(x)`, `a[i]`, `a[i:j]`, `m.k` | call, index, slice, field | +| `f(x)`, `a[i]`, `a[i:j]`, `m.k`, `m?.k`, `x++`, `x--` | call, index, slice, field, optional field, increment, decrement | Notes: @@ -163,11 +190,20 @@ Notes: `0xF0u8 << 1` is `224`. `>>` is arithmetic for signed types. - `~x` flips the bits within the type: `~0u8` is `255`, `~0` is `-1`. - Compound assignment works for every operator: `+= -= *= /= %= &= |= ^= <<= >>=`. +- `x++` and `x--` add or subtract 1 and give the old value; `++x` and `--x` + give the new one. They work on variables, elements and fields + (`items[i]++`, `p.count--`) holding numbers or chars, and overflow is an + error like any other arithmetic. - `and`, `or` and `not` always produce `true` or `false`. `not` applies to the whole comparison after it: `not x in items` means `not (x in items)`. - `+` with a string on either side converts the other side to a string: `"total: " + 5`. - `"ab" * 3` repeats a string; `[0] * 3` repeats a list; `[1] + [2]` joins lists. +- `a?.b` is `null` when `a` is null, instead of an error; `a?.f()` doesn't + call `f` then. Combine it with `??` for a default: `user?.name ?? "guest"`. +- `0..5` is the list `[0, 1, 2, 3, 4]` and `1..=3` is `[1, 2, 3]`; + `'a'..='e'` gives chars. In a `for` loop, `in` test, `switch` case or + `match` pattern no list is made, so `for i in 0..1_000_000_000` is fine. ## Strings and chars @@ -185,6 +221,19 @@ s[1:3] // "él" Escapes: `\n \t \r \0 \\ \" \' \{ \}`, `\xHH` (a character code in hex) and `\u{1F600}` (any Unicode character). +Raw strings keep backslashes as they are, which is handy for regular +expressions and Windows paths: `r"\d+-\d+"`, `r"C:\temp"`. + +Triple quotes make a string that can span several lines (a newline right +after the opening quotes is left out). Add `r` in front to make it raw too. + +``` +usage := """ +Usage: tool [options] + -h show "help" +""" +``` + Chars support a little arithmetic: `'a' + 1 == 'b'`, `'z' - 'a' == 25`, `'-' * 3 == "---"`. A char equals the one-character string holding it (`'a' == "a"`), and `char + string` concatenates. @@ -225,6 +274,23 @@ In a map literal, a bare name before `:` is a string key (`{name: 1}` is `{"name": 1}`). Put an expression in parentheses to use its value as the key: `{(key): 1}`. Keys may be strings, numbers, chars, bools or `null`. +`...` spreads one list into another, or one map into another (later keys +win): `[0, ...items, 99]`, `{...defaults, size: 2}`. + +### Comprehensions + +A comprehension builds a list or map with a loop in one expression: + +``` +squares := [x * x for x in 1..=10] +evens := [x for x in numbers if x % 2 == 0] +pairs := [[a, b] for a in 1..3 for b in "xy"] // nested loops +lengths := {word: len(word) for word in words} // a map +doubled := {k: v * 2 for k, v in prices if v > 0} +``` + +The loop variables only exist inside the comprehension. + Reading a missing map key or an out-of-range index is an error. Use `get(map, key, default)` when a key may be missing. @@ -249,6 +315,8 @@ while n > 0 { } for i in range(5) { } // 0, 1, 2, 3, 4 +for i in 0..5 { } // the same +for i in 1..=5 { } // 1, 2, 3, 4, 5 for i in range(2, 10, 2) { } // 2, 4, 6, 8 for item in items { } for i, item in items { } // index and item @@ -257,10 +325,77 @@ for key in person { } // map keys for key, value in person { } ``` +Loop variables can have a type, and `:` can be used instead of `in`, like in +C++ and Java: `for (string name : names) { }`. + `break` leaves the innermost loop and `continue` skips to its next iteration. Each loop iteration gets fresh loop variables, so closures created in a loop capture the value from their own iteration. +### C-style loops + +``` +for (i32 i = 0; i < 10; i++) { } // init; condition; step +for (i32 lo = 0, hi = n - 1; lo < hi; lo++, hi--) { } +for i := 0; i < 10; i++ { } // the parentheses are optional +for (;;) { } // forever, until break +for { } // the same + +string? line +do { + line = input("> ") +} while (line != null and line != "quit") // the body runs at least once +``` + +Variables declared in the first part belong to the loop. `continue` still +runs the step. + +### Bodies without braces + +Like C, the body of an `if`, `else`, `while`, `for` or `do` can be a single +statement instead of a `{ }` block, on the same line or the next one. A `;` +alone is an empty body. + +``` +if (x < 0) x = -x +else if (x == 0) print("zero") +else + print("positive") + +while (i < n) i++ +while (busy()); +``` + +When the condition is in parentheses, its `)` ends it, so the body can +follow directly. (Without braces or parentheses, the condition would run on +into the body.) + +### switch + +`switch` works like C: execution starts at the first `case` equal to the +value (or at `default`) and continues into the cases after it until a +`break`. + +``` +switch (command) { +case "start": +case "go": // either word runs the next lines + start() + break +case "stop", "halt": // several values in one case + stop() + break +default: + print("unknown command") +} +``` + +`break` inside a switch leaves the switch; `continue` goes to the loop around +it. For a version without fall-through, see `match` below. + +A case can also be a range: `case 90..=100:` matches any number from 90 to +100 (and so can a `match` pattern). + ### match `match` runs the first arm whose value equals the subject: @@ -321,7 +456,59 @@ func add(a, b) { `func(i32 x) -> i32 { return x * 2 }`. - To store a function in a typed variable, use the type `func`: `func handler = print`. +- Like C, if a program declares a top-level function `main` and never calls + it, `main()` runs after the rest of the file. It can take no parameters, + the list of command line arguments (`int main(list args)`), or a count and + the list (`int main(int argc, list argv)`). If it returns an integer, that's + the program's exit code. - Functions are closures: they keep access to the variables around them. +- `return a, b` returns the list `[a, b]`, which the caller can unpack: + `low, high := minmax(xs)`. + +### Lambdas + +Short anonymous functions, handy with `map`, `filter` and `sort`: + +``` +double := x => x * 2 +add := (a, b) => a + b +square := (i32 n) => n * n // parameters can have types +greet := () => { print("hi") } // a { block } body for statements + +map(prices, p => p * 1.2) +sort(people, p => p.age) +``` + +### Named arguments + +Arguments can be given by name after the positional ones, in any order. This +is most useful for parameters with defaults: + +``` +string label(string text, i32 width = 10, char fill = ' ') { ... } + +label("total", fill: '.') +label(width: 4, text: "id") +Box(height: 2) // struct fields too +``` + +### Any number of arguments + +A last parameter written `...name` collects the remaining arguments in a +list; with a type, each one must have that type. In a call, `...list` +passes a list's items as separate arguments. + +``` +i32 sum(i32 ...numbers) { + i32 total = 0 + for (n in numbers) total += n + return total +} + +sum() // 0 +sum(1, 2, 3) // 6 +sum(...[4, 5], 6) // 15 +``` ``` func makeCounter() { @@ -351,25 +538,189 @@ dog := {name: "Rex", speak: func() { return "Woof" }} dog.speak() ``` +## Structs + +A struct is a value with named, typed fields: + +``` +struct Point { + f64 x + f64 y = 0 // a default value +} + +struct Rect { + Point origin // fields can be structs + f64 width, height // several fields of one type + string label = "rect" +} + +p := Point(1, 2) // the fields in order +q := Point(5) // missing fields get their default (or zero) +Point origin // all defaults: Point(0, 0) +print(p) // Point{x: 1.0, y: 2.0} +p.x = 3 // fields keep their types: p.x = "far" is an error +type(p) // "Point" +``` + +- A struct's name is a type: `Point p = ...`, `f64 length(Point a, Point b)`, + `Point? find(...)`. A struct value only fits its own struct type. +- Fields can't be added or removed, and reading a missing field is an error. +- Two struct values are equal when they're the same struct with equal fields. +- Structs are references, like maps; `copy(p)` makes a (shallow) copy. +- Structs can be used before their declaration, like functions. +- Arguments can be given by name: `Rect(width: 3, height: 2)`. + +### Methods + +Functions written inside a struct are its methods. They get the value they're +called on as `this`: + +``` +struct Counter { + i32 count = 0 + string name = "counter" + + void add(i32 n = 1) { this.count += n } + bool done() { return this.count >= 10 } + func label() { return f"{this.name}: {this.count}" } + + // init() runs whenever a Counter is made, e.g. to check its fields. + void init() { assert(this.count >= 0, "count can't be negative") } +} + +c := Counter() +c.add(3) +print(c.label()) // counter: 3 +``` + +Methods can have a return type (`bool done()`), `func`, or nothing in front +of their name. A function outside the struct that takes it as its first +parameter can be called the same way: `f64 area(Rect r)` can be called as +`r.area()`. + +### Inheritance + +A struct can extend another one, C++ style. It gets the other struct's +fields (first) and methods, and can replace methods; `super.name()` calls the +version it replaced. + +``` +struct Animal { + string name + string sound() { return "..." } + string speak() { return f"{this.name} says {this.sound()}" } +} + +struct Dog : Animal { + string breed = "mutt" + string sound() { return "Woof" } + string speak() { return super.speak() + "!" } +} + +d := Dog("Rex", breed: "beagle") +d.speak() // "Rex says Woof!" +Animal a = d // a Dog is also an Animal +is_a(d, Animal) // true +type(d) // "Dog" +``` + +The struct being extended must be declared first (or imported). + +### Operators and special methods + +Methods with these names make structs work with the language's operators and +builtins: + +| Method | Used by | +|--------|---------| +| `operator+(other)`, `operator-`, `operator*`, `operator/`, `operator%`, `operator**` | `a + b` and the others (also `+=`, ...) | +| `operator==(other)` | `==`, `!=`, `in`, `index_of`, `unique`, ... | +| `operator<(other)` | `<`, `>`, `<=`, `>=`, `sort`, `min`, `max` | +| `operator[](index)` | `value[index]` | +| `to_string()` | `print`, `str`, f-strings, `+` with a string, and printing lists that contain it | +| `iter()` | `for x in value` and comprehensions: return a list (or anything else you can loop over) | +| `len()` | `len(value)` | +| `contains(item)` | `item in value` | + +``` +struct Vec { + f64 x + f64 y + Vec operator+(Vec o) { return Vec(this.x + o.x, this.y + o.y) } + Vec operator*(f64 k) { return Vec(this.x * k, this.y * k) } + bool operator==(Vec o) { return this.x == o.x and this.y == o.y } + string to_string() { return f"({this.x}, {this.y})" } +} + +print(Vec(1, 2) + Vec(3, 4) * 2) // (7.0, 10.0) +``` + +## Typed lists and maps + +`list` and `map` are lists and maps whose elements have a type: + +``` +list scores = [90, 85] +scores.push(77) // fine +scores.push("A+") // error: this is a list, so it can't hold a string +map prices = {tea: 2.5} +prices["cake"] = 4 // stored as 4.0 +list> grid = [[1, 2], [3, 4]] +f64 total(list values) { return sum(values) } +``` + +When a list or map is stored in a typed variable (or passed to a typed +parameter), its elements are checked and converted, and the list remembers +its element type: from then on `push`, `insert`, `extend` and element +assignments are checked too, through any variable that refers to it. A list +can't later be used as a different type (and `copy()` keeps the type); build +a new list instead: `list as_floats = [f64(s) for s in scores]`. + +## Enums + +``` +enum Color { Red, Green, Blue = 10, Purple } + +Color.Green // 1 +Color.Purple // 11 +``` + +An enum is a constant map from names to ints. Each member is one more than +the one before it, starting at 0, unless it's given a value. Its members +can't be changed. Members can be separated by commas or newlines. + ## Modules -`import(path)` runs another Ascend file once and returns its top-level names -as a map. Paths are relative to the importing file, and `.asc` can be left out. -Names starting with `_` stay private. +`import` runs another Ascend file once and stores its top-level names in a +map. Paths are relative to the importing file. Names starting with `_` stay +private. ``` // geometry.asc f64 circle_area(f64 r) { return PI * r ** 2 } // main.asc -geometry := import("geometry") +import geometry print(geometry.circle_area(2)) ``` +``` +import geometry as geo // choose the variable name +import lib.shapes // the file lib/shapes.asc, stored as `shapes` +import "../tools/strings.asc" as s // any path, in quotes +import geometry, lib.shapes // several at once +from geometry import circle_area // just some of the names +from geometry import area as a, PI_2 +from geometry import * // every exported name +``` + If there's no such file, `import` looks for an installed package with that -name, so after `apkg install colors` you can write `import("colors")`. See +name, so after `apkg install colors` you can write `import colors`. See [packages.md](packages.md). +`import(path)` also works as a function that returns the module's map: +`geometry := import("geometry")`. + ## Errors Errors stop the program with a message that points at the problem: @@ -380,8 +731,20 @@ script.asc:2:9: runtime error: division by zero | ^ ``` -They can be caught with `try`/`catch`. The error variable holds the error -message (or the value given to `throw`): +When an error happens inside function calls, the message is followed by the +calls it passed through, most recent first: + +``` +script.asc:2:17: runtime error: can't divide by zero + 2 | if (b == 0) throw "can't divide by zero" + | ^ + in divide(), called from script.asc:5:12 + in average(), called from script.asc:9:7 +``` + +Errors can be caught with `try`/`catch`. The error variable holds the error +message (or the value given to `throw`). A `finally` block runs however the +`try` ends: normally, with an error, or with `return`/`break`. ``` try { @@ -390,6 +753,13 @@ try { print("failed:", e) } +try { + file := open_things() + use(file) +} finally { + close_things() // runs even if use() fails; the error still goes on +} + throw "something went wrong" throw {code: 404, message: "not found"} // any value can be thrown ``` @@ -397,8 +767,8 @@ throw {code: 404, message: "not found"} // any value can be thrown `assert(condition, message)` throws if the condition is falsy, and `exit(code)` ends the program immediately (it is not caught by `try`). -Recursion that goes too deep (for example, a missing base case) is reported -as a "stack overflow" error instead of crashing. +Recursion can go 100,000 calls deep. Deeper than that (for example, with a +missing base case) is reported as an error instead of crashing. ## Command line @@ -407,6 +777,17 @@ ascend interactive REPL ascend file.asc [args...] run a program; extra arguments are in the list `args` ascend -e 'print(1 + 2)' run code from the command line ascend --check file.asc report errors without running the program +ascend test [folders...] run every test file (named *_test.asc or test_*.asc) +``` + +`ascend test` runs each test file in a fresh interpreter. A file passes if it +finishes without an error and with exit code 0, so `assert(...)` (or the +`testing` package) is all a test needs: + +``` +// math_test.asc +import geometry +assert(geometry.circle_area(1) == PI, "area of a unit circle") ``` Exit codes: `0` success, `65` syntax or type error found before running, `70` diff --git a/editors/vscode/package.json b/editors/vscode/package.json index e561924..66bdffb 100644 --- a/editors/vscode/package.json +++ b/editors/vscode/package.json @@ -2,7 +2,7 @@ "name": "ascend-language", "displayName": "Ascend", "description": "Syntax highlighting for the Ascend programming language", - "version": "0.3.0", + "version": "1.0.0", "publisher": "ascend", "license": "Apache-2.0", "engines": { diff --git a/editors/vscode/syntaxes/ascend.tmLanguage.json b/editors/vscode/syntaxes/ascend.tmLanguage.json index aa326bf..de85e0d 100644 --- a/editors/vscode/syntaxes/ascend.tmLanguage.json +++ b/editors/vscode/syntaxes/ascend.tmLanguage.json @@ -46,6 +46,22 @@ } ] }, + { + "name": "string.quoted.triple.raw.ascend", + "begin": "\\br\"\"\"", + "end": "\"\"\"" + }, + { + "name": "string.quoted.triple.ascend", + "begin": "\"\"\"", + "end": "\"\"\"", + "patterns": [{ "include": "#escapes" }] + }, + { + "name": "string.quoted.double.raw.ascend", + "begin": "\\br\"", + "end": "\"" + }, { "name": "string.quoted.double.ascend", "begin": "\"", @@ -78,7 +94,7 @@ }, "typed-function-declaration": { "comment": "C-style function: the return type, then the name, e.g. i32 add(i32 a, i32 b)", - "match": "\\b(i8|i16|i32|i64|u8|u16|u32|u64|int|byte|f32|f64|float|bool|char|string|list|map|any|void)(\\?)?\\s+([A-Za-z_][A-Za-z0-9_]*)\\s*(?=\\()", + "match": "\\b(i8|i16|i32|i64|u8|u16|u32|u64|int|byte|short|long|f32|f64|float|double|bool|char|string|list|map|any|void|auto)(\\?)?\\s+([A-Za-z_][A-Za-z0-9_]*)\\s*(?=\\()", "captures": { "1": { "name": "storage.type.ascend" }, "2": { "name": "storage.type.ascend" }, @@ -89,21 +105,38 @@ "patterns": [ { "name": "keyword.control.ascend", - "match": "\\b(if|else|while|for|in|break|continue|return|match|try|catch|throw)\\b" + "match": "\\b(if|else|while|for|in|break|continue|return|match|try|catch|finally|throw|switch|case|default|do)\\b" + }, + { "name": "keyword.control.import.ascend", "match": "^\\s*(import|from)\\b(?!\\s*[(:=])" }, + { "name": "keyword.control.import.ascend", "match": "\\b(as)\\b(?=\\s+[A-Za-z_])" }, + { + "match": "\\b(struct|enum)\\s+([A-Za-z_][A-Za-z0-9_]*)", + "captures": { + "1": { "name": "storage.type.ascend" }, + "2": { "name": "entity.name.type.ascend" } + } }, { "name": "storage.type.function.ascend", "match": "\\bfunc\\b" }, { "name": "storage.modifier.ascend", "match": "\\bconst\\b" }, + { + "match": "\\b(operator)\\s*(\\+|-|\\*\\*|\\*|/|%|==|<|\\[\\])", + "captures": { + "1": { "name": "keyword.other.operator.ascend" }, + "2": { "name": "entity.name.function.operator.ascend" } + } + }, { "name": "keyword.operator.logical.ascend", "match": "\\b(and|or|not)\\b" } ] }, "types": { "name": "storage.type.ascend", - "match": "\\b(i8|i16|i32|i64|u8|u16|u32|u64|int|byte|f32|f64|float|bool|char|string|list|map|any|void)\\b\\??" + "match": "\\b(i8|i16|i32|i64|u8|u16|u32|u64|int|byte|short|long|f32|f64|float|double|bool|char|string|list|map|any|void|auto)\\b\\??" }, "constants": { "patterns": [ { "name": "constant.language.ascend", "match": "\\b(true|false|null)\\b" }, - { "name": "support.constant.ascend", "match": "\\b(PI|E|INF|NAN|args)\\b" } + { "name": "support.constant.ascend", "match": "\\b(PI|E|INF|NAN|args)\\b" }, + { "name": "variable.language.this.ascend", "match": "\\b(this|super)\\b" } ] }, "function-call": { @@ -113,7 +146,10 @@ "operators": { "patterns": [ { "name": "keyword.operator.assignment.ascend", "match": ":=|<<=|>>=|[-+*/%&|^]=" }, + { "name": "keyword.operator.increment.ascend", "match": "\\+\\+|--" }, { "name": "keyword.operator.arrow.ascend", "match": "->|=>" }, + { "name": "keyword.operator.null.ascend", "match": "\\?\\?|\\?\\." }, + { "name": "keyword.operator.range.ascend", "match": "\\.\\.\\.|\\.\\.=?" }, { "name": "keyword.operator.bitwise.ascend", "match": "<<|>>" }, { "name": "keyword.operator.comparison.ascend", "match": "==|!=|<=|>=|<|>" }, { "name": "keyword.operator.logical.ascend", "match": "&&|\\|\\||!" }, diff --git a/examples/c_style.asc b/examples/c_style.asc new file mode 100644 index 0000000..899ee80 --- /dev/null +++ b/examples/c_style.asc @@ -0,0 +1,54 @@ +// Ascend can be written like C: for (;;) loops, ++ and --, switch, +// do/while, bodies without braces, casts, printf and a main() function. + +enum Op { Add, Sub, Mul, Div } + +f64 apply(i32 op, f64 a, f64 b) { + switch (op) { + case Op.Add: return a + b + case Op.Sub: return a - b + case Op.Mul: return a * b + case Op.Div: + if (b == 0) throw "division by zero" + return a / b + default: + throw "unknown operation" + } +} + +// Like C, main() runs after the rest of the file. Its return value is the +// program's exit code. +int main(list args) { + for (i32 i = 1; i <= 3; i++) { + printf("%d squared is %2d\n", i, i * i) + } + + // The Collatz sequence, with C-style bodies without braces. + i32 n = 27 + i32 steps = 0 + while (n != 1) { + if (n % 2 == 0) n /= 2 + else n = 3 * n + 1 + steps++ + } + printf("27 reaches 1 after %d steps\n", steps) + + // do/while runs its body at least once. + i32 tries = 0 + do { + tries++ + } while (tries < 3) + printf("tried %d times\n", tries) + + // Several variables in one for header. + for (i32 low = 0, high = 9; low < high; low += 3, high -= 3) printf("%d..%d ", low, high) + printf("\n") + + f64 average = (f64) 7 / 2 // a cast, so the division isn't an integer division + printf("%-8s|%8.2f|\n", "average", average) + + for (i32 op : [Op.Add, Op.Sub, Op.Mul, Op.Div]) { + printf("op %d: 6, 3 -> %g\n", op, apply(op, 6, 3)) + } + return 0 +} diff --git a/examples/functional.asc b/examples/functional.asc new file mode 100644 index 0000000..6aaff80 --- /dev/null +++ b/examples/functional.asc @@ -0,0 +1,53 @@ +// Lambdas, comprehensions, ranges, destructuring, spreading and named arguments. + +struct Sale { + string item + string region + f64 amount +} + +sales := [ + Sale("tea", "north", 12.5), + Sale("coffee", "south", 30), + Sale("tea", "south", 7.25), + Sale("cake", "north", 18), + Sale("coffee", "north", 22), +] + +// Lambdas: x => expression, or (a, b) => { statements } +big := filter(sales, s => s.amount >= 15) +print("big sales:", [s.item for s in big]) + +// Comprehensions build lists and maps from loops. +by_region := group_by(sales, s => s.region) +totals := {region: sum([s.amount for s in list]) for region, list in by_region} +print("totals:", totals) + +// Ranges: 1..5 stops before 5, 1..=5 includes it. +squares := [n * n for n in 1..=5] +print("squares:", squares, "evens below 10:", [n for n in 0..10 if n % 2 == 0]) + +// Functions can return several values, and callers can unpack them. +func spread_of(list values) { + return min(values), max(values) +} +low, high := spread_of([s.amount for s in sales]) +print(f"amounts go from {low} to {high}") + +// ...rest collects extra arguments; ...list spreads a list into them. +f64 average(f64 ...values) { + return sum(values) / len(values) +} +print("average:", average(...[s.amount for s in sales])) + +// Arguments can be given by name, in any order, after the positional ones. +string describe(Sale s, string currency = "$", i32 decimals = 2) { + return sprintf("%-7s %s%.*f", s.item, currency, decimals, s.amount) +} +for s in sales { + print(describe(s, decimals: 1)) +} + +// a ?? b gives b when a is null; a?.b gives null instead of an error. +string? best = max_by(sales, s => s.amount)?.item +print("best seller:", best ?? "nothing") diff --git a/examples/geometry.asc b/examples/geometry.asc index b9ba6e3..11a49c6 100644 --- a/examples/geometry.asc +++ b/examples/geometry.asc @@ -1,4 +1,4 @@ -// A module: other files load it with import("geometry"). +// A module: other files load it with `import geometry`. // Every top-level name is exported, except names starting with an underscore. const PI_2 = PI * 2 diff --git a/examples/geometry_test.asc b/examples/geometry_test.asc new file mode 100644 index 0000000..72eed74 --- /dev/null +++ b/examples/geometry_test.asc @@ -0,0 +1,8 @@ +// A test file: `ascend test` runs every file named *_test.asc or test_*.asc. +// A test passes when the file finishes without an error. +import geometry + +assert(geometry.circle_area(1) == PI, "the area of a unit circle is pi") +assert(round(geometry.circle_circumference(1), 4) == 6.2832, "circumference") +assert(geometry.distance({x: 0, y: 0}, {x: 3, y: 4}) == 5, "a 3-4-5 triangle") +print("geometry tests passed") diff --git a/examples/methods.asc b/examples/methods.asc new file mode 100644 index 0000000..003a255 --- /dev/null +++ b/examples/methods.asc @@ -0,0 +1,47 @@ +// Structs with methods. A method gets the value it's called on as `this`. + +struct Account { + string owner + f64 balance = 0 + list history = [] + + // init() runs whenever an Account is made. + void init() { + if (this.balance < 0) throw "an account can't start below zero" + } + + void deposit(f64 amount) { + this.balance += amount + this.history.push(f"+{amount:.2f}") + } + + bool withdraw(f64 amount) { + if (amount > this.balance) return false + this.balance -= amount + this.history.push(f"-{amount:.2f}") + return true + } + + string summary() { + return f"{this.owner}: {this.balance:.2f} ({len(this.history)} transactions)" + } +} + +account := Account("Ada", 100) +account.deposit(25.5) +if (!account.withdraw(500)) print("not enough money for that") +account.withdraw(40) +print(account.summary()) +print("history:", account.history) + +// Named arguments work for constructors too. +savings := Account(owner: "Grace", balance: 1000) +print(savings.summary()) + +try { + Account("Linus", -5) +} catch e { + print("error:", e) +} finally { + print("(finally always runs)") +} diff --git a/examples/modules.asc b/examples/modules.asc index 2f22ccc..48ae84c 100644 --- a/examples/modules.asc +++ b/examples/modules.asc @@ -1,8 +1,11 @@ -// import() runs another file once and returns its top-level names as a map. -// Paths are relative to this file; ".asc" can be left out. -geometry := import("geometry") +// `import name` runs another file once and stores its top-level names in a +// map. Paths are relative to this file; `lib.shapes` means lib/shapes.asc. +import geometry -print(f"area of a circle with radius 2: {geometry.circle_area(2):.3f}") +// `from ... import` takes just the names you need. +from geometry import circle_area, distance as dist + +print(f"area of a circle with radius 2: {circle_area(2):.3f}") print(f"circumference: {geometry.circle_circumference(2):.3f}") -print("distance:", geometry.distance({x: 0, y: 0}, {x: 3, y: 4})) +print("distance:", dist({x: 0, y: 0}, {x: 3, y: 4})) print("exports:", keys(geometry)) diff --git a/examples/scripting.asc b/examples/scripting.asc new file mode 100644 index 0000000..141e649 --- /dev/null +++ b/examples/scripting.asc @@ -0,0 +1,36 @@ +// Using Ascend for scripts: running programs, files and folders, CSV, +// hashing and dates. + +// Run a command through the shell and capture what it prints. +result := run("echo hello from the shell") +print("exit code", result.code, "printed:", trim(result.stdout)) + +// Work in a scratch folder inside the system's temporary folder. +folder := path_join(temp_dir(), "ascend-scripting-" + uuid()[0:8]) +make_dir(folder) + +try { + // Write a CSV file, then read it back and summarize it. + csv_file := path_join(folder, "prices.csv") + write_file(csv_file, csv_stringify([["item", "price"], ["tea", 2.5], ["cake", 4], ["coffee", 3.25]])) + + table := csv_parse(read_file(csv_file)) + map prices = {row[0]: f64(row[1]) for row in table[1:]} + printf("%d items, %.2f in total, the most expensive is %s\n", len(prices), sum(values(prices)), + max_by(keys(prices), item => prices[item])) + + // More files in a subfolder, then a look at everything. + make_dir(path_join(folder, "notes")) + write_file(path_join(folder, "notes", "todo.txt"), "buy milk\nwrite more Ascend\n") + for (string file : walk_dir(folder)) { + path := path_join(folder, file) + printf("%-16s %3d bytes sha256 %s...\n", file, file_size(path), sha256(read_file(path))[0:12]) + } +} finally { + remove_dir(folder) // clean up, even if something above failed +} + +print("the scratch folder is gone:", !is_dir(folder)) + +today := now() +printf("today is %04d-%02d-%02d, day %d of the year\n", today.year, today.month, today.day, today.yearday) diff --git a/examples/shapes.asc b/examples/shapes.asc new file mode 100644 index 0000000..5b523ca --- /dev/null +++ b/examples/shapes.asc @@ -0,0 +1,52 @@ +// Inheritance, operator overloading, special methods and typed lists. + +struct Point { + f64 x + f64 y + Point operator+(Point other) { return Point(this.x + other.x, this.y + other.y) } + bool operator==(Point other) { return this.x == other.x and this.y == other.y } + string to_string() { return f"({this.x}, {this.y})" } +} + +// The struct every shape extends. +struct Shape { + string name + Point origin = Point(0, 0) + f64 area() { return 0 } + string describe() { return f"{this.name} at {this.origin} with area {this.area():.2f}" } + // Shapes compare by area, so they can be sorted. + bool operator<(Shape other) { return this.area() < other.area() } + string to_string() { return this.name } +} + +struct Circle : Shape { + f64 radius = 1 + f64 area() { return PI * this.radius ** 2 } +} + +struct Rect : Shape { + f64 width = 1 + f64 height = 1 + f64 area() { return this.width * this.height } + string describe() { return super.describe() + (this.width == this.height ? " (a square)" : "") } +} + +list shapes = [ + Circle("wheel", radius: 2), + Rect("door", Point(4, 0), width: 1, height: 2), + Rect("tile", Point(1, 1) + Point(2, 2), width: 1.5, height: 1.5), +] + +for (Shape s : shapes) print(s.describe()) + +print("smallest to largest:", sort(shapes)) +print("largest:", max(shapes)) +print("total area:", round(sum([s.area() for s in shapes]), 2)) +print("circles:", [s for s in shapes if is_a(s, Circle)]) +print("same point:", Point(1, 2) == Point(1, 2)) + +try { + shapes.push("not a shape") +} catch e { + print("error:", e) +} diff --git a/examples/structs.asc b/examples/structs.asc new file mode 100644 index 0000000..46df536 --- /dev/null +++ b/examples/structs.asc @@ -0,0 +1,43 @@ +// Structs group named, typed fields. Enums give names to numbers. + +struct Point { + f64 x + f64 y +} + +struct Rect { + Point origin // fields can be structs too + f64 width = 1 + f64 height = 1 + string label = "rect" +} + +enum Shape { Square, Wide, Tall } + +f64 area(Rect r) { + return r.width * r.height +} + +i32 shape(Rect r) { + if (r.width == r.height) return Shape.Square + return r.width > r.height ? Shape.Wide : Shape.Tall +} + +// The constructor takes the fields in order; the rest get their defaults. +Rect door = Rect(Point(1, 2), 1, 2, "door") +print(door) +print("area:", door.area()) // the same as area(door) +print("tall?", door.shape() == Shape.Tall) + +door.width = 2 // fields keep their types: door.width = "big" is an error +print("square now?", door.shape() == Shape.Square) + +Rect unit // every field starts at its default +print(unit) +print(type(unit), type(unit.origin)) + +try { + door.depth = 3 +} catch e { + print("error:", e) +} diff --git a/packages/colors/README.md b/packages/colors/README.md index 0a7b5c2..b07495f 100644 --- a/packages/colors/README.md +++ b/packages/colors/README.md @@ -7,7 +7,7 @@ apkg install colors ``` ``` -colors := import("colors") +import colors print(colors.green("passed"), colors.red("failed")) print(colors.style("Warning:", "bold yellow on_black"), "disk almost full") diff --git a/packages/strings/README.md b/packages/strings/README.md index 0e08245..d105654 100644 --- a/packages/strings/README.md +++ b/packages/strings/README.md @@ -7,7 +7,7 @@ apkg install strings ``` ``` -strings := import("strings") +import strings print(strings.snake_case("parseHTTPResponse")) // parse_http_response print(strings.title("the quick brown fox")) // The Quick Brown Fox diff --git a/packages/testing/README.md b/packages/testing/README.md index f7a4c56..de3a421 100644 --- a/packages/testing/README.md +++ b/packages/testing/README.md @@ -7,8 +7,8 @@ apkg install testing ``` ``` -t := import("testing") -geometry := import("geometry") +import testing as t +import geometry t.test("area of a square", func() { t.equal(geometry.area(3, 3), 9) diff --git a/src/apkg_main.cpp b/src/apkg_main.cpp index 54c617b..8809974 100644 --- a/src/apkg_main.cpp +++ b/src/apkg_main.cpp @@ -36,7 +36,7 @@ void printUsage(std::ostream& out) { "\n" "Inside a project (a folder with an ascend.json) packages are installed into its\n" "ascend_packages folder and recorded in ascend.json. Anywhere else they're\n" - "installed for you, in ~/.ascend/packages. Programs use them with import(\"colors\").\n" + "installed for you, in ~/.ascend/packages. Programs use them with: import colors\n" "\n" "Environment:\n" " ASCEND_HOME where your packages and downloads are kept (default ~/.ascend)\n" diff --git a/src/big_stack.cpp b/src/big_stack.cpp new file mode 100644 index 0000000..062ce07 --- /dev/null +++ b/src/big_stack.cpp @@ -0,0 +1,78 @@ +#include "big_stack.hpp" + +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif + +namespace ascend { + +namespace { + +thread_local size_t threadStackSize = 0; + +struct ThreadTask { + const std::function* body; + size_t bytes; + int result = 0; +}; + +void runTask(ThreadTask& task) { + threadStackSize = task.bytes; + task.result = (*task.body)(); +} + +} // namespace + +size_t knownThreadStackSize() { return threadStackSize; } + +#if defined(_WIN32) + +namespace { +DWORD WINAPI threadMain(LPVOID parameter) { + runTask(*static_cast(parameter)); + return 0; +} +} // namespace + +int runWithLargeStack(const std::function& body, size_t bytes) { + ThreadTask task{&body, bytes}; + HANDLE thread = CreateThread(nullptr, bytes, threadMain, &task, STACK_SIZE_PARAM_IS_A_RESERVATION, nullptr); + if (thread == nullptr) return body(); + WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); + return task.result; +} + +#else + +namespace { +void* threadMain(void* parameter) { + runTask(*static_cast(parameter)); + return nullptr; +} +} // namespace + +int runWithLargeStack(const std::function& body, size_t bytes) { + ThreadTask task{&body, bytes}; + pthread_attr_t attributes; + if (pthread_attr_init(&attributes) != 0) return body(); + pthread_t thread; + bool started = pthread_attr_setstacksize(&attributes, bytes) == 0 && + pthread_create(&thread, &attributes, threadMain, &task) == 0; + pthread_attr_destroy(&attributes); + if (!started) return body(); + pthread_join(thread, nullptr); + return task.result; +} + +#endif + +} // namespace ascend diff --git a/src/include/ast.hpp b/src/include/ast.hpp index 5feba07..70931f2 100644 --- a/src/include/ast.hpp +++ b/src/include/ast.hpp @@ -22,7 +22,8 @@ using StmtPtr = std::unique_ptr; enum class ExprKind { Literal, Variable, Unary, Binary, Logical, Ternary, - Call, Index, Slice, Member, List, Map, Function, FString + Call, Index, Slice, Member, List, Map, Function, FString, IncDec, + Comprehension, Range, Spread }; struct Expr { @@ -60,7 +61,7 @@ struct BinaryExpr final : Expr { ExprPtr right; }; -// `and` / `or`, which only evaluate their right side when needed. +// `and` / `or` / `??`, which only evaluate their right side when needed. struct LogicalExpr final : Expr { LogicalExpr(int line, int column) : Expr(ExprKind::Logical, line, column) {} TokenType op = TokenType::And; @@ -75,10 +76,13 @@ struct TernaryExpr final : Expr { ExprPtr elseExpr; }; +// f(a, b, name: c, ...rest) struct CallExpr final : Expr { CallExpr(int line, int column) : Expr(ExprKind::Call, line, column) {} ExprPtr callee; std::vector args; + std::vector argNames; // one per argument: "" for positional ones + bool hasNamedArgs = false; }; struct IndexExpr final : Expr { @@ -95,11 +99,12 @@ struct SliceExpr final : Expr { ExprPtr end; }; -// object.name +// object.name, or object?.name (null when the object is null) struct MemberExpr final : Expr { MemberExpr(int line, int column) : Expr(ExprKind::Member, line, column) {} ExprPtr object; std::string name; + bool optional = false; }; struct ListExpr final : Expr { @@ -112,6 +117,44 @@ struct MapExpr final : Expr { std::vector> entries; }; +// x++, x--, ++x, --x (also on elements and fields: items[i]++, p.count--) +struct IncDecExpr final : Expr { + IncDecExpr(int line, int column) : Expr(ExprKind::IncDec, line, column) {} + ExprPtr target; // VariableExpr, IndexExpr or MemberExpr + bool increment = true; + bool prefix = false; // ++x gives the new value, x++ the old one +}; + +// [x * 2 for x in items if x > 0] {k: v for k, v in pairs} +struct ComprehensionClause { + bool isFor = true; // `for a, b in iterable`, or `if condition` + std::string first; + std::string second; // may be empty + ExprPtr expr; // the iterable or the condition +}; + +struct ComprehensionExpr final : Expr { + ComprehensionExpr(int line, int column) : Expr(ExprKind::Comprehension, line, column) {} + bool isMap = false; + ExprPtr key; // maps only + ExprPtr value; // the list element, or the map value + std::vector clauses; +}; + +// start..end (end not included) and start..=end (included) +struct RangeExpr final : Expr { + RangeExpr(int line, int column) : Expr(ExprKind::Range, line, column) {} + ExprPtr start; + ExprPtr end; + bool inclusive = false; +}; + +// ...items: spreads a list into a call's arguments or a list, or a map into a map +struct SpreadExpr final : Expr { + SpreadExpr(int line, int column) : Expr(ExprKind::Spread, line, column) {} + ExprPtr value; +}; + // f"text {expression:spec} more text" struct FStringPart { std::string text; // literal text (when expr is null) @@ -128,8 +171,9 @@ struct FStringExpr final : Expr { // Statements enum class StmtKind { - Expression, Declare, Assign, Block, If, While, For, - Break, Continue, Return, Function, Try, Throw, Match + Expression, Declare, Assign, Block, If, While, For, CFor, + Break, Continue, Return, Function, Try, Throw, Match, + Switch, Import, Struct, Enum, Group, MultiAssign }; struct Stmt { @@ -147,8 +191,8 @@ struct Stmt { // A sequence of statements with facts the interpreter uses to run it quickly. struct Block { std::vector statements; - bool declaresNames = false; // contains `:=` or `func` directly, so it needs its own scope - bool hasFunctions = false; // contains `func` declarations, which are hoisted + bool declaresNames = false; // declares variables directly, so it needs its own scope + bool hasFunctions = false; // contains function or struct declarations, which are hoisted }; struct Param { @@ -162,6 +206,7 @@ struct FunctionExpr final : Expr { std::string name; // empty for anonymous functions std::vector params; size_t requiredParams = 0; // parameters without a default value + bool variadic = false; // the last parameter (...rest) collects the remaining arguments in a list TypeSpec returnType; // `any` when the function has no return type Block body; }; @@ -172,11 +217,13 @@ struct ExpressionStmt final : Stmt { }; // name := value type name = value type name const [type] name = value +// auto name = value (the variable gets the type of its first value) struct DeclareStmt final : Stmt { DeclareStmt(int line, int column) : Stmt(StmtKind::Declare, line, column) {} std::string name; TypeSpec type; // `any` when no type was written bool hasType = false; + bool inferType = false; // `auto` bool constant = false; ExprPtr value; // null for `type name`, which starts at the type's zero value }; @@ -201,21 +248,39 @@ struct IfStmt final : Stmt { StmtPtr elseBranch; // a BlockStmt, another IfStmt (else if), or null }; +// while (condition) { } and do { } while (condition) struct WhileStmt final : Stmt { WhileStmt(int line, int column) : Stmt(StmtKind::While, line, column) {} ExprPtr condition; Block body; + bool doWhile = false; // the body runs once before the condition is checked }; // for first in iterable { } / for first, second in iterable { } +// The variables may have types: for (string name : names) { } struct ForStmt final : Stmt { ForStmt(int line, int column) : Stmt(StmtKind::For, line, column) {} std::string first; std::string second; // empty when only one loop variable is given + TypeSpec firstType; + TypeSpec secondType; ExprPtr iterable; Block body; }; +// for (init; condition; step) { } Each part may be empty, and init and step +// may list several statements separated by commas. +struct CForStmt final : Stmt { + CForStmt(int line, int column) : Stmt(StmtKind::CFor, line, column) {} + std::vector init; + ExprPtr condition; // null: loop until `break` + std::vector step; + Block body; + // The body creates functions, which may capture the loop variables, so + // each iteration gets its own copy of them (like `let` in JavaScript). + bool copyPerIteration = false; +}; + struct BreakStmt final : Stmt { BreakStmt(int line, int column) : Stmt(StmtKind::Break, line, column) {} }; @@ -237,8 +302,19 @@ struct FunctionStmt final : Stmt { struct TryStmt final : Stmt { TryStmt(int line, int column) : Stmt(StmtKind::Try, line, column) {} Block body; + bool hasCatch = false; std::string errorName; // may be empty: `catch { ... }` Block handler; + bool hasFinally = false; + Block finallyBody; // runs however the try block ends +}; + +// a, b := pair a, b = b, a x, y = point_list +struct MultiAssignStmt final : Stmt { + MultiAssignStmt(int line, int column) : Stmt(StmtKind::MultiAssign, line, column) {} + std::vector targets; // VariableExpr (for :=), or anything assignable + bool declare = false; + std::vector values; // one list to unpack, or one value per target }; struct ThrowStmt final : Stmt { @@ -261,6 +337,84 @@ struct MatchStmt final : Stmt { std::vector arms; }; +// switch (value) { +// case 1, 2: +// print("small") +// break +// default: +// print("other") +// } +// Like C, execution continues into the next case unless there's a `break`. +struct SwitchCase { + std::vector values; // empty for `default` + size_t start = 0; // index of its first statement in the body +}; + +struct SwitchStmt final : Stmt { + SwitchStmt(int line, int column) : Stmt(StmtKind::Switch, line, column) {} + ExprPtr subject; + std::vector cases; + Block body; // every case's statements, in order +}; + +// import geometry import lib.shapes as shapes +// import "path/to/file" as name from geometry import area, Point as P +struct ImportName { + std::string name; // the module path, or the exported name for `from` + std::string binding; // the variable it's stored in +}; + +struct ImportStmt final : Stmt { + ImportStmt(int line, int column) : Stmt(StmtKind::Import, line, column) {} + bool from = false; + std::string fromPath; // for `from path import ...` + bool all = false; // from path import * + std::vector names; // modules (import), or exported names (from) +}; + +// struct Point { +// f64 x +// f64 y = 0 +// } +struct StructField { + std::string name; + TypeSpec type; // `any` when no type was written + ExprPtr defaultValue; // may be null: then the type's zero value + int line = 0; + int column = 0; +}; + +struct StructStmt final : Stmt { + StructStmt(int line, int column) : Stmt(StmtKind::Struct, line, column) {} + std::string name; + std::string parent; // struct Dog : Animal { ... } + std::vector fields; + // name -> method. Methods get the value they're called on as their first + // parameter, `this`. + std::vector>> methods; +}; + +// enum Color { Red, Green = 5, Blue } makes the constant map Color. +struct EnumMember { + std::string name; + ExprPtr value; // may be null: one more than the previous member + int line = 0; + int column = 0; +}; + +struct EnumStmt final : Stmt { + EnumStmt(int line, int column) : Stmt(StmtKind::Enum, line, column) {} + std::string name; + std::vector members; +}; + +// Several statements written as one, run in the enclosing scope: +// `i32 a = 1, b = 2`, or `i = 0, j = n` in a for header. +struct GroupStmt final : Stmt { + GroupStmt(int line, int column) : Stmt(StmtKind::Group, line, column) {} + std::vector statements; +}; + struct Program { Block body; }; diff --git a/src/include/big_stack.hpp b/src/include/big_stack.hpp new file mode 100644 index 0000000..59422ed --- /dev/null +++ b/src/include/big_stack.hpp @@ -0,0 +1,21 @@ +#ifndef ASCEND_BIG_STACK_HPP +#define ASCEND_BIG_STACK_HPP + +#include +#include + +namespace ascend { + +// Runs `body` on a new thread with a large stack and returns its result, so +// Ascend programs can recurse deeply (each Ascend call uses a few KB of the +// C++ stack). The memory is reserved, not used, until the stack grows. If the +// thread can't be created, `body` runs on the current thread instead. +int runWithLargeStack(const std::function& body, size_t bytes = size_t(512) * 1024 * 1024); + +// The size of the current thread's stack when it was started by +// runWithLargeStack, or 0. +size_t knownThreadStackSize(); + +} // namespace ascend + +#endif // ASCEND_BIG_STACK_HPP diff --git a/src/include/interpreter.hpp b/src/include/interpreter.hpp index 4d14e6a..05c2745 100644 --- a/src/include/interpreter.hpp +++ b/src/include/interpreter.hpp @@ -32,6 +32,15 @@ struct ExitRequest { int code; }; +// One function call that an uncaught error passed through: the function, and +// where it was called from (line 0: by a builtin; line -1: by the runtime). +struct TraceFrame { + std::string function; // e.g. "divide()" or "Vec.length()" + const SourceFile* source; + int line; + int column; +}; + // A tree-walking interpreter for Ascend programs. class Interpreter { public: @@ -46,18 +55,36 @@ class Interpreter { void execute(std::unique_ptr program, std::unique_ptr source = nullptr); // Calls an Ascend or builtin function. `line`/`column` locate errors. - Value call(const Value& callee, std::vector args, int line = 0, int column = 0); + // `named` holds arguments given by name: f(1, size: 2). + Value call(const Value& callee, std::vector args, int line = 0, int column = 0, + const std::vector>* named = nullptr); // Runs another Ascend file (once) and returns its top-level names as a // map. Relative paths are relative to the file that is running. Value importModule(const std::string& path); + // The calls an uncaught error passed through, innermost first. Reset by + // each execute() and whenever an error is caught. + const std::vector& traceback() const { return trace_; } + + // Calls the program's main() function, C style: with no arguments, with + // the list `args`, or with (count, args), depending on its parameters. + // Returns its result as an exit code (0 unless it returns an integer). + int runMain(); + + // Stores `value` in a map element, checking the field types of structs + // and refusing changes to enums. + void setElement(MapObj& map, const Value& key, Value value); + // Operations shared with the standard library. Value binaryOp(TokenType op, const Value& left, const Value& right); Value indexGet(const Value& object, const Value& index); Value slice(const Value& object, const Value& start, const Value& end); bool contains(const Value& container, const Value& item); + // A struct's iter() result for loops (other values are returned as they are). + Value iterable(const Value& value); + // Allocation of objects the cycle collector should know about. std::shared_ptr newEnvironment(std::shared_ptr parent); std::shared_ptr newList(std::vector items = {}); @@ -87,6 +114,18 @@ class Interpreter { size_t trackedObjectCount() const { return heap_.size(); } private: + // Calls struct methods for printing and comparing (see ValueHooks). + class Hooks final : public ValueHooks { + public: + explicit Hooks(Interpreter& interpreter) : interpreter_(interpreter) {} + bool customString(const Value& value, std::string& out) override; + bool customEqual(const Value& a, const Value& b, bool& out) override; + bool customCompare(const Value& a, const Value& b, int& out) override; + + private: + Interpreter& interpreter_; + }; + enum class Flow { Normal, Break, Continue, Return }; using EnvPtr = std::shared_ptr; class FrameGuard; @@ -100,7 +139,15 @@ class Interpreter { Flow execDeclare(const DeclareStmt& stmt, const EnvPtr& env); Flow execAssign(const AssignStmt& stmt, const EnvPtr& env); Flow execFor(const ForStmt& stmt, const EnvPtr& env); + Flow execCFor(const CForStmt& stmt, const EnvPtr& env); + Flow execWhile(const WhileStmt& stmt, const EnvPtr& env); + Flow execSwitch(const SwitchStmt& stmt, const EnvPtr& env); + Flow execImport(const ImportStmt& stmt, const EnvPtr& env); + Flow execEnum(const EnumStmt& stmt, const EnvPtr& env); + Flow execMultiAssign(const MultiAssignStmt& stmt, const EnvPtr& env); + void assignTo(const Expr& target, Value value, const EnvPtr& env); Flow execTry(const TryStmt& stmt, const EnvPtr& env); + Flow execTryCatch(const TryStmt& stmt, const EnvPtr& env); Flow execMatch(const MatchStmt& stmt, const EnvPtr& env); Flow execReturn(const ReturnStmt& stmt, const EnvPtr& env); void hoistFunctions(const Block& block, const EnvPtr& env); @@ -112,10 +159,20 @@ class Interpreter { Value evalUnary(const UnaryExpr& expr, const EnvPtr& env); Value evalMember(const MemberExpr& expr, const EnvPtr& env); Value evalFString(const FStringExpr& expr, const EnvPtr& env); + Value evalIncDec(const IncDecExpr& expr, const EnvPtr& env); + Value evalComprehension(const ComprehensionExpr& expr, const EnvPtr& env); + Value evalRange(const RangeExpr& expr, const EnvPtr& env); + bool inRange(const Value& value, const RangeExpr& range, const EnvPtr& env); + bool matchesPattern(const Value& subject, const Expr& pattern, const EnvPtr& env); + Value makeStruct(const StructStmt& stmt, const EnvPtr& env); + Value constructStruct(const std::shared_ptr& type, std::vector args, int line, int column, + const std::vector>* named = nullptr); Value makeFunction(const FunctionExpr& expr, const EnvPtr& env, const std::string& name); void indexSet(const Value& object, const Value& index, Value value); - Value zeroValue(const TypeSpec& type); + Value zeroValue(const TypeSpec& type, const EnvPtr& env); + void defineName(const EnvPtr& env, const std::string& name, Value value, const TypeSpec& type = TypeSpec(), + bool constant = false); Value coerceVariable(const std::string& name, const TypeSpec& type, Value value); void noteErrorSite(int line, int column); void applyErrorSite(RuntimeError& error); @@ -144,9 +201,13 @@ class Interpreter { int errorColumn_ = 0; const SourceFile* errorSource_ = nullptr; bool replMode_ = false; + bool runningMain_ = false; int callDepth_ = 0; int maxCallDepth_; StackLimit stack_; + std::vector trace_; + Hooks hooks_{*this}; + ValueHooks* previousHooks_ = nullptr; std::vector> heap_; size_t nextCollection_; diff --git a/src/include/lexer.hpp b/src/include/lexer.hpp index ac97202..0f3a0d4 100644 --- a/src/include/lexer.hpp +++ b/src/include/lexer.hpp @@ -15,9 +15,10 @@ enum class TokenType { // Punctuation LeftParen, RightParen, LeftBracket, RightBracket, LeftBrace, RightBrace, Comma, Dot, Colon, Semicolon, Question, Arrow /* -> */, FatArrow /* => */, + QuestionQuestion /* ?? */, QuestionDot /* ?. */, DotDot /* .. */, DotDotEqual /* ..= */, Ellipsis /* ... */, // Operators - Plus, Minus, Star, StarStar, Slash, Percent, + Plus, Minus, Star, StarStar, Slash, Percent, PlusPlus, MinusMinus, Ampersand, Pipe, Caret, Tilde, ShiftLeft, ShiftRight, Equal, EqualEqual, NotEqual, Less, LessEqual, Greater, GreaterEqual, Declare, // := @@ -86,6 +87,8 @@ class Lexer { void identifier(); void number(char first); void string(char quote); + void longString(bool raw); + void rawString(); void character(); void fString(); bool escape(char32_t& out); diff --git a/src/include/parser.hpp b/src/include/parser.hpp index 4b30c03..4b71f1d 100644 --- a/src/include/parser.hpp +++ b/src/include/parser.hpp @@ -36,38 +36,62 @@ class Parser { // Statements StmtPtr statement(); - StmtPtr simpleStatement(); + StmtPtr simpleStatement(bool requireEnd = true, bool allowMulti = true); StmtPtr ifStatement(const Token& keyword); StmtPtr whileStatement(const Token& keyword); + StmtPtr doStatement(const Token& keyword); StmtPtr forStatement(const Token& keyword); + StmtPtr cForStatement(const Token& keyword, bool parenthesized); + std::vector forClauses(); + void loopVariable(std::string& name, TypeSpec& type); + StmtPtr switchStatement(const Token& keyword); + StmtPtr importStatement(const Token& keyword); + StmtPtr fromImportStatement(const Token& keyword); + std::string modulePath(std::string& lastPart); + StmtPtr structDeclaration(const Token& keyword); + size_t operatorName(size_t offset, std::string& name) const; + StmtPtr enumDeclaration(const Token& keyword); StmtPtr returnStatement(const Token& keyword); StmtPtr tryStatement(const Token& keyword); StmtPtr throwStatement(const Token& keyword); StmtPtr functionDeclaration(const Token& keyword); - StmtPtr typedDeclaration(); + StmtPtr typedDeclaration(bool requireEnd = true); StmtPtr typedFunctionDeclaration(); StmtPtr constDeclaration(const Token& keyword); StmtPtr matchStatement(const Token& keyword); std::unique_ptr functionRest(const Token& keyword, const std::string& name, const TypeSpec* returnType = nullptr); - TypeSpec parseType(bool allowVoid); - size_t typeLength(size_t offset) const; + void parameters(FunctionExpr& function); + ExprPtr lambda(const Token& start, bool parenthesized); + ExprPtr spreadOrExpression(); + ExprPtr comprehension(const Token& open, ExprPtr key, ExprPtr value, TokenType closing); + TypeSpec parseType(bool allowVoid, bool allowStruct = true); + size_t typeLength(size_t offset, bool allowStruct = false) const; + bool isDeclaration(size_t typeTokens) const; static std::string unknownTypeMessage(const std::string& name); Block block(const std::string& what); + Block body(const std::string& what); static void noteStatement(Block& block, const Stmt& stmt); void parseStatementInto(Block& block); ExprPtr condition(const std::string& keyword); + size_t matchingParen(size_t open) const; + bool hasTopLevelSemicolon(size_t from, size_t to) const; + bool isWord(size_t ahead, const char* word) const; + bool looksLikeFromImport() const; + bool looksLikeSwitch() const; void expectStatementEnd(); void synchronize(); // Expressions, from lowest to highest precedence ExprPtr expression(); ExprPtr ternary(); + ExprPtr nullish(); ExprPtr logicOr(); ExprPtr logicAnd(); ExprPtr logicNot(); ExprPtr equality(); ExprPtr comparison(); + ExprPtr range(); ExprPtr bitOr(); ExprPtr bitXor(); ExprPtr bitAnd(); @@ -105,6 +129,8 @@ class Parser { std::vector errors_; bool incomplete_ = false; int depth_ = 0; + int functionCount_ = 0; // functions parsed so far (to spot loop bodies that create closures) + bool noLambda_ = false; // in match patterns, `x => ...` is a pattern and its arm StackLimit stack_; }; diff --git a/src/include/process.hpp b/src/include/process.hpp index 7dbf2ef..44028f5 100644 --- a/src/include/process.hpp +++ b/src/include/process.hpp @@ -12,6 +12,20 @@ namespace ascend { // With `quiet`, the program's output is discarded. int runProcess(const std::vector& args, bool quiet = false); +// What a program printed, and how it ended. +struct CapturedOutput { + int code = -1; + std::string out; + std::string err; +}; + +// Runs a program (no shell) with `input` as its standard input, and captures +// what it prints. Returns false if it couldn't be started. +bool captureProcess(const std::vector& args, const std::string& input, CapturedOutput& result); + +// The same for a command run by the system shell (sh, or cmd on Windows). +bool captureShell(const std::string& command, const std::string& input, CapturedOutput& result); + // Reads an environment variable ("" if it isn't set). std::string environmentVariable(const std::string& name); diff --git a/src/include/runner.hpp b/src/include/runner.hpp index 5b53a47..8e809ab 100644 --- a/src/include/runner.hpp +++ b/src/include/runner.hpp @@ -5,10 +5,11 @@ #include #include +#include namespace ascend { -constexpr const char* kVersion = "0.4.0"; +constexpr const char* kVersion = "1.0.0"; // Process exit codes (following the BSD sysexits convention). constexpr int kExitSyntaxError = 65; @@ -41,6 +42,10 @@ RunResult runSource(Interpreter& interpreter, const std::string& source, const R // turns a first line starting with #! into a comment. void prepareSource(std::string& text); +// The "in f(), called from file:line:column" lines printed after an uncaught +// runtime error. +std::string formatTraceback(const std::vector& frames, const std::string& mainFile); + } // namespace ascend #endif // ASCEND_RUNNER_HPP diff --git a/src/include/semantic_analyzer.hpp b/src/include/semantic_analyzer.hpp index 837b588..b04583e 100644 --- a/src/include/semantic_analyzer.hpp +++ b/src/include/semantic_analyzer.hpp @@ -6,6 +6,7 @@ #include "typecheck.hpp" #include +#include #include #include @@ -13,21 +14,38 @@ namespace ascend { // Checks a parsed program for mistakes that are grammatically valid but can // never work, so they are reported before anything runs: -// - `break` / `continue` outside a loop +// - `break` outside a loop or switch, `continue` outside a loop // - `return` outside a function, or a return that doesn't match the // function's declared return type // - a function with two parameters of the same name // - a literal that doesn't fit its declared type: `u8 x = 300` +// - a type name that is neither builtin nor a struct the program declares +// - a struct with two fields of the same name, an enum with repeated members class SemanticAnalyzer { public: std::vector analyze(const Program& program); + // True if the program declares a top-level `main` function and never + // mentions it, so the runner should call it (like C's main). + bool shouldCallMain() const { return hasMain_ && !mentionsMain_; } + + // The REPL checks one line at a time, so a struct declared on an earlier + // line looks unknown; it turns this off. + bool checkTypeNames = true; + private: + struct TypeUse { + std::string name; + int line; + int column; + }; + void visitBlock(const Block& block); void visitStmt(const Stmt& stmt); void visitExpr(const Expr* expr); void visitFunction(const FunctionExpr& function); void report(int line, int column, std::string message); + void noteType(const TypeSpec& type, int line, int column); // Checks `expr` against `type` when its type is known without running it // (a literal, list, map, function or f-string). `describe` builds the @@ -35,7 +53,13 @@ class SemanticAnalyzer { void checkType(const Expr* expr, const TypeSpec& type, const std::function& describe); int loopDepth_ = 0; + int switchDepth_ = 0; std::vector functions_; + std::vector typeUses_; // struct type names used, checked at the end + std::set declared_; // names the program declares + bool importsEverything_ = false; // from x import *: any name may be declared + bool hasMain_ = false; + bool mentionsMain_ = false; std::vector diagnostics_; }; diff --git a/src/include/stdlib.hpp b/src/include/stdlib.hpp index a5839d5..37a6a2c 100644 --- a/src/include/stdlib.hpp +++ b/src/include/stdlib.hpp @@ -15,6 +15,9 @@ class Interpreter; // (PI, E, ...) in the interpreter's builtin scope. void registerStandardLibrary(Interpreter& interpreter); +// The builtins in stdlib_extra.cpp: run(), HTTP, paths, CSV, base64, ... +void registerExtraLibrary(Interpreter& interpreter); + // The integers produced by range(start, stop, step). struct RangeSpec { int64_t start = 0; diff --git a/src/include/typecheck.hpp b/src/include/typecheck.hpp index e41c54c..c6234c1 100644 --- a/src/include/typecheck.hpp +++ b/src/include/typecheck.hpp @@ -22,6 +22,12 @@ struct CoerceFailure { // Lists, maps and functions are not converted. Returns false on failure. bool coerceToType(const Value& value, const TypeSpec& type, Value& out, CoerceFailure& failure); +// Checks (and converts) a value being added to a list or stored in a +// map. Throws RuntimeError if it doesn't fit; does nothing for +// untyped lists and maps. +Value checkListItem(const ListObj& list, Value value); +void checkMapEntry(const MapObj& map, const Value& key, Value& value); + // "a string", "300", "null", "'a'"... for error messages. std::string describeValue(const Value& value); diff --git a/src/include/types.hpp b/src/include/types.hpp index da466a0..bb83ca2 100644 --- a/src/include/types.hpp +++ b/src/include/types.hpp @@ -2,6 +2,7 @@ #define ASCEND_TYPES_HPP #include +#include #include namespace ascend { @@ -24,22 +25,34 @@ uint64_t unsignedMax(NumKind kind); // for unsigned integer kinds const char* numKindName(NumKind kind); std::string rangeText(NumKind kind); // e.g. "0 to 255" -enum class TypeKind : uint8_t { Any, Void, Bool, Number, Char, String, List, Map, Function }; +// Struct: a type declared with `struct Name { ... }`, named by structName. +enum class TypeKind : uint8_t { Any, Void, Bool, Number, Char, String, List, Map, Function, Struct }; // A type annotation, such as the `u8` in `u8 x = 1` or `string?` (nullable). struct TypeSpec { TypeKind kind = TypeKind::Any; NumKind num = NumKind::Int; // used when kind == Number bool nullable = false; + std::string structName; // used when kind == Struct + // list: element is T. map: key is K and element is V. Null for + // plain `list` and `map`, which hold anything. + std::shared_ptr element; + std::shared_ptr key; bool isAny() const { return kind == TypeKind::Any; } std::string name() const; }; -// Looks up a type name such as "i32", "byte" or "string". Returns false for -// names that aren't types. +// Whether two types are the same, including their element types. +bool sameType(const TypeSpec& a, const TypeSpec& b); + +// Looks up a builtin type name such as "i32", "byte" or "string". Returns +// false for other names (which may be struct types). bool lookupType(const std::string& name, TypeSpec& out); +// A struct type with the given name. +TypeSpec structType(const std::string& name); + } // namespace ascend #endif // ASCEND_TYPES_HPP diff --git a/src/include/value.hpp b/src/include/value.hpp index 9c64d07..4966bcd 100644 --- a/src/include/value.hpp +++ b/src/include/value.hpp @@ -18,6 +18,8 @@ namespace ascend { class Environment; class Interpreter; struct FunctionExpr; +struct StructStmt; +struct StructType; struct ListObj; struct MapObj; struct Callable; @@ -90,6 +92,7 @@ class Value { Callable& asFunction() const { return *std::get<8>(data_); } const std::shared_ptr& listPtr() const { return std::get<6>(data_); } const std::shared_ptr& mapPtr() const { return std::get<7>(data_); } + const std::shared_ptr& functionPtr() const { return std::get<8>(data_); } bool truthy() const; const char* typeName() const; // "int", "u8", "f32", "char", "string", ... @@ -106,16 +109,53 @@ class Value { struct ListObj final : GcObject { std::vector items; + // Set once the list has been used as a list: every item is a T, and + // stays one (push, insert and assignments are checked). + std::shared_ptr elementType; void forEachChild(const std::function& visit) const override; void clearReferences() override { items.clear(); } }; +// The fields of a struct type, in declaration order (inherited ones first). +struct StructInfo { + std::string name; + std::vector> fields; + std::vector ancestors; // the struct it extends, that one's parent, ... + + // True for this struct and the structs it extends. + bool isA(const std::string& type) const { + if (type == name) return true; + for (const auto& ancestor : ancestors) { + if (ancestor == type) return true; + } + return false; + } + + // The declared type of a field, or nullptr if there's no such field. + const TypeSpec* fieldType(const std::string& field) const { + for (const auto& entry : fields) { + if (entry.first == field) return &entry.second; + } + return nullptr; + } +}; + // An insertion-ordered hash map. Keys may be null, bools, numbers, chars or // strings. Equal numbers are the same key (m[1] and m[1.0]), and so are a char // and the one-character string holding it. +// +// A struct value is a map with `structInfo` set: its keys are the struct's +// fields, and the interpreter checks the type of each value stored in them. struct MapObj final : GcObject { std::vector> entries; + std::shared_ptr structType; // the struct it's a value of; null for ordinary maps + std::string frozenAs; // set for enums, whose members can't change: "enum Color" + // Set once the map has been used as a map (like ListObj::elementType). + std::shared_ptr keyType; + std::shared_ptr valueType; + + const StructInfo* structInfo() const; // the struct's fields, or nullptr const Value* find(const Value& key) const; // nullptr if missing (or unhashable) Value* find(const Value& key); @@ -132,7 +172,7 @@ struct MapObj final : GcObject { }; struct Callable { - enum class Kind { Native, User }; + enum class Kind { Native, User, Struct }; explicit Callable(Kind kind) : kind(kind) {} virtual ~Callable() = default; @@ -174,6 +214,72 @@ struct FunctionObj final : Callable, GcObject { const SourceFile* source; // the file the function was written in }; +// A struct type, which is called to make values: Point(1, 2). It keeps the +// scope it was declared in, where the fields' default values are evaluated. +struct StructField; + +struct StructType final : Callable, GcObject { + // A field and the struct that declared it (whose scope its default is evaluated in). + struct FieldRef { + const StructField* field; + const StructType* owner; + }; + + StructType(std::shared_ptr info, const StructStmt* declaration, + std::shared_ptr closure, const SourceFile* source) + : Callable(Kind::Struct), + info(std::move(info)), + declaration(declaration), + closure(std::move(closure)), + source(source) {} + + GcObject* gcObject() override { return this; } + void forEachChild(const std::function& visit) const override; + void clearReferences() override { + closure.reset(); + methods.clear(); + parent.reset(); + } + + // A method by name, or nullptr. + const Value* method(const std::string& name) const { + for (const auto& entry : methods) { + if (entry.first == name) return &entry.second; + } + return nullptr; + } + + std::shared_ptr info; + const StructStmt* declaration; // owned by the interpreter's list of programs + std::shared_ptr closure; + const SourceFile* source; + std::vector> methods; // functions taking `this` first (inherited ones too) + std::shared_ptr parent; // the struct this one extends, if any + std::vector fields; // every field, inherited ones first +}; + +// Lets the interpreter supply struct methods (to_string, operator== and +// operator<) to code that has no interpreter: printing, comparing, sorting. +class ValueHooks { +public: + virtual ~ValueHooks() = default; + // Each returns false if the value has no such method. + virtual bool customString(const Value& value, std::string& out) = 0; + virtual bool customEqual(const Value& a, const Value& b, bool& out) = 0; + virtual bool customCompare(const Value& a, const Value& b, int& out) = 0; +}; + +// The hooks of the interpreter running on this thread (or null). +ValueHooks*& activeValueHooks(); + +// The struct value's method `name`, or nullptr (for any other value too). +const Value* structMethod(const Value& value, const std::string& name); + +inline const StructInfo* MapObj::structInfo() const { return structType ? structType->info.get() : nullptr; } + +// The type name type() reports: "int", "string", ... or a struct's name. +std::string typeNameOf(const Value& value); + // str(): how print() shows a value (strings and chars without quotes). std::string toString(const Value& value); // repr(): how the REPL and containers show a value (strings and chars quoted). diff --git a/src/interpreter.cpp b/src/interpreter.cpp index 870976c..69ab23f 100644 --- a/src/interpreter.cpp +++ b/src/interpreter.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -169,6 +170,60 @@ std::string arityMessage(const std::string& name, size_t min, long long max, siz std::string functionLabel(const std::string& name) { return name.empty() ? "this function" : name + "()"; } +// Puts arguments given by name into the slots of the parameters (or struct +// fields) they name. `slots` and `filled` already hold the positional ones. +void bindNamedArgs(const std::string& who, const char* noun, const std::vector& names, + const std::vector>& named, std::vector& slots, + std::vector& filled, int line, int column, const std::string& restName = "") { + for (const auto& entry : named) { + auto it = std::find(names.begin(), names.end(), entry.first); + if (it == names.end()) { + if (!restName.empty() && entry.first == restName) { + throw RuntimeError(who + ": '..." + restName + "' collects the extra arguments, so it can't be given by name", + line, column); + } + std::string list; + for (const auto& name : names) list += (list.empty() ? "" : ", ") + name; + throw RuntimeError(who + " has no " + noun + " named '" + entry.first + "'" + + (list.empty() ? "" : std::string(" (its ") + noun + "s are: " + list + ")"), + line, column); + } + size_t index = static_cast(it - names.begin()); + if (filled[index]) throw RuntimeError(who + " got two values for '" + entry.first + "'", line, column); + slots[index] = entry.second; + filled[index] = true; + } +} + +// Calls `visit(key, item)` for each element of a list (key: index), string +// (each char) or map (key: the key, item: the value). +template +void forEachElement(const Value& iterable, Visit visit) { + switch (iterable.type()) { + case Value::Type::List: { + const auto& items = iterable.asList().items; + for (size_t i = 0; i < items.size(); ++i) visit(Value::fromInt(static_cast(i)), items[i]); + return; + } + case Value::Type::String: { + const std::string text = iterable.asString(); + int64_t index = 0; + for (size_t pos = 0; pos < text.size(); ++index) { + char32_t c = decodeUtf8(text, pos); + visit(Value::fromInt(index), Value::fromChar(c)); + } + return; + } + case Value::Type::Map: { + const auto entries = iterable.asMap().entries; + for (const auto& entry : entries) visit(entry.first, entry.second); + return; + } + default: + throw RuntimeError("can't loop over " + aTypeName(iterable) + " value (expected a list, map, string or range)"); + } +} + bool isAbsolutePath(const std::string& path) { return (!path.empty() && (path[0] == '/' || path[0] == '\\')) || (path.size() > 1 && path[1] == ':'); } @@ -190,17 +245,28 @@ bool mayBePackage(const std::string& path) { // Switches the current function and source file for the duration of a call, // and restores them afterwards, even when an error unwinds the call. +// +// When an error unwinds through a call, the guard adds the call to the +// traceback (`label` names the function, `line`/`column` the call site). class Interpreter::FrameGuard { public: - FrameGuard(Interpreter& interpreter, const SourceFile* source, const FunctionExpr* function) + FrameGuard(Interpreter& interpreter, const SourceFile* source, const FunctionExpr* function, + const std::string* name = nullptr, int line = 0, int column = 0) : interpreter_(interpreter), savedSource_(interpreter.currentSource_), - savedFunction_(interpreter.currentFunction_) { + savedFunction_(interpreter.currentFunction_), + name_(name), + line_(line), + column_(column) { interpreter_.currentSource_ = source; interpreter_.currentFunction_ = function; ++interpreter_.callDepth_; } ~FrameGuard() { + if (name_ != nullptr && std::uncaught_exceptions() > 0) { + interpreter_.trace_.push_back( + {name_->empty() ? "an anonymous function" : *name_ + "()", savedSource_, line_, column_}); + } interpreter_.currentSource_ = savedSource_; interpreter_.currentFunction_ = savedFunction_; --interpreter_.callDepth_; @@ -214,8 +280,35 @@ class Interpreter::FrameGuard { Interpreter& interpreter_; const SourceFile* savedSource_; const FunctionExpr* savedFunction_; + const std::string* name_; // the function's name, for tracebacks (null: not a call) + int line_; + int column_; }; +namespace { + +// Adds a builtin to the traceback when an error from a function it called +// (like the callback given to map()) unwinds through it. +class NativeFrame { +public: + NativeFrame(std::vector& trace, const std::string& name, const SourceFile* source, int line, int column) + : trace_(trace), name_(name), source_(source), line_(line), column_(column) {} + ~NativeFrame() { + if (!trace_.empty() && std::uncaught_exceptions() > 0) trace_.push_back({name_ + "()", source_, line_, column_}); + } + NativeFrame(const NativeFrame&) = delete; + NativeFrame& operator=(const NativeFrame&) = delete; + +private: + std::vector& trace_; + const std::string& name_; + const SourceFile* source_; + int line_; + int column_; +}; + +} // namespace + // Notes where a runtime error happened, as the error unwinds through the // expressions and statements being evaluated: the innermost one is the // location reported. Doing this in a destructor instead of catching and @@ -251,6 +344,8 @@ Interpreter::Interpreter(std::ostream& out, std::istream& in) } catch (...) { random_.seed(static_cast(std::chrono::steady_clock::now().time_since_epoch().count())); } + previousHooks_ = activeValueHooks(); + activeValueHooks() = &hooks_; builtins_ = newEnvironment(nullptr); globals_ = newEnvironment(builtins_); registerStandardLibrary(*this); @@ -258,6 +353,7 @@ Interpreter::Interpreter(std::ostream& out, std::istream& in) } Interpreter::~Interpreter() { + activeValueHooks() = previousHooks_; // Break every remaining reference cycle so all memory is released. modules_.clear(); std::vector> live; @@ -401,6 +497,7 @@ void Interpreter::execute(std::unique_ptr program, std::unique_ptr program, std::unique_ptrkind == StmtKind::Expression) { + if (echo && stmt->kind == StmtKind::Expression && + stmt->as().expr->kind != ExprKind::IncDec) { Value result = eval(*stmt->as().expr, env); if (!result.isNull()) out_ << toRepr(result) << '\n'; continue; @@ -552,19 +650,37 @@ Interpreter::Flow Interpreter::execStmt(const Stmt& stmt, const EnvPtr& env) { return Flow::Normal; } - case StmtKind::While: { - const auto& loop = stmt.as(); - while (eval(*loop.condition, env).truthy()) { - Flow flow = execScoped(loop.body, env); - if (flow == Flow::Break) break; - if (flow == Flow::Return) return flow; - } - return Flow::Normal; - } + case StmtKind::While: + return execWhile(stmt.as(), env); case StmtKind::For: return execFor(stmt.as(), env); + case StmtKind::CFor: + return execCFor(stmt.as(), env); + + case StmtKind::Switch: + return execSwitch(stmt.as(), env); + + case StmtKind::Import: + return execImport(stmt.as(), env); + + case StmtKind::Struct: + return Flow::Normal; // defined when its block started, like functions + + case StmtKind::Enum: + return execEnum(stmt.as(), env); + + case StmtKind::MultiAssign: + return execMultiAssign(stmt.as(), env); + + case StmtKind::Group: + for (const auto& inner : stmt.as().statements) { + Flow flow = exec(*inner, env); + if (flow != Flow::Normal) return flow; + } + return Flow::Normal; + case StmtKind::Break: return Flow::Break; @@ -612,14 +728,20 @@ Interpreter::Flow Interpreter::execScoped(const Block& block, const EnvPtr& env) void Interpreter::hoistFunctions(const Block& block, const EnvPtr& env) { if (!block.hasFunctions) return; for (const auto& stmt : block.statements) { - if (stmt->kind != StmtKind::Function) continue; - const FunctionExpr& function = *stmt->as().function; - env->define(function.name, makeFunction(function, env, function.name)); + if (stmt->kind == StmtKind::Function) { + const FunctionExpr& function = *stmt->as().function; + env->define(function.name, makeFunction(function, env, function.name)); + } else if (stmt->kind == StmtKind::Struct) { + const auto& declaration = stmt->as(); + ErrorSite site(*this, stmt->line, stmt->column); + defineName(env, declaration.name, makeStruct(declaration, env)); + } } } -// The starting value of `type name` without an initializer. -Value Interpreter::zeroValue(const TypeSpec& type) { +// The starting value of `type name` without an initializer. A struct starts +// with each field at its default: `Point p` is `Point()`. +Value Interpreter::zeroValue(const TypeSpec& type, const EnvPtr& env) { if (type.nullable) return Value(); switch (type.kind) { case TypeKind::Bool: return Value::fromBool(false); @@ -627,12 +749,72 @@ Value Interpreter::zeroValue(const TypeSpec& type) { return isFloatKind(type.num) ? Value::fromFloat(0.0, type.num) : Value::fromInt(0, type.num); case TypeKind::Char: return Value::fromChar(0); case TypeKind::String: return Value::fromString(""); - case TypeKind::List: return Value::fromList(newList()); - case TypeKind::Map: return Value::fromMap(newMap()); + case TypeKind::List: { + auto list = newList(); + list->elementType = type.element; + return Value::fromList(list); + } + case TypeKind::Map: { + auto map = newMap(); + map->keyType = type.key; + map->valueType = type.element; + return Value::fromMap(map); + } + case TypeKind::Struct: { + Variable* variable = env->find(type.structName); + if (variable == nullptr || !variable->value.isFunction() || + variable->value.asFunction().kind != Callable::Kind::Struct) { + throw RuntimeError("'" + type.structName + "' isn't a struct type"); + } + Value constructor = variable->value; + return constructStruct(std::static_pointer_cast(constructor.functionPtr()), {}, 0, 0); + } default: return Value(); } } +// Defines a name, unless it's a constant already defined in this scope. +void Interpreter::defineName(const EnvPtr& env, const std::string& name, Value value, const TypeSpec& type, + bool constant) { + Variable* existing = env->findLocal(name); + if (existing != nullptr && existing->constant) { + throw RuntimeError("'" + name + "' is a constant and can't be declared again"); + } + env->define(name, std::move(value), type, constant); +} + +namespace { + +// The type `auto x = value` gives x. +TypeSpec typeOfValue(const Value& value) { + TypeSpec type; + switch (value.type()) { + case Value::Type::Null: break; + case Value::Type::Bool: type.kind = TypeKind::Bool; break; + case Value::Type::Int: + case Value::Type::Float: + type.kind = TypeKind::Number; + type.num = value.numKind(); + break; + case Value::Type::Char: type.kind = TypeKind::Char; break; + case Value::Type::String: type.kind = TypeKind::String; break; + case Value::Type::List: + type.kind = TypeKind::List; + type.element = value.asList().elementType; + break; + case Value::Type::Map: + if (value.asMap().structInfo()) return structType(value.asMap().structInfo()->name); + type.kind = TypeKind::Map; + type.key = value.asMap().keyType; + type.element = value.asMap().valueType; + break; + case Value::Type::Function: type.kind = TypeKind::Function; break; + } + return type; +} + +} // namespace + Value Interpreter::coerceVariable(const std::string& name, const TypeSpec& type, Value value) { if (type.isAny()) return value; Value converted; @@ -652,7 +834,7 @@ Interpreter::Flow Interpreter::execDeclare(const DeclareStmt& decl, const EnvPtr Value value; if (!decl.value) { - value = zeroValue(decl.type); + value = zeroValue(decl.type, env); } else if (decl.value->kind == ExprKind::Function) { // `name := func(...) {...}` gives the function a name for messages. const auto& function = decl.value->as(); @@ -661,7 +843,7 @@ Interpreter::Flow Interpreter::execDeclare(const DeclareStmt& decl, const EnvPtr value = eval(*decl.value, env); } if (decl.hasType) value = coerceVariable(decl.name, decl.type, std::move(value)); - env->define(decl.name, std::move(value), decl.type, decl.constant); + env->define(decl.name, std::move(value), decl.inferType ? typeOfValue(value) : decl.type, decl.constant); return Flow::Normal; } @@ -730,7 +912,7 @@ Interpreter::Flow Interpreter::execAssign(const AssignStmt& stmt, const EnvPtr& } else { result = eval(*stmt.value, env); } - object.asMap().set(key, std::move(result)); + setElement(object.asMap(), key, std::move(result)); return Flow::Normal; } @@ -768,14 +950,43 @@ Interpreter::Flow Interpreter::execFor(const ForStmt& loop, const EnvPtr& env) { auto iteration = [&](Value key, Value item) -> Flow { auto scope = newEnvironment(env); if (loop.second.empty()) { - scope->define(loop.first, std::move(item)); + scope->define(loop.first, coerceVariable(loop.first, loop.firstType, std::move(item)), loop.firstType); } else { - scope->define(loop.first, std::move(key)); - scope->define(loop.second, std::move(item)); + scope->define(loop.first, coerceVariable(loop.first, loop.firstType, std::move(key)), loop.firstType); + scope->define(loop.second, coerceVariable(loop.second, loop.secondType, std::move(item)), loop.secondType); } return execBlock(loop.body, scope); }; + // `for i in 0..n` counts without building a list. + if (loop.iterable->kind == ExprKind::Range) { + const auto& range = loop.iterable->as(); + Value startValue = eval(*range.start, env); + Value endValue = eval(*range.end, env); + int64_t start = 0; + int64_t end = 0; + bool chars = startValue.isChar() && endValue.isChar(); + bool valid = startValue.isChar() == endValue.isChar() && + (startValue.isChar() ? (start = static_cast(startValue.asChar()), true) + : startValue.isInt() && toInt64(startValue, start)) && + (endValue.isChar() ? (end = static_cast(endValue.asChar()), true) + : endValue.isInt() && toInt64(endValue, end)); + if (!valid) { + throw RuntimeError("a range needs two integers or two chars, not " + aTypeName(startValue) + " and " + + aTypeName(endValue), + range.line, range.column); + } + int64_t index = 0; + for (int64_t i = start; range.inclusive ? i <= end : i < end; ++i) { + Flow flow = iteration(Value::fromInt(index++), + chars ? Value::fromChar(static_cast(i)) : Value::fromInt(i)); + if (flow == Flow::Break) break; + if (flow == Flow::Return) return flow; + if (i == std::numeric_limits::max()) break; + } + return Flow::Normal; + } + // Fast path: `for i in range(...)` counts without building a list. if (loop.iterable->kind == ExprKind::Call && rangeFunction_ != nullptr) { const auto& call = loop.iterable->as(); @@ -799,7 +1010,7 @@ Interpreter::Flow Interpreter::execFor(const ForStmt& loop, const EnvPtr& env) { } } - Value iterable = eval(*loop.iterable, env); + Value iterable = this->iterable(eval(*loop.iterable, env)); switch (iterable.type()) { case Value::Type::List: { // Index-based so the body may safely add or remove elements. @@ -839,14 +1050,254 @@ Interpreter::Flow Interpreter::execFor(const ForStmt& loop, const EnvPtr& env) { } } +// a, b := pair a, b = b, a x, y := point (a struct's fields, in order) +Interpreter::Flow Interpreter::execMultiAssign(const MultiAssignStmt& stmt, const EnvPtr& env) { + std::vector values; + if (stmt.values.size() == 1) { + const Expr& source = *stmt.values[0]; + Value packed = eval(source, env); + if (packed.isList()) { + values = packed.asList().items; + } else if (packed.isMap() && packed.asMap().structInfo()) { + for (const auto& field : packed.asMap().structInfo()->fields) { + values.push_back(*packed.asMap().find(Value::fromString(field.first))); + } + } else { + throw RuntimeError("can't unpack " + aTypeName(packed) + " value into " + std::to_string(stmt.targets.size()) + + " variables (it needs a list or a struct)", + source.line, source.column); + } + if (values.size() != stmt.targets.size()) { + throw RuntimeError("can't unpack " + std::to_string(values.size()) + (values.size() == 1 ? " value" : " values") + " into " + + std::to_string(stmt.targets.size()) + " variables", + source.line, source.column); + } + } else { + for (const auto& value : stmt.values) values.push_back(eval(*value, env)); + } + for (size_t i = 0; i < stmt.targets.size(); ++i) { + const Expr& target = *stmt.targets[i]; + ErrorSite site(*this, target.line, target.column); + if (stmt.declare) { + defineName(env, target.as().name, std::move(values[i])); + } else { + assignTo(target, std::move(values[i]), env); + } + } + return Flow::Normal; +} + +// target = value, for a variable, element or field. +void Interpreter::assignTo(const Expr& target, Value value, const EnvPtr& env) { + switch (target.kind) { + case ExprKind::Variable: { + const std::string& name = target.as().name; + Variable* variable = env->find(name); + if (variable == nullptr) { + std::string hint = suggestName(name, *env); + if (hint.empty()) hint = " (to create it, write '" + name + " := ...')"; + throw RuntimeError("undefined variable '" + name + "'" + hint, target.line, target.column); + } + if (variable->constant) throw RuntimeError("can't change '" + name + "': it's a constant", target.line, target.column); + variable->value = coerceVariable(name, variable->type, std::move(value)); + return; + } + case ExprKind::Index: { + const auto& index = target.as(); + Value object = eval(*index.object, env); + Value key = eval(*index.index, env); + indexSet(object, key, std::move(value)); + return; + } + case ExprKind::Member: { + const auto& member = target.as(); + Value object = eval(*member.object, env); + if (!object.isMap()) { + throw RuntimeError("can't set field '" + member.name + "' on " + aTypeName(object) + + " value (only maps have fields)", + member.line, member.column); + } + setElement(object.asMap(), Value::fromString(member.name), std::move(value)); + return; + } + default: + throw RuntimeError("can't assign to this expression"); + } +} + +Interpreter::Flow Interpreter::execWhile(const WhileStmt& loop, const EnvPtr& env) { + if (loop.doWhile) { + while (true) { + Flow flow = execScoped(loop.body, env); + if (flow == Flow::Break) break; + if (flow == Flow::Return) return flow; + if (!eval(*loop.condition, env).truthy()) break; + } + return Flow::Normal; + } + while (eval(*loop.condition, env).truthy()) { + Flow flow = execScoped(loop.body, env); + if (flow == Flow::Break) break; + if (flow == Flow::Return) return flow; + } + return Flow::Normal; +} + +// for (init; condition; step) body +Interpreter::Flow Interpreter::execCFor(const CForStmt& loop, const EnvPtr& env) { + EnvPtr scope = loop.init.empty() ? env : newEnvironment(env); + for (const auto& init : loop.init) exec(*init, scope); + while (true) { + if (loop.condition && !eval(*loop.condition, scope).truthy()) break; + Flow flow = execScoped(loop.body, scope); + if (flow == Flow::Break) break; + if (flow == Flow::Return) return flow; + if (loop.copyPerIteration && scope != env) { + // Functions made in this iteration keep the variables as they are now. + auto next = newEnvironment(env); + for (const auto& entry : scope->locals()) { + next->define(entry.first, entry.second.value, entry.second.type, entry.second.constant); + } + scope = next; + } + for (const auto& step : loop.step) exec(*step, scope); + } + return Flow::Normal; +} + +// Runs from the first case equal to the value (or `default`) to the end of +// the switch, or until `break`. +Interpreter::Flow Interpreter::execSwitch(const SwitchStmt& stmt, const EnvPtr& env) { + Value subject = eval(*stmt.subject, env); + const SwitchCase* chosen = nullptr; + const SwitchCase* fallback = nullptr; + for (const auto& entry : stmt.cases) { + if (entry.values.empty()) { + fallback = &entry; + continue; + } + for (const auto& value : entry.values) { + if (matchesPattern(subject, *value, env)) { + chosen = &entry; + break; + } + } + if (chosen != nullptr) break; + } + if (chosen == nullptr) chosen = fallback; + if (chosen == nullptr) return Flow::Normal; + + EnvPtr scope = stmt.body.declaresNames ? newEnvironment(env) : env; + hoistFunctions(stmt.body, scope); + for (size_t i = chosen->start; i < stmt.body.statements.size(); ++i) { + Flow flow = exec(*stmt.body.statements[i], scope); + if (flow == Flow::Break) return Flow::Normal; + if (flow != Flow::Normal) return flow; + } + return Flow::Normal; +} + +Interpreter::Flow Interpreter::execImport(const ImportStmt& stmt, const EnvPtr& env) { + if (!stmt.from) { + for (const auto& item : stmt.names) defineName(env, item.binding, importModule(item.name)); + return Flow::Normal; + } + Value module = importModule(stmt.fromPath); + const MapObj& exports = module.asMap(); + if (stmt.all) { + for (const auto& entry : exports.entries) defineName(env, entry.first.asString(), entry.second); + return Flow::Normal; + } + for (const auto& item : stmt.names) { + const Value* value = exports.find(Value::fromString(item.name)); + if (value == nullptr) { + std::string names; + for (const auto& entry : exports.entries) names += (names.empty() ? "" : ", ") + entry.first.asString(); + throw RuntimeError("\"" + stmt.fromPath + "\" has no '" + item.name + "'" + + (names.empty() ? " (it exports nothing)" : " (it has: " + names + ")")); + } + defineName(env, item.binding, *value); + } + return Flow::Normal; +} + +// enum Color { Red, Green = 5, Blue } defines the constant map +// Color = {Red: 0, Green: 5, Blue: 6}. +Interpreter::Flow Interpreter::execEnum(const EnumStmt& stmt, const EnvPtr& env) { + auto members = newMap(); + int64_t next = 0; + for (const auto& member : stmt.members) { + Value value; + if (member.value) { + value = eval(*member.value, env); + int64_t number = 0; + if (value.isInt() && toInt64(value, number) && number < std::numeric_limits::max()) next = number + 1; + } else { + if (next == std::numeric_limits::max()) { + throw RuntimeError("enum " + stmt.name + ": the value of '" + member.name + "' is too large", + member.line, member.column); + } + value = Value::fromInt(next++); + } + members->set(Value::fromString(member.name), std::move(value)); + } + members->frozenAs = "enum " + stmt.name; + defineName(env, stmt.name, Value::fromMap(members), TypeSpec(), true); + return Flow::Normal; +} + Interpreter::Flow Interpreter::execTry(const TryStmt& stmt, const EnvPtr& env) { + if (!stmt.hasFinally) return execTryCatch(stmt, env); + + // `finally` runs however the rest ends. It must not disturb what's under + // way: a value being returned, or an error on its way out. + auto runFinally = [&]() -> Flow { + Value savedReturn = returnValue_; + bool savedPending = runtimeErrorPending(); + bool savedSite = hasErrorSite_; + int savedLine = errorLine_; + int savedColumn = errorColumn_; + const SourceFile* savedSource = errorSource_; + runtimeErrorPending() = false; + Flow flow = execScoped(stmt.finallyBody, env); + if (flow == Flow::Normal) { + returnValue_ = std::move(savedReturn); + runtimeErrorPending() = savedPending; + hasErrorSite_ = savedSite; + errorLine_ = savedLine; + errorColumn_ = savedColumn; + errorSource_ = savedSource; + } + return flow; + }; + + Flow flow; + try { + flow = execTryCatch(stmt, env); + } catch (...) { + Flow override = runFinally(); + if (override != Flow::Normal) { // a return or break in finally replaces the error + clearErrorSite(); + trace_.clear(); + return override; + } + throw; + } + Flow override = runFinally(); + return override != Flow::Normal ? override : flow; +} + +Interpreter::Flow Interpreter::execTryCatch(const TryStmt& stmt, const EnvPtr& env) { + if (!stmt.hasCatch) return execScoped(stmt.body, env); Value caught; try { return execScoped(stmt.body, env); } catch (const RuntimeError& error) { clearErrorSite(); // handled, so where it happened no longer matters + trace_.clear(); caught = Value::fromString(error.what()); } catch (const ThrownValue& thrown) { + trace_.clear(); caught = thrown.value; } auto scope = newEnvironment(env); @@ -864,7 +1315,7 @@ Interpreter::Flow Interpreter::execMatch(const MatchStmt& stmt, const EnvPtr& en continue; } for (const auto& pattern : arm.patterns) { - if (valuesEqual(subject, eval(*pattern, env))) return execScoped(arm.body, env); + if (matchesPattern(subject, *pattern, env)) return execScoped(arm.body, env); } } return fallback != nullptr ? execScoped(fallback->body, env) : Flow::Normal; @@ -893,12 +1344,19 @@ Value Interpreter::evalExpr(const Expr& expr, const EnvPtr& env) { case ExprKind::Binary: { const auto& binary = expr.as(); Value left = eval(*binary.left, env); + if (binary.op == TokenType::In && binary.right->kind == ExprKind::Range) { + return Value::fromBool(inRange(left, binary.right->as(), env)); // no list needed + } Value right = eval(*binary.right, env); return binaryOp(binary.op, left, right); } case ExprKind::Logical: { const auto& logical = expr.as(); + if (logical.op == TokenType::QuestionQuestion) { + Value value = eval(*logical.left, env); + return value.isNull() ? eval(*logical.right, env) : value; + } bool left = eval(*logical.left, env).truthy(); if (logical.op == TokenType::And && !left) return Value::fromBool(false); if (logical.op == TokenType::Or && left) return Value::fromBool(true); @@ -936,7 +1394,19 @@ Value Interpreter::evalExpr(const Expr& expr, const EnvPtr& env) { const auto& literal = expr.as(); auto list = newList(); list->items.reserve(literal.elements.size()); - for (const auto& element : literal.elements) list->items.push_back(eval(*element, env)); + for (const auto& element : literal.elements) { + if (element->kind == ExprKind::Spread) { + Value items = eval(*element->as().value, env); + if (!items.isList()) { + throw RuntimeError("'...' in a list needs a list to spread, not " + aTypeName(items), element->line, + element->column); + } + std::vector copy = items.asList().items; // the list may be spread into itself + list->items.insert(list->items.end(), copy.begin(), copy.end()); + continue; + } + list->items.push_back(eval(*element, env)); + } return Value::fromList(list); } @@ -944,6 +1414,16 @@ Value Interpreter::evalExpr(const Expr& expr, const EnvPtr& env) { const auto& literal = expr.as(); auto map = newMap(); for (const auto& entry : literal.entries) { + if (!entry.second) { // {...other} + Value other = eval(*entry.first->as().value, env); + if (!other.isMap()) { + throw RuntimeError("'...' in a map needs a map to spread, not " + aTypeName(other), entry.first->line, + entry.first->column); + } + const auto entries = other.asMap().entries; + for (const auto& item : entries) map->set(item.first, item.second); + continue; + } Value key = eval(*entry.first, env); Value value = eval(*entry.second, env); ErrorSite site(*this, entry.first->line, entry.first->column); // e.g. a list used as a key @@ -959,10 +1439,178 @@ Value Interpreter::evalExpr(const Expr& expr, const EnvPtr& env) { case ExprKind::FString: return evalFString(expr.as(), env); + + case ExprKind::IncDec: + return evalIncDec(expr.as(), env); + + case ExprKind::Comprehension: + return evalComprehension(expr.as(), env); + + case ExprKind::Range: + return evalRange(expr.as(), env); + + case ExprKind::Spread: + throw RuntimeError("'...' can only be used in a call's arguments, a list or a map"); } return Value(); } +// [value for x in items if test]: runs the clauses like nested loops, each +// iteration in its own scope. +Value Interpreter::evalComprehension(const ComprehensionExpr& expr, const EnvPtr& env) { + auto list = expr.isMap ? nullptr : newList(); + auto map = expr.isMap ? newMap() : nullptr; + std::function run = [&](size_t index, const EnvPtr& scope) { + if (index == expr.clauses.size()) { + if (expr.isMap) { + Value key = eval(*expr.key, scope); + Value value = eval(*expr.value, scope); + ErrorSite site(*this, expr.key->line, expr.key->column); + map->set(key, std::move(value)); + } else { + list->items.push_back(eval(*expr.value, scope)); + } + return; + } + const ComprehensionClause& clause = expr.clauses[index]; + if (!clause.isFor) { + if (eval(*clause.expr, scope).truthy()) run(index + 1, scope); + return; + } + Value iterable = this->iterable(eval(*clause.expr, scope)); + ErrorSite site(*this, clause.expr->line, clause.expr->column); + forEachElement(iterable, [&](Value key, Value item) { + auto inner = newEnvironment(scope); + if (clause.second.empty()) { + inner->define(clause.first, iterable.isMap() ? std::move(key) : std::move(item)); + } else { + inner->define(clause.first, std::move(key)); + inner->define(clause.second, std::move(item)); + } + run(index + 1, inner); + }); + }; + run(0, env); + return expr.isMap ? Value::fromMap(map) : Value::fromList(list); +} + +namespace { + +// The ends of a range as integers (chars count as their codes). +bool rangeBound(const Value& value, int64_t& out) { + if (value.isChar()) { + out = static_cast(value.asChar()); + return true; + } + return value.isInt() && toInt64(value, out); +} + +} // namespace + +// 0..5 is [0, 1, 2, 3, 4]; 1..=3 is [1, 2, 3]; 'a'..='e' is the chars a to e. +Value Interpreter::evalRange(const RangeExpr& expr, const EnvPtr& env) { + Value startValue = eval(*expr.start, env); + Value endValue = eval(*expr.end, env); + int64_t start = 0; + int64_t end = 0; + bool chars = startValue.isChar() && endValue.isChar(); + if ((startValue.isChar() != endValue.isChar()) || !rangeBound(startValue, start) || !rangeBound(endValue, end)) { + throw RuntimeError("a range needs two integers or two chars, not " + aTypeName(startValue) + " and " + + aTypeName(endValue)); + } + if (expr.inclusive) { + if (end == std::numeric_limits::max()) throw RuntimeError("the range's end is too large"); + ++end; + } + if (end > start && static_cast(end) - static_cast(start) > 10000000) { + throw RuntimeError("the range is too large to make a list from (for loops over ranges have no limit)"); + } + auto list = newList(); + for (int64_t i = start; i < end; ++i) { + list->items.push_back(chars ? Value::fromChar(static_cast(i)) : Value::fromInt(i)); + } + return Value::fromList(list); +} + +bool Interpreter::inRange(const Value& value, const RangeExpr& range, const EnvPtr& env) { + Value startValue = eval(*range.start, env); + Value endValue = eval(*range.end, env); + int64_t start = 0; + int64_t end = 0; + if (!rangeBound(startValue, start) || !rangeBound(endValue, end)) { + throw RuntimeError("a range needs two integers or two chars, not " + aTypeName(startValue) + " and " + + aTypeName(endValue)); + } + if (value.isChar() || value.isInt()) { + int64_t x = 0; + if (!rangeBound(value, x)) return false; // a u64 above the int64 range + return x >= start && (range.inclusive ? x <= end : x < end); + } + if (value.isFloat()) { + double x = value.asFloat(); + return x >= static_cast(start) && + (range.inclusive ? x <= static_cast(end) : x < static_cast(end)); + } + return false; +} + +// A case or match pattern: a range matches values inside it, anything else +// matches equal values. +bool Interpreter::matchesPattern(const Value& subject, const Expr& pattern, const EnvPtr& env) { + if (pattern.kind == ExprKind::Range) return inRange(subject, pattern.as(), env); + return valuesEqual(subject, eval(pattern, env)); +} + +// x++ and friends: like x += 1, but an expression whose value is the old +// value (x++) or the new one (++x). +Value Interpreter::evalIncDec(const IncDecExpr& expr, const EnvPtr& env) { + const char* symbol = expr.increment ? "++" : "--"; + auto step = [&](const Value& current) { + if (!current.isNumber() && !current.isChar()) { + throw RuntimeError(std::string("can't use '") + symbol + "' on " + aTypeName(current) + + " value (it needs a number or a char)"); + } + return binaryOp(expr.increment ? TokenType::Plus : TokenType::Minus, current, Value::fromInt(1)); + }; + + const Expr& target = *expr.target; + switch (target.kind) { + case ExprKind::Variable: { + const std::string& name = target.as().name; + Variable* variable = env->find(name); + if (variable == nullptr) { + throw RuntimeError("undefined variable '" + name + "'" + suggestName(name, *env), target.line, + target.column); + } + if (variable->constant) throw RuntimeError("can't change '" + name + "': it's a constant"); + Value old = variable->value; + variable->value = coerceVariable(name, variable->type, step(old)); + return expr.prefix ? variable->value : old; + } + case ExprKind::Index: { + const auto& index = target.as(); + Value object = eval(*index.object, env); + Value key = eval(*index.index, env); + Value old = indexGet(object, key); + indexSet(object, key, step(old)); + return expr.prefix ? indexGet(object, key) : old; + } + case ExprKind::Member: { + const auto& member = target.as(); + Value object = eval(*member.object, env); + if (!object.isMap()) { + throw RuntimeError("can't change field '" + member.name + "' of " + aTypeName(object) + " value"); + } + Value key = Value::fromString(member.name); + Value old = indexGet(object, key); + setElement(object.asMap(), key, step(old)); + return expr.prefix ? indexGet(object, key) : old; + } + default: + throw RuntimeError(std::string("'") + symbol + "' needs a variable, element or field"); + } +} + Value Interpreter::evalVariable(const VariableExpr& expr, const EnvPtr& env) { Variable* variable = env->find(expr.name); if (variable == nullptr) { @@ -988,6 +1636,7 @@ Value Interpreter::evalUnary(const UnaryExpr& expr, const EnvPtr& env) { Value Interpreter::evalMember(const MemberExpr& expr, const EnvPtr& env) { Value object = eval(*expr.object, env); + if (expr.optional && object.isNull()) return Value(); // a?.b if (!object.isMap()) { std::string hint; Variable* function = env->find(expr.name); @@ -998,7 +1647,21 @@ Value Interpreter::evalMember(const MemberExpr& expr, const EnvPtr& env) { expr.line, expr.column); } const Value* field = object.asMap().find(Value::fromString(expr.name)); - if (field == nullptr) throw RuntimeError("map has no key \"" + expr.name + "\"", expr.line, expr.column); + if (field == nullptr) { + const MapObj& map = object.asMap(); + if (map.structInfo()) { + if (map.structType->method(expr.name) != nullptr) { + throw RuntimeError("'" + expr.name + "' is a method of " + map.structInfo()->name + "; call it: ." + + expr.name + "()", + expr.line, expr.column); + } + throw RuntimeError(map.structInfo()->name + " has no field '" + expr.name + "'", expr.line, expr.column); + } + if (!map.frozenAs.empty()) { + throw RuntimeError(map.frozenAs + " has no member '" + expr.name + "'", expr.line, expr.column); + } + throw RuntimeError("map has no key \"" + expr.name + "\"", expr.line, expr.column); + } return *field; } @@ -1024,19 +1687,52 @@ Value Interpreter::evalCall(const CallExpr& expr, const EnvPtr& env) { std::vector args; args.reserve(expr.args.size() + 1); - if (expr.callee->kind == ExprKind::Member) { + const bool superCall = expr.callee->kind == ExprKind::Member && + expr.callee->as().object->kind == ExprKind::Variable && + expr.callee->as().object->as().name == "super"; + if (superCall) { + // super.name(...): the method of the struct this one extends, on `this`. + const auto& member = expr.callee->as(); + Variable* parent = env->find("super"); + Variable* self = env->find("this"); + if (parent == nullptr || self == nullptr || !parent->value.isFunction() || + parent->value.asFunction().kind != Callable::Kind::Struct) { + throw RuntimeError("'super' can only be used in the methods of a struct that extends another", member.line, + member.column); + } + const auto& parentType = static_cast(parent->value.asFunction()); + const Value* method = parentType.method(member.name); + if (method == nullptr) { + throw RuntimeError(parentType.info->name + " has no method '" + member.name + "'", member.line, member.column); + } + callee = *method; + args.push_back(self->value); + } else if (expr.callee->kind == ExprKind::Member) { const auto& member = expr.callee->as(); Value object = eval(*member.object, env); + if (member.optional && object.isNull()) return Value(); // a?.f() does nothing when a is null const Value* field = object.isMap() ? object.asMap().find(Value::fromString(member.name)) : nullptr; + const Value* method = nullptr; + if (field == nullptr && object.isMap() && object.asMap().structType) { + method = object.asMap().structType->method(member.name); + } if (field != nullptr) { callee = *field; + } else if (method != nullptr) { + callee = *method; + args.push_back(std::move(object)); } else { Variable* function = env->find(member.name); if (function == nullptr || !function->value.isFunction()) { - std::string message = - object.isMap() - ? "map has no key \"" + member.name + "\" and there is no function named '" + member.name + "'" - : aTypeName(object) + " value has no method '" + member.name + "'"; + std::string message; + if (object.isMap() && object.asMap().structInfo()) { + message = object.asMap().structInfo()->name + " has no method or field '" + member.name + + "' and there is no function named '" + member.name + "'"; + } else if (object.isMap()) { + message = "map has no key \"" + member.name + "\" and there is no function named '" + member.name + "'"; + } else { + message = aTypeName(object) + " value has no method '" + member.name + "'"; + } throw RuntimeError(message + suggestName(member.name, *env), member.line, member.column); } callee = function->value; @@ -1046,22 +1742,269 @@ Value Interpreter::evalCall(const CallExpr& expr, const EnvPtr& env) { callee = eval(*expr.callee, env); } - for (const auto& arg : expr.args) args.push_back(eval(*arg, env)); - return call(callee, std::move(args), expr.line, expr.column); + std::vector> named; + for (size_t i = 0; i < expr.args.size(); ++i) { + const Expr& arg = *expr.args[i]; + if (arg.kind == ExprKind::Spread) { + Value items = eval(*arg.as().value, env); + if (!items.isList()) { + throw RuntimeError("'...' needs a list to spread into the arguments, not " + aTypeName(items), arg.line, + arg.column); + } + const auto& values = items.asList().items; + args.insert(args.end(), values.begin(), values.end()); + continue; + } + Value value = eval(arg, env); + if (expr.hasNamedArgs && !expr.argNames[i].empty()) { + named.emplace_back(expr.argNames[i], std::move(value)); + } else { + args.push_back(std::move(value)); + } + } + return call(callee, std::move(args), expr.line, expr.column, named.empty() ? nullptr : &named); } Value Interpreter::makeFunction(const FunctionExpr& expr, const EnvPtr& env, const std::string& name) { return Value::fromFunction(track(std::make_shared(name, &expr, env, currentSource_))); } -Value Interpreter::call(const Value& callee, std::vector args, int line, int column) { +Value Interpreter::makeStruct(const StructStmt& stmt, const EnvPtr& env) { + auto info = std::make_shared(); + info->name = stmt.name; + std::shared_ptr parent; + if (!stmt.parent.empty()) { + Variable* found = env->find(stmt.parent); + if (found == nullptr || !found->value.isFunction() || found->value.asFunction().kind != Callable::Kind::Struct) { + throw RuntimeError("struct " + stmt.name + " extends '" + stmt.parent + + "', which isn't a struct (declare it before " + stmt.name + ")"); + } + parent = std::static_pointer_cast(found->value.functionPtr()); + info->fields = parent->info->fields; + info->ancestors.push_back(parent->info->name); + info->ancestors.insert(info->ancestors.end(), parent->info->ancestors.begin(), parent->info->ancestors.end()); + } + for (const auto& field : stmt.fields) { + if (parent != nullptr && parent->info->fieldType(field.name) != nullptr) { + throw RuntimeError("struct " + stmt.name + " can't declare '" + field.name + "' again: it already has it from " + + stmt.parent, + field.line, field.column); + } + info->fields.emplace_back(field.name, field.type); + } + auto type = track(std::make_shared(info, &stmt, env, currentSource_)); + if (parent != nullptr) { + type->parent = parent; + type->fields = parent->fields; + type->methods = parent->methods; // inherited; the struct's own methods replace them + } + for (const auto& field : stmt.fields) type->fields.push_back({&field, type.get()}); + // A struct's own methods can call the ones it replaced with super.name(). + EnvPtr methodScope = env; + if (parent != nullptr) { + methodScope = newEnvironment(env); + methodScope->define("super", Value::fromFunction(parent)); + } + for (const auto& method : stmt.methods) { + Value function = makeFunction(*method.second, methodScope, method.second->name); + bool replaced = false; + for (auto& existing : type->methods) { + if (existing.first == method.first) { + existing.second = function; + replaced = true; + } + } + if (!replaced) type->methods.emplace_back(method.first, function); + } + return Value::fromFunction(type); +} + +bool Interpreter::Hooks::customString(const Value& value, std::string& out) { + const Value* method = structMethod(value, "to_string"); + if (method == nullptr) return false; + Value result = interpreter_.call(*method, {value}); + if (!result.isString()) { + throw RuntimeError(value.asMap().structInfo()->name + ".to_string() must return a string, not " + + aTypeName(result)); + } + out = result.asString(); + return true; +} + +bool Interpreter::Hooks::customEqual(const Value& a, const Value& b, bool& out) { + const Value* method = structMethod(a, "operator=="); + if (method == nullptr) return false; + out = interpreter_.call(*method, {a, b}).truthy(); + return true; +} + +bool Interpreter::Hooks::customCompare(const Value& a, const Value& b, int& out) { + const Value* method = structMethod(a, "operator<"); + if (method == nullptr) return false; + if (interpreter_.call(*method, {a, b}).truthy()) { + out = -1; + } else { + const Value* reverse = structMethod(b, "operator<"); + out = reverse != nullptr && interpreter_.call(*reverse, {b, a}).truthy() ? 1 : 0; + } + return true; +} + +Value Interpreter::iterable(const Value& value) { + const Value* iter = structMethod(value, "iter"); + return iter != nullptr ? call(*iter, {value}) : value; +} + +// Point(1, 2): the arguments fill the fields in order; fields left out get +// their default value, or their type's zero value. +Value Interpreter::constructStruct(const std::shared_ptr& typePtr, std::vector args, int line, + int column, const std::vector>* named) { + const StructType& type = *typePtr; + const StructStmt& declaration = *type.declaration; + const std::string& name = declaration.name; + if (args.size() > type.fields.size()) { + std::string fields; + for (const auto& field : type.fields) fields += (fields.empty() ? "" : ", ") + field.field->name; + throw RuntimeError(name + "() takes at most " + std::to_string(type.fields.size()) + " argument" + + (type.fields.size() == 1 ? "" : "s") + (fields.empty() ? "" : " (" + fields + ")") + + ", but got " + std::to_string(args.size()), + line, column); + } + if (callDepth_ >= maxCallDepth_) { + throw RuntimeError("maximum recursion depth (" + std::to_string(maxCallDepth_) + ") exceeded while making a " + + name, + line, column); + } + checkStack(); + + // Point(1, y: 2): line the arguments up with the fields (inherited ones first). + const size_t count = type.fields.size(); + std::vector slots(count); + std::vector filled(count, false); + for (size_t i = 0; i < args.size(); ++i) { + slots[i] = std::move(args[i]); + filled[i] = true; + } + if (named != nullptr && !named->empty()) { + std::vector names; + for (const auto& field : type.fields) names.push_back(field.field->name); + bindNamedArgs(name + "()", "field", names, *named, slots, filled, line, column); + } + + auto object = newMap(); + object->structType = typePtr; + EnvPtr scope; // where default values are evaluated, made when first needed + FrameGuard frame(*this, type.source, nullptr, &name, line, column); + for (size_t i = 0; i < count; ++i) { + const StructField& field = *type.fields[i].field; + const StructType& owner = *type.fields[i].owner; + Value value; + if (filled[i]) { + value = std::move(slots[i]); + } else if (field.defaultValue) { + if (!scope || &owner != &type) scope = newEnvironment(owner.closure); + value = eval(*field.defaultValue, scope); + } else { + ErrorSite site(*this, field.line, field.column); + value = zeroValue(field.type, owner.closure); + } + Value converted; + CoerceFailure failure; + if (!coerceToType(value, field.type, converted, failure)) { + std::string message = withHint(name + "(): field '" + field.name + "' is " + field.type.name() + + ", so it can't hold " + failure.what, + failure); + if (filled[i]) { + RuntimeError error(message, line, column); // a bad argument: report it at the call + error.source = frame.callerSource(); + throw error; + } + throw RuntimeError(message, field.line, field.column); + } + object->set(Value::fromString(field.name), std::move(converted)); + } + Value result = Value::fromMap(object); + // An init() method can check or adjust the new value. + if (const Value* init = type.method("init")) call(*init, {result}, line, column); + return result; +} + +void Interpreter::setElement(MapObj& map, const Value& key, Value value) { + if (!map.frozenAs.empty()) throw RuntimeError("the members of " + map.frozenAs + " can't be changed"); + if (map.valueType) checkMapEntry(map, key, value); + if (map.structInfo()) { + const StructInfo& info = *map.structInfo(); + const TypeSpec* type = key.isString() ? info.fieldType(key.asString()) : nullptr; + if (type == nullptr) { + std::string fields; + for (const auto& field : info.fields) fields += (fields.empty() ? "" : ", ") + field.first; + throw RuntimeError(info.name + " has no field " + (key.isString() ? "'" + key.asString() + "'" : toRepr(key)) + + " (its fields are: " + (fields.empty() ? "none" : fields) + ")"); + } + if (!type->isAny()) { + Value converted; + CoerceFailure failure; + if (!coerceToType(value, *type, converted, failure)) { + throw RuntimeError(withHint("field '" + key.asString() + "' of " + info.name + " is " + type->name() + + ", so it can't hold " + failure.what, + failure)); + } + value = std::move(converted); + } + } + map.set(key, std::move(value)); +} + +int Interpreter::runMain() { + Variable* main = globals_->findLocal("main"); + if (main == nullptr || !main->value.isFunction() || main->value.asFunction().kind != Callable::Kind::User) return 0; + Value function = main->value; + const auto& declaration = *static_cast(function.asFunction()).declaration; + + std::vector args; + Variable* programArgs = globals_->find("args"); + Value list = programArgs != nullptr && programArgs->value.isList() ? programArgs->value : Value::fromList(newList()); + if (declaration.params.size() == 1) { + args.push_back(list); + } else if (declaration.params.size() >= 2) { + args.push_back(Value::fromInt(static_cast(list.asList().items.size()))); + args.push_back(list); + } + + const SourceFile* saved = currentSource_; + clearErrorSite(); + trace_.clear(); + Value result; + try { + ErrorSite site(*this, declaration.line, declaration.column); + runningMain_ = true; // main() has no call site to show in a traceback + result = call(function, std::move(args), declaration.line, declaration.column); + } catch (RuntimeError& error) { + applyErrorSite(error); + currentSource_ = saved; + throw; + } + currentSource_ = saved; + int64_t code = 0; + if (result.isInt() && toInt64(result, code)) return static_cast(code); + return 0; +} + +Value Interpreter::call(const Value& callee, std::vector args, int line, int column, + const std::vector>* named) { if (!callee.isFunction()) { throw RuntimeError(aTypeName(callee) + " value can't be called like a function", line, column); } Callable& callable = callee.asFunction(); + const bool hasNamed = named != nullptr && !named->empty(); if (callable.kind == Callable::Kind::Native) { auto& native = static_cast(callable); + if (hasNamed) { + throw RuntimeError(native.name + "() doesn't take arguments by name (like '" + (*named)[0].first + + ": ...'); pass them in order", + line, column); + } size_t count = args.size(); if (count < static_cast(native.minArgs) || (native.maxArgs >= 0 && count > static_cast(native.maxArgs))) { @@ -1069,16 +2012,47 @@ Value Interpreter::call(const Value& callee, std::vector args, int line, line, column); } // Errors from the builtin are reported at the call (the enclosing ErrorSite). + NativeFrame frame(trace_, native.name, currentSource_, line, column); return native.fn(*this, args); } + if (callable.kind == Callable::Kind::Struct) { + return constructStruct(std::static_pointer_cast(callee.functionPtr()), std::move(args), line, column, + named); + } + auto& function = static_cast(callable); const FunctionExpr& decl = *function.declaration; - if (args.size() < decl.requiredParams || args.size() > decl.params.size()) { - throw RuntimeError(arityMessage(function.name, decl.requiredParams, static_cast(decl.params.size()), - args.size()), + const size_t fixed = decl.variadic ? decl.params.size() - 1 : decl.params.size(); + if ((!hasNamed && args.size() < decl.requiredParams) || (!decl.variadic && args.size() > decl.params.size())) { + throw RuntimeError(arityMessage(function.name, decl.requiredParams, + decl.variadic ? -1 : static_cast(decl.params.size()), args.size()), line, column); } + // Line the arguments up with the parameters; extra ones go to ...rest. + std::vector slots(fixed); + std::vector filled(fixed, false); + std::vector rest; + for (size_t i = 0; i < args.size(); ++i) { + if (i < fixed) { + slots[i] = std::move(args[i]); + filled[i] = true; + } else { + rest.push_back(std::move(args[i])); + } + } + if (hasNamed) { + std::vector names; + for (size_t i = 0; i < fixed; ++i) names.push_back(decl.params[i].name); + bindNamedArgs(functionLabel(function.name), "parameter", names, *named, slots, filled, line, column, + decl.variadic ? decl.params.back().name : ""); + for (size_t i = 0; i < fixed; ++i) { + if (!filled[i] && !decl.params[i].defaultValue) { + throw RuntimeError(functionLabel(function.name) + " needs a value for '" + decl.params[i].name + "'", line, + column); + } + } + } if (callDepth_ >= maxCallDepth_) { throw RuntimeError("maximum recursion depth (" + std::to_string(maxCallDepth_) + ") exceeded; is there a recursive call without a base case?", @@ -1088,10 +2062,30 @@ Value Interpreter::call(const Value& callee, std::vector args, int line, auto scope = newEnvironment(function.closure); { - FrameGuard frame(*this, function.source, &decl); + FrameGuard frame(*this, function.source, &decl, &function.name, runningMain_ ? -1 : line, column); + runningMain_ = false; for (size_t i = 0; i < decl.params.size(); ++i) { const Param& param = decl.params[i]; - Value arg = i < args.size() ? std::move(args[i]) : eval(*param.defaultValue, scope); + if (i == fixed) { // ...rest: the remaining arguments, as a list + for (auto& extra : rest) { + Value converted; + CoerceFailure failure; + if (!coerceToType(extra, param.type, converted, failure)) { + RuntimeError error(withHint(functionLabel(function.name) + " expects each of '" + param.name + + "' to be " + param.type.name() + ", but got " + failure.what, + failure), + line, column); + error.source = frame.callerSource(); + throw error; + } + extra = std::move(converted); + } + TypeSpec listType; + listType.kind = TypeKind::List; + scope->define(param.name, Value::fromList(newList(std::move(rest))), listType); + continue; + } + Value arg = filled[i] ? std::move(slots[i]) : eval(*param.defaultValue, scope); if (!param.type.isAny()) { Value converted; CoerceFailure failure; @@ -1133,6 +2127,32 @@ Value Interpreter::binaryOp(TokenType op, const Value& a, const Value& b) { NumOp numOp; if (toNumOp(op, numOp) && a.isNumber() && b.isNumber()) return numericOp(numOp, a, b); + // Structs can define operators: Vec operator+(Vec other) { ... } + if (a.isMap() && a.asMap().structType && !a.asMap().structType->methods.empty()) { + const char* name = nullptr; + switch (op) { + case TokenType::Plus: name = "operator+"; break; + case TokenType::Minus: name = "operator-"; break; + case TokenType::Star: name = "operator*"; break; + case TokenType::Slash: name = "operator/"; break; + case TokenType::Percent: name = "operator%"; break; + case TokenType::StarStar: name = "operator**"; break; + default: break; + } + if (name != nullptr) { + if (const Value* method = structMethod(a, name)) return call(*method, {a, b}); + } + if (const Value* less = structMethod(a, "operator<")) { + switch (op) { + case TokenType::Less: return Value::fromBool(call(*less, {a, b}).truthy()); + case TokenType::Greater: return Value::fromBool(compareValues(b, a) < 0); + case TokenType::LessEqual: return Value::fromBool(compareValues(b, a) >= 0); + case TokenType::GreaterEqual: return Value::fromBool(!call(*less, {a, b}).truthy()); + default: break; + } + } + } + switch (op) { case TokenType::Plus: if (a.isChar() && b.isInt()) return shiftChar(a, b, 1); @@ -1218,6 +2238,7 @@ Value Interpreter::binaryOp(TokenType op, const Value& a, const Value& b) { } bool Interpreter::contains(const Value& container, const Value& item) { + if (const Value* method = structMethod(container, "contains")) return call(*method, {container, item}).truthy(); switch (container.type()) { case Value::Type::String: if (!item.isString() && !item.isChar()) { @@ -1238,6 +2259,7 @@ bool Interpreter::contains(const Value& container, const Value& item) { } Value Interpreter::indexGet(const Value& object, const Value& index) { + if (const Value* method = structMethod(object, "operator[]")) return call(*method, {object, index}); switch (object.type()) { case Value::Type::List: { const auto& items = object.asList().items; @@ -1261,12 +2283,13 @@ Value Interpreter::indexGet(const Value& object, const Value& index) { void Interpreter::indexSet(const Value& object, const Value& index, Value value) { switch (object.type()) { case Value::Type::List: { - auto& items = object.asList().items; - items[resolveIndex(index, items.size(), "list")] = std::move(value); + ListObj& list = object.asList(); + size_t position = resolveIndex(index, list.items.size(), "list"); + list.items[position] = checkListItem(list, std::move(value)); return; } case Value::Type::Map: - object.asMap().set(index, std::move(value)); + setElement(object.asMap(), index, std::move(value)); return; case Value::Type::String: throw RuntimeError("strings can't be changed in place; build a new string instead"); diff --git a/src/lexer.cpp b/src/lexer.cpp index e6bb78c..6e49ab7 100644 --- a/src/lexer.cpp +++ b/src/lexer.cpp @@ -188,6 +188,17 @@ void Lexer::scanToken() { fString(); return; } + if (c == 'r' && peek() == '"') { // r"raw: \d+ stays as written" + advance(); + if (peek() == '"' && peek(1) == '"') { + advance(); + advance(); + longString(true); + } else { + rawString(); + } + return; + } if (isIdentifierStart(c)) { identifier(); return; @@ -205,15 +216,42 @@ void Lexer::scanToken() { case '{': brackets_.push_back('{'); add(TokenType::LeftBrace); return; case '}': closeBracket(); add(TokenType::RightBrace); return; case ',': add(TokenType::Comma); return; - case '.': add(TokenType::Dot); return; + case '.': + if (match('.')) { + if (match('.')) { + add(TokenType::Ellipsis); + } else { + add(match('=') ? TokenType::DotDotEqual : TokenType::DotDot); + } + } else { + add(TokenType::Dot); + } + return; case ';': add(TokenType::Semicolon); return; - case '?': add(TokenType::Question); return; + case '?': + if (match('?')) { + add(TokenType::QuestionQuestion); + } else if (peek() == '.' && isIdentifierStart(peek(1))) { + advance(); + add(TokenType::QuestionDot); + } else { + add(TokenType::Question); + } + return; case '~': add(TokenType::Tilde); return; case ':': add(match('=') ? TokenType::Declare : TokenType::Colon); return; - case '+': add(match('=') ? TokenType::PlusEqual : TokenType::Plus); return; + case '+': + if (match('+')) { + add(TokenType::PlusPlus); + } else { + add(match('=') ? TokenType::PlusEqual : TokenType::Plus); + } + return; case '-': if (match('>')) { add(TokenType::Arrow); + } else if (match('-')) { + add(TokenType::MinusMinus); } else { add(match('=') ? TokenType::MinusEqual : TokenType::Minus); } @@ -265,7 +303,13 @@ void Lexer::scanToken() { } return; case '"': - string(c); + if (peek() == '"' && peek(1) == '"') { + advance(); + advance(); + longString(false); + } else { + string(c); + } return; case '\'': character(); @@ -487,6 +531,57 @@ void Lexer::string(char quote) { tokens_.back().text = std::move(value); } +// r"C:\temp\new" : backslashes are kept as written (handy for regular expressions). +void Lexer::rawString() { + std::string value; + while (true) { + if (atEnd() || peek() == '\n') { + error(startLine_, startColumn_, "unterminated raw string (missing closing \")"); + return; + } + if (peek() == '"') { + advance(); + break; + } + value += advance(); + } + add(TokenType::String); + tokens_.back().text = std::move(value); +} + +// """text that can span +// several lines""" (a newline right after the opening quotes is skipped) +void Lexer::longString(bool raw) { + std::string value; + if (peek() == '\r' && peek(1) == '\n') advance(); + if (peek() == '\n') advance(); + while (true) { + if (atEnd()) { + error(startLine_, startColumn_, "unterminated string (missing closing \"\"\")"); + return; + } + if (peek() == '"' && peek(1) == '"' && peek(2) == '"') { + advance(); + advance(); + advance(); + break; + } + if (peek() == '\\' && !raw) { + advance(); + char32_t escaped; + if (escape(escaped)) appendUtf8(value, escaped); + continue; + } + if (peek() == '\r' && peek(1) == '\n') { + advance(); // keep \n line endings only + continue; + } + value += advance(); + } + add(TokenType::String); + tokens_.back().text = std::move(value); +} + // 'a', '\n', '\u{1F600}': exactly one character. void Lexer::character() { char32_t value = 0; diff --git a/src/main.cpp b/src/main.cpp index d89a7f8..16970ac 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,7 +1,12 @@ +#include "big_stack.hpp" #include "interpreter.hpp" #include "repl.hpp" #include "runner.hpp" +#include +#include +#include +#include #include #include #include @@ -17,6 +22,7 @@ void printUsage(std::ostream& out) { " ascend [args...] run a program\n" " ascend -e [args...] run code given on the command line\n" " ascend --check check a program for errors without running it\n" + " ascend test [paths...] run the tests: files named *_test.asc or test_*.asc\n" " ascend -h | --help show this help\n" " ascend -v | --version show the version\n" "\n" @@ -31,10 +37,83 @@ bool readFile(const std::string& path, std::string& contents) { return true; } +bool isTestFile(const std::filesystem::path& path) { + std::string name = path.filename().string(); + if (path.extension() != ".asc") return false; + return name.compare(0, 5, "test_") == 0 || (name.size() > 9 && name.compare(name.size() - 9, 9, "_test.asc") == 0); +} + +// ascend test [paths...]: runs every test file in the given files and +// folders (default: the current folder), each in a fresh interpreter. A test +// file fails if it ends with an error or a nonzero exit code. +int runTests(const std::vector& paths) { + namespace fs = std::filesystem; + std::vector files; + for (const auto& path : paths) { + std::error_code error; + if (fs::is_regular_file(path, error)) { + files.emplace_back(path); + continue; + } + if (!fs::is_directory(path, error)) { + std::cerr << "error: '" << path << "' is not a file or folder\n"; + return 1; + } + fs::recursive_directory_iterator it(path, fs::directory_options::skip_permission_denied, error), end; + for (; !error && it != end; it.increment(error)) { + std::string name = it->path().filename().string(); + bool skipped = !name.empty() && (name[0] == '.' || name == "ascend_packages" || name == "build" || + name == "node_modules"); + if (it->is_directory() && skipped) { + it.disable_recursion_pending(); + continue; + } + if (it->is_regular_file() && isTestFile(it->path())) files.push_back(it->path()); + } + } + std::sort(files.begin(), files.end()); + if (files.empty()) { + std::cout << "No test files found (test files are named like math_test.asc or test_math.asc)\n"; + return 0; + } + + int failed = 0; + auto started = std::chrono::steady_clock::now(); + for (const auto& file : files) { + std::string name = file.generic_string(); + std::string source; + std::cout << "=== " << name << "\n"; + std::cout.flush(); + int code = 1; + if (!readFile(file.string(), source)) { + std::cerr << "error: can't open file '" << name << "'\n"; + } else { + ascend::Interpreter interpreter; + interpreter.defineGlobal("args", ascend::Value::fromList(interpreter.newList())); + ascend::RunOptions options; + options.filename = name; + code = ascend::runSource(interpreter, source, options, std::cerr).exitCode; + } + std::cout.flush(); + std::cerr.flush(); + if (code == 0) { + std::cout << "--- PASS " << name << "\n"; + } else { + std::cout << "--- FAIL " << name << " (exit code " << code << ")\n"; + ++failed; + } + } + double seconds = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + char time[32]; + std::snprintf(time, sizeof time, "%.2f", seconds); + std::cout << "\n" << (files.size() - static_cast(failed)) << " of " << files.size() << " test files passed" + << (failed > 0 ? " (" + std::to_string(failed) + " failed)" : "") << " in " << time << "s\n"; + return failed > 0 ? 1 : 0; +} + } // namespace -int main(int argc, char* argv[]) { - std::vector args(argv + 1, argv + argc); +static int runCommandLine(const std::vector& args) { if (args.empty()) { ascend::Interpreter interpreter; @@ -55,6 +134,11 @@ int main(int argc, char* argv[]) { std::string source; ascend::RunOptions options; size_t scriptArgsStart; + if (first == "test") { + std::vector paths(args.begin() + 1, args.end()); + if (paths.empty()) paths.push_back("."); + return runTests(paths); + } if (first == "--check") { if (args.size() != 2) { std::cerr << "error: --check needs exactly one file\n\n"; @@ -101,3 +185,9 @@ int main(int argc, char* argv[]) { return ascend::runSource(interpreter, source, options, std::cerr).exitCode; } + +int main(int argc, char* argv[]) { + std::vector args(argv + 1, argv + argc); + // Programs run on a thread with a large stack, so they can recurse deeply. + return ascend::runWithLargeStack([&] { return runCommandLine(args); }); +} diff --git a/src/parser.cpp b/src/parser.cpp index e4c274c..4f48131 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -28,7 +28,97 @@ bool isAssignmentOperator(TokenType type) { } bool isAssignable(const Expr& expr) { - return expr.kind == ExprKind::Variable || expr.kind == ExprKind::Index || expr.kind == ExprKind::Member; + if (expr.kind == ExprKind::Member) return !expr.as().optional; + return expr.kind == ExprKind::Variable || expr.kind == ExprKind::Index; +} + +// Sets a flag for the rest of a scope, and restores it even when a syntax +// error unwinds the parser. +class FlagGuard { +public: + FlagGuard(bool& flag, bool value) : flag_(flag), saved_(flag) { flag_ = value; } + ~FlagGuard() { flag_ = saved_; } + FlagGuard(const FlagGuard&) = delete; + FlagGuard& operator=(const FlagGuard&) = delete; + +private: + bool& flag_; + bool saved_; +}; + +// Gives a method its first parameter, the value it's called on. +void addThisParameter(FunctionExpr& method) { + Param self; + self.name = "this"; + method.params.insert(method.params.begin(), std::move(self)); + ++method.requiredParams; +} + +// Tokens that continue an expression after a closing parenthesis. Anything +// else after `if (...)` starts the body: `if (x > 0) print(x)`. +bool continuesExpression(TokenType type) { + switch (type) { + case TokenType::Plus: + case TokenType::Minus: + case TokenType::Star: + case TokenType::StarStar: + case TokenType::Slash: + case TokenType::Percent: + case TokenType::Ampersand: + case TokenType::Pipe: + case TokenType::Caret: + case TokenType::ShiftLeft: + case TokenType::ShiftRight: + case TokenType::EqualEqual: + case TokenType::NotEqual: + case TokenType::Less: + case TokenType::LessEqual: + case TokenType::Greater: + case TokenType::GreaterEqual: + case TokenType::In: + case TokenType::And: + case TokenType::Or: + case TokenType::Question: + case TokenType::Dot: + case TokenType::LeftBracket: + case TokenType::LeftParen: + return true; + default: + return false; + } +} + +// Tokens that can start an operand, so `(i32) x` is a cast. +bool startsOperand(TokenType type) { + switch (type) { + case TokenType::Identifier: + case TokenType::Int: + case TokenType::Float: + case TokenType::String: + case TokenType::Char: + case TokenType::FStringStart: + case TokenType::LeftParen: + case TokenType::LeftBracket: + case TokenType::True: + case TokenType::False: + case TokenType::Null: + case TokenType::Minus: + case TokenType::Not: + case TokenType::Tilde: + case TokenType::PlusPlus: + case TokenType::MinusMinus: + return true; + default: + return false; + } +} + +// A name that can be used as a variable: "geometry", not "my-lib" or "if". +bool isIdentifierText(const std::string& text) { + Lexer lexer(text); + std::vector tokens = lexer.tokenize(); + return lexer.errors().empty() && !tokens.empty() && tokens[0].type == TokenType::Identifier && + tokens[0].lexeme == text; } template @@ -85,10 +175,22 @@ std::unique_ptr Parser::parseProgram() { // Records facts about a block's statements that the interpreter relies on. void Parser::noteStatement(Block& block, const Stmt& stmt) { - if (stmt.kind == StmtKind::Declare) block.declaresNames = true; - if (stmt.kind == StmtKind::Function) { - block.declaresNames = true; - block.hasFunctions = true; + switch (stmt.kind) { + case StmtKind::Declare: + case StmtKind::Import: + case StmtKind::Enum: + block.declaresNames = true; + break; + case StmtKind::Function: + case StmtKind::Struct: + block.declaresNames = true; + block.hasFunctions = true; + break; + case StmtKind::Group: + for (const auto& inner : stmt.as().statements) noteStatement(block, *inner); + break; + default: + break; } } @@ -120,21 +222,51 @@ StmtPtr Parser::statement() { case TokenType::Const: advance(); return constDeclaration(token); case TokenType::Match: advance(); return matchStatement(token); case TokenType::Identifier: { - // C-style declarations: `u8 age = 30` and functions `i32 add(i32 a, i32 b) { }`. - size_t length = typeLength(0); - if (length > 0 && peek(length).type == TokenType::Identifier) { + // Words that only have a special meaning at the start of a statement, + // so they can still be used as names elsewhere. + TokenType next = peek(1).type; + if (isWord(0, "import") && (next == TokenType::Identifier || next == TokenType::String)) { + advance(); + return importStatement(token); + } + if (isWord(0, "from") && looksLikeFromImport()) { + advance(); + return fromImportStatement(token); + } + if ((isWord(0, "struct") || isWord(0, "enum")) && next == TokenType::Identifier && + (peek(2).type == TokenType::LeftBrace || peek(2).type == TokenType::Newline || + (token.lexeme == "struct" && peek(2).type == TokenType::Colon))) { + advance(); + return token.lexeme == "struct" ? structDeclaration(token) : enumDeclaration(token); + } + if (isWord(0, "switch") && looksLikeSwitch()) { + advance(); + return switchStatement(token); + } + size_t afterDo = current_ + 1; + while (afterDo + 1 < tokens_.size() && tokens_[afterDo].type == TokenType::Newline) ++afterDo; + if (isWord(0, "do") && (tokens_[afterDo].type == TokenType::LeftBrace || next == TokenType::Identifier)) { + advance(); + return doStatement(token); + } + if (isWord(0, "auto") && next == TokenType::Identifier) return typedDeclaration(); + if ((isWord(0, "case") && next != TokenType::Declare && !isAssignmentOperator(next)) || + (isWord(0, "default") && next == TokenType::Colon)) { + error(token, "'" + token.lexeme + "' can only be used inside a switch"); + } + + // C-style declarations: `u8 age = 30`, `Point p = Point(1, 2)` and + // functions `i32 add(i32 a, i32 b) { }`. + size_t length = typeLength(0, true); + if (length > 0 && peek(length).type == TokenType::Identifier && isDeclaration(length)) { if (peek(length + 1).type == TokenType::LeftParen) return typedFunctionDeclaration(); return typedDeclaration(); } - if (peek(1).type == TokenType::Colon) { + if (next == TokenType::Colon) { // The other common style, `age: u8 = 30`: suggest the C-style form. std::string type = typeLength(2) > 0 ? peek(2).lexeme : "i32"; error(token, "write the type before the name, like '" + type + " " + token.lexeme + " = ...'"); } - if (length == 0 && peek(1).type == TokenType::Identifier) { - TypeSpec unused; - if (!lookupType(token.lexeme, unused)) error(token, unknownTypeMessage(token.lexeme)); - } break; } case TokenType::Break: { @@ -170,6 +302,9 @@ StmtPtr Parser::statement() { stmt->body = block("block"); return stmt; } + case TokenType::PlusPlus: + case TokenType::MinusMinus: + break; // ++x as a statement case TokenType::Else: error(token, "'else' without a matching 'if'"); case TokenType::Catch: @@ -181,9 +316,41 @@ StmtPtr Parser::statement() { } // Expression statements, declarations (x := 1) and assignments (x = 1, x += 1). -StmtPtr Parser::simpleStatement() { +StmtPtr Parser::simpleStatement(bool requireEnd, bool allowMulti) { ExprPtr expr = expression(); + // a, b := pair a, b = b, a + if (allowMulti && check(TokenType::Comma)) { + auto stmt = std::make_unique(expr->line, expr->column); + stmt->targets.push_back(std::move(expr)); + while (match(TokenType::Comma)) stmt->targets.push_back(expression()); + if (match(TokenType::Declare)) { + stmt->declare = true; + for (const auto& target : stmt->targets) { + if (target->kind != ExprKind::Variable) { + error(peek(), "':=' can only declare plain variable names (use '=' to change elements or fields)"); + } + } + } else if (match(TokenType::Equal)) { + for (const auto& target : stmt->targets) { + if (!isAssignable(*target)) error(previous(), "can't assign to one of these expressions"); + } + } else { + error(peek(), "expected '=' or ':=' after the variables, found " + describeToken(peek())); + } + const Token& start = peek(); + skipNewlines(); + do { + stmt->values.push_back(expression()); + } while (match(TokenType::Comma)); + if (stmt->values.size() != 1 && stmt->values.size() != stmt->targets.size()) { + error(start, "there are " + std::to_string(stmt->targets.size()) + " variables but " + + std::to_string(stmt->values.size()) + " values"); + } + if (requireEnd) expectStatementEnd(); + return stmt; + } + if (check(TokenType::Declare)) { const Token& op = advance(); if (expr->kind != ExprKind::Variable) { @@ -193,7 +360,7 @@ StmtPtr Parser::simpleStatement() { stmt->name = expr->as().name; skipNewlines(); stmt->value = expression(); - expectStatementEnd(); + if (requireEnd) expectStatementEnd(); return stmt; } @@ -205,20 +372,20 @@ StmtPtr Parser::simpleStatement() { stmt->target = std::move(expr); skipNewlines(); stmt->value = expression(); - expectStatementEnd(); + if (requireEnd) expectStatementEnd(); return stmt; } auto stmt = std::make_unique(expr->line, expr->column); stmt->expr = std::move(expr); - expectStatementEnd(); + if (requireEnd) expectStatementEnd(); return stmt; } StmtPtr Parser::ifStatement(const Token& keyword) { auto stmt = nodeAt(keyword); stmt->condition = condition("if"); - stmt->thenBody = block("'if' body"); + stmt->thenBody = body("'if' body"); if (peekPastNewlines().type == TokenType::Else) { skipNewlines(); const Token& elseToken = advance(); @@ -227,7 +394,7 @@ StmtPtr Parser::ifStatement(const Token& keyword) { stmt->elseBranch = ifStatement(ifToken); } else { auto elseBlock = nodeAt(elseToken); - elseBlock->body = block("'else' body"); + elseBlock->body = body("'else' body"); stmt->elseBranch = std::move(elseBlock); } } @@ -237,29 +404,392 @@ StmtPtr Parser::ifStatement(const Token& keyword) { StmtPtr Parser::whileStatement(const Token& keyword) { auto stmt = nodeAt(keyword); stmt->condition = condition("while"); - stmt->body = block("'while' body"); + stmt->body = body("'while' body"); + return stmt; +} + +// do { ... } while (condition) +StmtPtr Parser::doStatement(const Token& keyword) { + auto stmt = nodeAt(keyword); + stmt->doWhile = true; + stmt->body = body("'do' body"); + skipNewlines(); + consume(TokenType::While, "'while' and a condition after the 'do' body"); + stmt->condition = condition("while"); + expectStatementEnd(); return stmt; } +// for x in items { } for (x in items) { } for (string s : names) { } +// for (i32 i = 0; i < n; i++) { } for i := 0; i < n; i++ { } +// for { } (loops until `break`) StmtPtr Parser::forStatement(const Token& keyword) { + if (check(TokenType::LeftBrace)) { + auto stmt = nodeAt(keyword); + stmt->body = block("'for' body"); + return stmt; + } + bool parenthesized = false; + if (check(TokenType::LeftParen)) { + size_t close = matchingParen(current_); + if (close != std::string::npos) { + if (hasTopLevelSemicolon(current_ + 1, close)) return cForStatement(keyword, true); + parenthesized = true; + } + } else { + // Without parentheses the header ends at the body's '{' or the end of the line. + size_t end = current_; + int depth = 0; + for (; end < tokens_.size(); ++end) { + TokenType type = tokens_[end].type; + if (type == TokenType::Eof || (depth == 0 && (type == TokenType::LeftBrace || type == TokenType::Newline))) { + break; + } + if (type == TokenType::LeftParen || type == TokenType::LeftBracket || type == TokenType::LeftBrace) ++depth; + if (type == TokenType::RightParen || type == TokenType::RightBracket || type == TokenType::RightBrace) --depth; + } + if (hasTopLevelSemicolon(current_, end)) return cForStatement(keyword, false); + } + auto stmt = nodeAt(keyword); - bool parenthesized = match(TokenType::LeftParen); // allow for (x in xs) { ... } - stmt->first = consume(TokenType::Identifier, "a loop variable name after 'for'").lexeme; - if (match(TokenType::Comma)) { - stmt->second = consume(TokenType::Identifier, "a second loop variable name after ','").lexeme; + if (parenthesized) advance(); + loopVariable(stmt->first, stmt->firstType); + if (match(TokenType::Comma)) loopVariable(stmt->second, stmt->secondType); + if (!match(TokenType::In) && !match(TokenType::Colon)) { + error(peek(), "expected 'in' after the loop variable, found " + describeToken(peek())); + } + if (parenthesized) { + stmt->iterable = expression(); + consume(TokenType::RightParen, "')' after the 'for' header"); + } else { + stmt->iterable = condition("in"); + } + stmt->body = body("'for' body"); + return stmt; +} + +// A loop variable, optionally with a type: `item`, `string name`, `auto x`. +void Parser::loopVariable(std::string& name, TypeSpec& type) { + if (isWord(0, "auto") && peek(1).type == TokenType::Identifier) { + advance(); + } else { + size_t length = typeLength(0, true); + if (length > 0 && peek(length).type == TokenType::Identifier) type = parseType(false); } - consume(TokenType::In, "'in' after the loop variable"); - stmt->iterable = condition("in"); + name = consume(TokenType::Identifier, "a loop variable name after 'for'").lexeme; +} + +StmtPtr Parser::cForStatement(const Token& keyword, bool parenthesized) { + auto stmt = nodeAt(keyword); + if (parenthesized) advance(); + if (!check(TokenType::Semicolon)) stmt->init = forClauses(); + consume(TokenType::Semicolon, "';' after the first part of the 'for' header"); + if (!check(TokenType::Semicolon)) stmt->condition = expression(); + consume(TokenType::Semicolon, "';' after the loop condition"); + TokenType end = parenthesized ? TokenType::RightParen : TokenType::LeftBrace; + if (!check(end)) stmt->step = forClauses(); if (parenthesized) consume(TokenType::RightParen, "')' after the 'for' header"); - stmt->body = block("'for' body"); + int functionsBefore = functionCount_; + stmt->body = body("'for' body"); + stmt->copyPerIteration = functionCount_ != functionsBefore; + return stmt; +} + +// The statements in the first or last part of a C-style for header, +// separated by commas: `i32 i = 0, j = n` or `i++, j--`. +std::vector Parser::forClauses() { + std::vector clauses; + do { + size_t length = typeLength(0, true); + bool declaration = (isWord(0, "auto") && peek(1).type == TokenType::Identifier) || + (length > 0 && peek(length).type == TokenType::Identifier); + clauses.push_back(declaration ? typedDeclaration(false) : simpleStatement(false, false)); + } while (match(TokenType::Comma)); + return clauses; +} + +// switch (value) { case 1: ... break default: ... } +StmtPtr Parser::switchStatement(const Token& keyword) { + auto stmt = nodeAt(keyword); + stmt->subject = condition("switch"); + skipNewlines(); + const Token& open = consume(TokenType::LeftBrace, "'{' to start the switch cases"); + NestingGuard guard(*this, open); + bool sawDefault = false; + while (true) { + skipSeparators(); + if (check(TokenType::RightBrace) || isAtEnd()) break; + bool isCase = isWord(0, "case") && peek(1).type != TokenType::Declare && !isAssignmentOperator(peek(1).type); + bool isDefault = isWord(0, "default") && peek(1).type == TokenType::Colon; + if (!isCase && !isDefault) { + if (stmt->cases.empty()) { + errors_.push_back({peek().line, peek().column, + "expected 'case' or 'default' in the switch, found " + describeToken(peek())}); + synchronize(); + continue; + } + parseStatementInto(stmt->body); + continue; + } + size_t start = current_; + try { + const Token& label = advance(); + SwitchCase entry; + entry.start = stmt->body.statements.size(); + if (isDefault) { + if (sawDefault) error(label, "a switch can only have one 'default'"); + sawDefault = true; + } else { + do { + entry.values.push_back(expression()); + } while (match(TokenType::Comma)); + } + consume(TokenType::Colon, isDefault ? "':' after 'default'" : "':' after the case value"); + stmt->cases.push_back(std::move(entry)); + } catch (const ParseError&) { + synchronize(); + if (current_ == start && !isAtEnd()) advance(); + } + } + if (!check(TokenType::RightBrace)) { + error(peek(), "expected '}' to close the switch that starts on line " + std::to_string(open.line) + + ", found " + describeToken(peek())); + } + advance(); + return stmt; +} + +// import geometry import lib.shapes as shapes import "../tools/x.asc" as x +StmtPtr Parser::importStatement(const Token& keyword) { + auto stmt = nodeAt(keyword); + do { + const Token& start = peek(); + ImportName item; + std::string lastPart; + item.name = modulePath(lastPart); + if (isWord(0, "as")) { + advance(); + item.binding = consume(TokenType::Identifier, "a name after 'as'").lexeme; + } else if (isIdentifierText(lastPart)) { + item.binding = lastPart; + } else { + error(start, "can't use \"" + lastPart + "\" as a variable name; give the module a name with 'as', like: " + "import " + start.lexeme + " as name"); + } + stmt->names.push_back(std::move(item)); + } while (match(TokenType::Comma)); + expectStatementEnd(); + return stmt; +} + +// from geometry import area, Point as P from "./helpers" import * +StmtPtr Parser::fromImportStatement(const Token& keyword) { + auto stmt = nodeAt(keyword); + stmt->from = true; + std::string lastPart; + stmt->fromPath = modulePath(lastPart); + if (!isWord(0, "import")) error(peek(), "expected 'import' after the module name, found " + describeToken(peek())); + advance(); + if (match(TokenType::Star)) { + stmt->all = true; + } else { + bool parenthesized = match(TokenType::LeftParen); + do { + if (parenthesized && check(TokenType::RightParen)) break; // trailing comma + ImportName item; + item.name = consume(TokenType::Identifier, "a name to import").lexeme; + item.binding = item.name; + if (isWord(0, "as")) { + advance(); + item.binding = consume(TokenType::Identifier, "a name after 'as'").lexeme; + } + stmt->names.push_back(std::move(item)); + } while (match(TokenType::Comma)); + if (parenthesized) consume(TokenType::RightParen, "')' after the imported names"); + } + expectStatementEnd(); + return stmt; +} + +// A module: a dotted name (lib.shapes means the file lib/shapes.asc, or the +// package lib's file shapes.asc) or a path in a string. `lastPart` is set to +// the name the module is stored under by default. +std::string Parser::modulePath(std::string& lastPart) { + if (check(TokenType::String)) { + const Token& token = advance(); + std::string path = token.text; + if (path.empty()) error(token, "the module path is empty"); + size_t slash = path.find_last_of("/\\"); + lastPart = slash == std::string::npos ? path : path.substr(slash + 1); + size_t dot = lastPart.rfind('.'); + if (dot != std::string::npos && dot > 0) lastPart = lastPart.substr(0, dot); // "shapes.asc" + return path; + } + std::string path = consume(TokenType::Identifier, "a module name").lexeme; + lastPart = path; + while (check(TokenType::Dot) && peek(1).type == TokenType::Identifier) { + advance(); + lastPart = advance().lexeme; + path += "/" + lastPart; + } + return path; +} + +// struct Point { +// f64 x +// f64 y = 0 +// } +StmtPtr Parser::structDeclaration(const Token& keyword) { + auto stmt = nodeAt(keyword); + stmt->name = consume(TokenType::Identifier, "a name after 'struct'").lexeme; + if (match(TokenType::Colon)) { // struct Dog : Animal + stmt->parent = consume(TokenType::Identifier, "the name of the struct to extend after ':'").lexeme; + } + skipNewlines(); + const Token& open = consume(TokenType::LeftBrace, "'{' to start the struct's fields"); + NestingGuard guard(*this, open); + while (true) { + skipSeparators(); + if (check(TokenType::RightBrace) || isAtEnd()) break; + size_t start = current_; + try { + // Methods: `f64 area() { ... }`, `func describe() { ... }`, `reset() { ... }`, + // and operators: `Vec operator+(Vec other) { ... }`. + std::string opName; + size_t length = typeLength(0, true); + bool typedMethod = !check(TokenType::Func) && !isWord(0, "operator") && length > 0 && + ((peek(length).type == TokenType::Identifier && peek(length + 1).type == TokenType::LeftParen) || + operatorName(length, opName) > 0); + size_t afterFunc = check(TokenType::Func) ? 1 : 0; + bool untypedMethod = !typedMethod && ((peek(afterFunc).type == TokenType::Identifier && + peek(afterFunc + 1).type == TokenType::LeftParen) || + operatorName(afterFunc, opName) > 0); + if (typedMethod || untypedMethod) { + TypeSpec returnType; + if (typedMethod) { + returnType = parseType(true); + } else { + match(TokenType::Func); + } + const Token& nameToken = peek(); + std::string name; + size_t count = operatorName(0, name); + if (count > 0) { + for (size_t i = 0; i < count; ++i) advance(); + } else { + name = advance().lexeme; + } + auto method = functionRest(nameToken, stmt->name + "." + name, typedMethod ? &returnType : nullptr); + addThisParameter(*method); + stmt->methods.emplace_back(name, std::move(method)); + continue; + } + + TypeSpec type; + if (length > 0 && peek(length).type == TokenType::Identifier) type = parseType(false); + do { + const Token& name = consume(TokenType::Identifier, "a field name"); + if (check(TokenType::LeftParen)) { + error(peek(), "a method needs its own line: '" + type.name() + " " + name.lexeme + "() { ... }'"); + } + if (check(TokenType::Colon)) { + error(peek(), "write the field's type before its name, like 'f64 " + name.lexeme + "'"); + } + StructField field; + field.name = name.lexeme; + field.type = type; + field.line = name.line; + field.column = name.column; + if (match(TokenType::Equal)) { + skipNewlines(); + field.defaultValue = expression(); + } + stmt->fields.push_back(std::move(field)); + } while (match(TokenType::Comma)); + expectStatementEnd(); + } catch (const ParseError&) { + synchronize(); + if (current_ == start && !isAtEnd()) advance(); + } + } + if (!check(TokenType::RightBrace)) { + error(peek(), "expected '}' to close the struct that starts on line " + std::to_string(open.line) + + ", found " + describeToken(peek())); + } + advance(); + return stmt; +} + +// `operator+`, `operator==`, `operator[]`... starting `offset` tokens ahead and +// followed by '('. Returns how many tokens the name takes (0 if it isn't one). +size_t Parser::operatorName(size_t offset, std::string& name) const { + if (!isWord(offset, "operator")) return 0; + const Token& symbol = peek(offset + 1); + switch (symbol.type) { + case TokenType::Plus: + case TokenType::Minus: + case TokenType::Star: + case TokenType::Slash: + case TokenType::Percent: + case TokenType::StarStar: + case TokenType::EqualEqual: + case TokenType::Less: + if (peek(offset + 2).type != TokenType::LeftParen) return 0; + name = "operator" + symbol.lexeme; + return 2; + case TokenType::LeftBracket: + if (peek(offset + 2).type != TokenType::RightBracket || peek(offset + 3).type != TokenType::LeftParen) return 0; + name = "operator[]"; + return 3; + default: + return 0; + } +} + +// enum Color { Red, Green = 5, Blue } +StmtPtr Parser::enumDeclaration(const Token& keyword) { + auto stmt = nodeAt(keyword); + stmt->name = consume(TokenType::Identifier, "a name after 'enum'").lexeme; + skipNewlines(); + const Token& open = consume(TokenType::LeftBrace, "'{' to start the enum's members"); + NestingGuard guard(*this, open); + while (true) { + skipSeparators(); + if (check(TokenType::RightBrace) || isAtEnd()) break; + const Token& name = consume(TokenType::Identifier, "an enum member name"); + EnumMember member; + member.name = name.lexeme; + member.line = name.line; + member.column = name.column; + if (match(TokenType::Equal)) { + skipNewlines(); + member.value = expression(); + } + stmt->members.push_back(std::move(member)); + if (!match(TokenType::Comma) && !check(TokenType::Newline) && !check(TokenType::Semicolon) && + !check(TokenType::RightBrace)) { + error(peek(), "expected ',' or '}' after the enum member, found " + describeToken(peek())); + } + } + if (!check(TokenType::RightBrace)) { + error(peek(), "expected '}' to close the enum that starts on line " + std::to_string(open.line) + + ", found " + describeToken(peek())); + } + advance(); return stmt; } StmtPtr Parser::returnStatement(const Token& keyword) { auto stmt = nodeAt(keyword); if (!check(TokenType::Newline) && !check(TokenType::Semicolon) && !check(TokenType::RightBrace) && - !isAtEnd()) { + !check(TokenType::Else) && !isAtEnd()) { stmt->value = expression(); + if (check(TokenType::Comma)) { // return a, b returns the list [a, b] + auto list = std::make_unique(stmt->value->line, stmt->value->column); + list->elements.push_back(std::move(stmt->value)); + while (match(TokenType::Comma)) list->elements.push_back(expression()); + stmt->value = std::move(list); + } } expectStatementEnd(); return stmt; @@ -268,18 +798,28 @@ StmtPtr Parser::returnStatement(const Token& keyword) { StmtPtr Parser::tryStatement(const Token& keyword) { auto stmt = nodeAt(keyword); stmt->body = block("'try' body"); - if (peekPastNewlines().type != TokenType::Catch) { - error(peekPastNewlines(), "expected 'catch' after the 'try' block, found " + describeToken(peekPastNewlines())); + if (peekPastNewlines().type == TokenType::Catch) { + skipNewlines(); + advance(); // catch + stmt->hasCatch = true; + if (match(TokenType::LeftParen)) { + stmt->errorName = consume(TokenType::Identifier, "an error variable name").lexeme; + consume(TokenType::RightParen, "')' after the error variable name"); + } else if (check(TokenType::Identifier)) { + stmt->errorName = advance().lexeme; + } + stmt->handler = block("'catch' body"); + } + const Token& after = peekPastNewlines(); + if (after.type == TokenType::Identifier && after.lexeme == "finally") { + skipNewlines(); + advance(); + stmt->hasFinally = true; + stmt->finallyBody = block("'finally' body"); + } + if (!stmt->hasCatch && !stmt->hasFinally) { + error(after, "expected 'catch' or 'finally' after the 'try' block, found " + describeToken(after)); } - skipNewlines(); - advance(); // catch - if (match(TokenType::LeftParen)) { - stmt->errorName = consume(TokenType::Identifier, "an error variable name").lexeme; - consume(TokenType::RightParen, "')' after the error variable name"); - } else if (check(TokenType::Identifier)) { - stmt->errorName = advance().lexeme; - } - stmt->handler = block("'catch' body"); return stmt; } @@ -294,38 +834,101 @@ StmtPtr Parser::throwStatement(const Token& keyword) { } // How many tokens the type name starting `offset` tokens ahead takes up -// (1 for `u8`, 2 for `string?`), or 0 if there is no type name there. -size_t Parser::typeLength(size_t offset) const { +// (1 for `u8`, 2 for `string?`), or 0 if there is no type name there. With +// `allowStruct`, any other name counts as a (struct) type name too; callers +// check that a variable name follows. +size_t Parser::typeLength(size_t offset, bool allowStruct) const { const Token& token = peek(offset); TypeSpec type; - bool isType = token.type == TokenType::Func || (token.type == TokenType::Identifier && lookupType(token.lexeme, type)); + bool isType = token.type == TokenType::Func || + (token.type == TokenType::Identifier && (allowStruct || lookupType(token.lexeme, type))); if (!isType) return 0; - return peek(offset + 1).type == TokenType::Question ? 2 : 1; + size_t length = 1; + // list, map> + if ((token.lexeme == "list" || token.lexeme == "map") && peek(offset + 1).type == TokenType::Less) { + int depth = 0; + size_t i = offset + 1; + for (;; ++i) { + TokenType kind = peek(i).type; + if (kind == TokenType::Less) { + ++depth; + } else if (kind == TokenType::Greater) { + --depth; + } else if (kind == TokenType::ShiftRight) { + depth -= 2; + } else if (kind != TokenType::Identifier && kind != TokenType::Comma && kind != TokenType::Question && + kind != TokenType::Func) { + return 0; + } + if (depth <= 0) break; + } + if (depth < 0) return 0; + length = i - offset + 1; + } + return peek(offset + length).type == TokenType::Question ? length + 1 : length; +} + +// After `Name name` or `Name? name` at the start of a statement: is it a +// declaration? `Point p` always is, but `ready ? go() : wait()` isn't. +bool Parser::isDeclaration(size_t typeTokens) const { + TypeSpec builtin; + if (typeTokens == 1 || peek().type == TokenType::Func || lookupType(peek().lexeme, builtin)) return true; + TokenType after = peek(typeTokens + 1).type; + if (after == TokenType::Equal || after == TokenType::Newline || after == TokenType::Semicolon || + after == TokenType::Eof || after == TokenType::RightBrace) { + return true; + } + if (after != TokenType::LeftParen) return false; + size_t close = matchingParen(current_ + typeTokens + 1); + if (close == std::string::npos) return false; + TokenType next = tokens_[close + 1].type; + return next == TokenType::LeftBrace || next == TokenType::Newline; } std::string Parser::unknownTypeMessage(const std::string& name) { return "unknown type '" + name + - "' (the types are: i8 i16 i32 i64 u8 u16 u32 u64 int byte f32 f64 float bool char string list map " - "func any)"; + "' (the types are: i8 i16 i32 i64 u8 u16 u32 u64 int byte short long f32 f64 float double bool char string " + "list map func any, and the structs you declare)"; } // type name = value or type name (which starts at the type's zero value) -StmtPtr Parser::typedDeclaration() { +// Several at once, C style: i32 a = 1, b = 2. auto name = value takes the +// value's type. +StmtPtr Parser::typedDeclaration(bool requireEnd) { const Token& start = peek(); - auto stmt = nodeAt(start); - stmt->type = parseType(false); - stmt->hasType = true; - const Token& name = consume(TokenType::Identifier, "a variable name after the type"); - stmt->name = name.lexeme; - if (check(TokenType::Declare)) { - error(peek(), "use '=' in a typed declaration, like '" + stmt->type.name() + " " + name.lexeme + " = ...'"); - } - if (match(TokenType::Equal)) { - skipNewlines(); - stmt->value = expression(); + bool inferType = isWord(0, "auto"); + TypeSpec type; + if (inferType) { + advance(); + } else { + type = parseType(false); } - expectStatementEnd(); - return stmt; + std::vector declarations; + do { + const Token& name = consume(TokenType::Identifier, "a variable name after the type"); + auto stmt = nodeAt(declarations.empty() ? start : name); + stmt->type = type; + stmt->hasType = !inferType; + stmt->inferType = inferType; + stmt->name = name.lexeme; + if (check(TokenType::Declare)) { + error(peek(), "use '=' in a typed declaration, like '" + (inferType ? "auto" : type.name()) + " " + + name.lexeme + " = ...'"); + } + if (match(TokenType::Equal)) { + skipNewlines(); + stmt->value = expression(); + } else if (inferType) { + error(peek(), "'auto " + name.lexeme + "' needs a value to take its type from, like 'auto " + name.lexeme + + " = 0'"); + } + declarations.push_back(std::move(stmt)); + } while (match(TokenType::Comma)); + if (requireEnd) expectStatementEnd(); + if (declarations.size() == 1) return std::move(declarations[0]); + auto group = nodeAt(start); + group->statements = std::move(declarations); + return group; } // A function with its return type in front, C style: `i32 add(i32 a, i32 b) { ... }` @@ -342,8 +945,11 @@ StmtPtr Parser::typedFunctionDeclaration() { StmtPtr Parser::constDeclaration(const Token& keyword) { auto stmt = nodeAt(keyword); stmt->constant = true; - size_t length = typeLength(0); - if (length > 0 && peek(length).type == TokenType::Identifier) { + size_t length = typeLength(0, true); + if (isWord(0, "auto") && peek(1).type == TokenType::Identifier) { + advance(); + stmt->inferType = true; + } else if (length > 0 && peek(length).type == TokenType::Identifier) { stmt->type = parseType(false); stmt->hasType = true; } @@ -386,6 +992,8 @@ StmtPtr Parser::matchStatement(const Token& keyword) { if (sawDefault) error(defaultToken, "a match can only have one default ('else') arm"); sawDefault = true; } else { + // `x => ...` here is a pattern and its arm, not a lambda. + FlagGuard patterns(noLambda_, true); do { arm.patterns.push_back(expression()); } while (match(TokenType::Comma)); @@ -413,8 +1021,9 @@ StmtPtr Parser::matchStatement(const Token& keyword) { return stmt; } -// A type name like `i32`, `string` or `u8?` (the ? allows null). -TypeSpec Parser::parseType(bool allowVoid) { +// A type name like `i32`, `string`, `u8?` (the ? allows null) or the name of +// a struct. +TypeSpec Parser::parseType(bool allowVoid, bool allowStruct) { const Token& token = peek(); std::string name; if (token.type == TokenType::Identifier) { @@ -425,9 +1034,32 @@ TypeSpec Parser::parseType(bool allowVoid) { error(token, "expected a type name (like i32, f64, string or bool), found " + describeToken(token)); } TypeSpec type; - if (!lookupType(name, type)) error(token, unknownTypeMessage(name)); + if (name == "auto") error(token, "'auto' can only be used to declare a variable with a value"); + if (!lookupType(name, type)) { + if (!allowStruct) error(token, unknownTypeMessage(name)); + type = structType(name); // checked once the whole program is parsed + } if (type.kind == TypeKind::Void && !allowVoid) error(token, "'void' can only be used as a return type"); advance(); + // list and map: the types of the elements. + if ((type.kind == TypeKind::List || type.kind == TypeKind::Map) && match(TokenType::Less)) { + if (type.kind == TypeKind::Map) { + type.key = std::make_shared(parseType(false, true)); + consume(TokenType::Comma, "',' between the key and value types, like map"); + } + type.element = std::make_shared(parseType(false, true)); + if (check(TokenType::ShiftRight)) { + // The '>>' in list> closes two lists: use one '>' here, leave one. + Token& shift = tokens_[current_]; + shift.type = TokenType::Greater; + shift.lexeme = ">"; + ++shift.column; + } else { + consume(TokenType::Greater, type.kind == TypeKind::Map ? "'>' after the value type" : "'>' after the element type"); + } + } else if (type.kind == TypeKind::Map && check(TokenType::Less)) { + error(peek(), "write map types like map"); + } if (match(TokenType::Question)) { if (type.kind == TypeKind::Any || type.kind == TypeKind::Void) { error(previous(), "'" + name + "' can't be made nullable with '?'"); @@ -449,38 +1081,85 @@ StmtPtr Parser::functionDeclaration(const Token& keyword) { std::unique_ptr Parser::functionRest(const Token& keyword, const std::string& name, const TypeSpec* returnType) { auto function = nodeAt(keyword); + ++functionCount_; function->name = name; if (returnType != nullptr) function->returnType = *returnType; consume(TokenType::LeftParen, name.empty() ? "'(' after 'func'" : "'(' after the function name"); + parameters(*function); + if (check(TokenType::Arrow)) { + if (returnType != nullptr) error(peek(), "the return type is already written before the function name"); + advance(); + function->returnType = parseType(true); + } + function->body = block("function body"); + return function; +} + +// The parameters after '(' up to and including ')'. A typed parameter is +// written C style, `i32 count`; `...rest` collects any further arguments. +void Parser::parameters(FunctionExpr& function) { bool sawDefault = false; while (!check(TokenType::RightParen)) { Param parameter; - size_t length = typeLength(0); - if (length > 0 && peek(length).type == TokenType::Identifier) parameter.type = parseType(false); + size_t length = typeLength(0, true); + if (length > 0 && (peek(length).type == TokenType::Identifier || peek(length).type == TokenType::Ellipsis)) { + parameter.type = parseType(false); + } + bool rest = match(TokenType::Ellipsis); const Token& param = consume(TokenType::Identifier, "a parameter name"); parameter.name = param.lexeme; if (check(TokenType::Colon)) { std::string type = typeLength(1) > 0 ? peek(1).lexeme : "i32"; error(peek(), "write the parameter's type before its name, like '" + type + " " + param.lexeme + "'"); } + if (rest) { + if (check(TokenType::Equal)) error(peek(), "'..." + param.lexeme + "' can't have a default value"); + function.variadic = true; + function.params.push_back(std::move(parameter)); + match(TokenType::Comma); + if (!check(TokenType::RightParen)) { + error(peek(), "'..." + param.lexeme + "' collects the remaining arguments, so it must be the last parameter"); + } + break; + } if (match(TokenType::Equal)) { parameter.defaultValue = expression(); sawDefault = true; } else if (sawDefault) { error(param, "parameter '" + param.lexeme + "' needs a default value because an earlier parameter has one"); } else { - ++function->requiredParams; + ++function.requiredParams; } - function->params.push_back(std::move(parameter)); + function.params.push_back(std::move(parameter)); if (!match(TokenType::Comma)) break; } consume(TokenType::RightParen, "')' after the parameters"); - if (check(TokenType::Arrow)) { - if (returnType != nullptr) error(peek(), "the return type is already written before the function name"); - advance(); - function->returnType = parseType(true); +} + +// x => x * 2 (a, b) => a + b (i32 n) => { return n * n } +ExprPtr Parser::lambda(const Token& start, bool parenthesized) { + auto function = nodeAt(start); + ++functionCount_; + NestingGuard guard(*this, start); + if (parenthesized) { + advance(); // ( + parameters(*function); + } else { + Param parameter; + parameter.name = advance().lexeme; + function->params.push_back(std::move(parameter)); + function->requiredParams = 1; + } + consume(TokenType::FatArrow, "'=>' after the lambda's parameters"); + FlagGuard lambdas(noLambda_, false); + if (check(TokenType::LeftBrace)) { + function->body = block("lambda body"); + } else { + skipNewlines(); + auto result = std::make_unique(peek().line, peek().column); + result->value = expression(); + function->body.statements.push_back(std::move(result)); } - function->body = block("function body"); return function; } @@ -502,16 +1181,103 @@ Block Parser::block(const std::string& what) { return result; } +// The body of an if, else, while, for or do: a { block }, or C style a single +// statement (on the same line or the next), or `;` for nothing. +Block Parser::body(const std::string& what) { + if (match(TokenType::Semicolon)) return Block(); // while (busy()); + skipNewlines(); + if (check(TokenType::LeftBrace)) return block(what); + if (isAtEnd() || check(TokenType::RightBrace) || check(TokenType::Else)) { + error(peek(), "expected '{' to start the " + what + ", found " + describeToken(peek())); + } + NestingGuard guard(*this, peek()); + Block result; + StmtPtr stmt = statement(); + noteStatement(result, *stmt); + result.statements.push_back(std::move(stmt)); + return result; +} + +// The condition of an if, while, switch or match. When it's in parentheses, +// the closing ')' ends it, so the body can follow on the same line without +// braces: `if (x > 0) print(x)`. ExprPtr Parser::condition(const std::string& keyword) { if (check(TokenType::LeftBrace) || check(TokenType::Newline) || isAtEnd()) { error(peek(), "expected an expression after '" + keyword + "', found " + describeToken(peek())); } + if (check(TokenType::LeftParen)) { + size_t close = matchingParen(current_); + if (close != std::string::npos && !continuesExpression(tokens_[close + 1].type)) { + const Token& open = advance(); + if (check(TokenType::RightParen)) error(peek(), "expected a condition between '(' and ')'"); + ExprPtr inner = expression(); + consume(TokenType::RightParen, "')' to close the '(' on line " + std::to_string(open.line)); + return inner; + } + } return expression(); } +// The index of the ')' matching the '(' at `open`, or npos. +size_t Parser::matchingParen(size_t open) const { + int depth = 0; + for (size_t i = open; i < tokens_.size(); ++i) { + TokenType type = tokens_[i].type; + if (type == TokenType::LeftParen) ++depth; + if (type == TokenType::RightParen && --depth == 0) return i; + if (type == TokenType::Eof) break; + } + return std::string::npos; +} + +bool Parser::hasTopLevelSemicolon(size_t from, size_t to) const { + int depth = 0; + for (size_t i = from; i < to && i < tokens_.size(); ++i) { + TokenType type = tokens_[i].type; + if (type == TokenType::LeftParen || type == TokenType::LeftBracket || type == TokenType::LeftBrace) ++depth; + if (type == TokenType::RightParen || type == TokenType::RightBracket || type == TokenType::RightBrace) --depth; + if (type == TokenType::Semicolon && depth == 0) return true; + } + return false; +} + +bool Parser::isWord(size_t ahead, const char* word) const { + const Token& token = peek(ahead); + return token.type == TokenType::Identifier && token.lexeme == word; +} + +// from import ... +bool Parser::looksLikeFromImport() const { + size_t i = 1; + if (peek(i).type == TokenType::String) { + ++i; + } else if (peek(i).type == TokenType::Identifier) { + ++i; + while (peek(i).type == TokenType::Dot && peek(i + 1).type == TokenType::Identifier) i += 2; + } else { + return false; + } + return isWord(i, "import"); +} + +// switch (x) { or switch x { (and not a call to a function named switch) +bool Parser::looksLikeSwitch() const { + TokenType next = peek(1).type; + if (next == TokenType::LeftParen) { + size_t close = matchingParen(current_ + 1); + if (close == std::string::npos) return false; + size_t i = close + 1; + while (i + 1 < tokens_.size() && tokens_[i].type == TokenType::Newline) ++i; + return tokens_[i].type == TokenType::LeftBrace; + } + return startsOperand(next) && next != TokenType::Minus && next != TokenType::PlusPlus && + next != TokenType::MinusMinus && next != TokenType::LeftBracket; +} + void Parser::expectStatementEnd() { if (match(TokenType::Newline) || match(TokenType::Semicolon)) return; if (check(TokenType::RightBrace) || isAtEnd()) return; + if (check(TokenType::Else)) return; // if (x) a() else b() error(peek(), "expected a newline or ';' after the statement, found " + describeToken(peek())); } @@ -567,7 +1333,7 @@ ExprPtr Parser::expression() { } ExprPtr Parser::ternary() { - ExprPtr condition = logicOr(); + ExprPtr condition = nullish(); if (!check(TokenType::Question)) return condition; const Token& question = advance(); auto expr = nodeAt(question); @@ -584,6 +1350,21 @@ ExprPtr Parser::ternary() { return expr; } +// a ?? b: a, unless it's null. +ExprPtr Parser::nullish() { + ExprPtr left = logicOr(); + while (check(TokenType::QuestionQuestion)) { + const Token& op = advance(); + skipNewlines(); + auto expr = nodeAt(op); + expr->op = TokenType::QuestionQuestion; + expr->left = std::move(left); + expr->right = logicOr(); + left = std::move(expr); + } + return left; +} + ExprPtr Parser::logicOr() { ExprPtr left = logicAnd(); while (check(TokenType::Or)) { @@ -643,10 +1424,24 @@ ExprPtr Parser::equality() { } ExprPtr Parser::comparison() { - return binary(&Parser::bitOr, {TokenType::Less, TokenType::LessEqual, TokenType::Greater, + return binary(&Parser::range, {TokenType::Less, TokenType::LessEqual, TokenType::Greater, TokenType::GreaterEqual, TokenType::In}); } +// 0..10 (10 not included) and 1..=10 (included). Binds looser than +// arithmetic, so 0..n + 1 is 0..(n + 1). +ExprPtr Parser::range() { + ExprPtr start = bitOr(); + if (!check(TokenType::DotDot) && !check(TokenType::DotDotEqual)) return start; + const Token& op = advance(); + skipNewlines(); + auto expr = nodeAt(op); + expr->inclusive = op.type == TokenType::DotDotEqual; + expr->start = std::move(start); + expr->end = bitOr(); + return expr; +} + // Bitwise operators bind tighter than comparisons (unlike C), so // `flags & 1 == 0` means `(flags & 1) == 0`. ExprPtr Parser::bitOr() { return binary(&Parser::bitXor, {TokenType::Pipe}); } @@ -687,6 +1482,38 @@ ExprPtr Parser::unary() { expr->operand = unary(); return expr; } + if (check(TokenType::PlusPlus) || check(TokenType::MinusMinus)) { + const Token& op = advance(); + NestingGuard guard(*this, op); + auto expr = nodeAt(op); + expr->increment = op.type == TokenType::PlusPlus; + expr->prefix = true; + expr->target = unary(); + if (!isAssignable(*expr->target)) { + error(op, "'" + op.lexeme + "' needs a variable, element or field to change"); + } + return expr; + } + // A C-style cast: (i32) x, (f64) total / count. It binds like a unary operator. + if (check(TokenType::LeftParen) && peek(1).type == TokenType::Identifier && peek(2).type == TokenType::RightParen && + startsOperand(peek(3).type)) { + TypeSpec type; + bool castable = lookupType(peek(1).lexeme, type) && + (type.kind == TypeKind::Number || type.kind == TypeKind::Bool || type.kind == TypeKind::Char || + type.kind == TypeKind::String); + if (castable) { + const Token& open = advance(); + const Token& name = advance(); + advance(); + NestingGuard guard(*this, open); + auto callee = nodeAt(name); + callee->name = name.lexeme; + auto call = nodeAt(open); + call->callee = std::move(callee); + call->args.push_back(unary()); + return call; + } + } return power(); } @@ -720,23 +1547,95 @@ ExprPtr Parser::postfix() { member->object = std::move(expr); member->name = name.lexeme; expr = std::move(member); + } else if (check(TokenType::QuestionDot)) { + advance(); + const Token& name = consume(TokenType::Identifier, "a field or method name after '?.'"); + auto member = nodeAt(name); + member->object = std::move(expr); + member->name = name.lexeme; + member->optional = true; + expr = std::move(member); + } else if ((check(TokenType::PlusPlus) || check(TokenType::MinusMinus)) && isAssignable(*expr)) { + const Token& op = advance(); + auto incdec = std::make_unique(expr->line, expr->column); + incdec->increment = op.type == TokenType::PlusPlus; + incdec->target = std::move(expr); + return incdec; } else { return expr; } } } +// Arguments: positional ones, then named ones (`size: 10`); `...items` +// spreads a list into positional arguments. ExprPtr Parser::finishCall(ExprPtr callee) { auto call = std::make_unique(callee->line, callee->column); call->callee = std::move(callee); + FlagGuard lambdas(noLambda_, false); while (!check(TokenType::RightParen)) { - call->args.push_back(expression()); + if (check(TokenType::Identifier) && peek(1).type == TokenType::Colon) { + const Token& name = advance(); + advance(); + if (std::find(call->argNames.begin(), call->argNames.end(), name.lexeme) != call->argNames.end()) { + error(name, "the argument '" + name.lexeme + "' is given twice"); + } + call->hasNamedArgs = true; + call->argNames.push_back(name.lexeme); + call->args.push_back(expression()); + } else { + if (call->hasNamedArgs) error(peek(), "positional arguments must come before named ones"); + call->argNames.emplace_back(); + call->args.push_back(spreadOrExpression()); + } if (!match(TokenType::Comma)) break; } consume(TokenType::RightParen, "')' after the arguments"); return call; } +ExprPtr Parser::spreadOrExpression() { + if (!check(TokenType::Ellipsis)) return expression(); + const Token& dots = advance(); + auto spread = nodeAt(dots); + spread->value = expression(); + return spread; +} + +// The rest of [value for x in items if test] or {key: value for ...}. +ExprPtr Parser::comprehension(const Token& open, ExprPtr key, ExprPtr value, TokenType closing) { + auto expr = nodeAt(open); + expr->isMap = key != nullptr; + expr->key = std::move(key); + expr->value = std::move(value); + while (true) { + skipNewlines(); + if (match(TokenType::For)) { + ComprehensionClause clause; + clause.first = consume(TokenType::Identifier, "a loop variable name after 'for'").lexeme; + if (match(TokenType::Comma)) { + clause.second = consume(TokenType::Identifier, "a second loop variable name after ','").lexeme; + } + if (!match(TokenType::In) && !match(TokenType::Colon)) { + error(peek(), "expected 'in' after the loop variable, found " + describeToken(peek())); + } + clause.expr = expression(); + expr->clauses.push_back(std::move(clause)); + } else if (match(TokenType::If)) { + ComprehensionClause clause; + clause.isFor = false; + clause.expr = expression(); + expr->clauses.push_back(std::move(clause)); + } else { + break; + } + } + skipNewlines(); + consume(closing, closing == TokenType::RightBracket ? "']' to close the list comprehension" + : "'}' to close the map comprehension"); + return expr; +} + ExprPtr Parser::finishIndex(ExprPtr object, const Token& bracket) { ExprPtr start; if (!check(TokenType::Colon)) { @@ -792,12 +1691,19 @@ ExprPtr Parser::primary() { return nodeAt(token); } case TokenType::Identifier: { + if (!noLambda_ && peek(1).type == TokenType::FatArrow) return lambda(token, false); advance(); auto variable = nodeAt(token); variable->name = token.lexeme; return variable; } case TokenType::LeftParen: { + if (!noLambda_) { + size_t close = matchingParen(current_); + if (close != std::string::npos && tokens_[close + 1].type == TokenType::FatArrow) { + return lambda(token, true); + } + } advance(); ExprPtr inner = expression(); consume(TokenType::RightParen, "')' to close the '(' on line " + std::to_string(token.line)); @@ -823,8 +1729,13 @@ ExprPtr Parser::primary() { ExprPtr Parser::listLiteral(const Token& bracket) { auto list = nodeAt(bracket); + FlagGuard lambdas(noLambda_, false); while (!check(TokenType::RightBracket)) { - list->elements.push_back(expression()); + ExprPtr element = spreadOrExpression(); + if (list->elements.empty() && check(TokenType::For) && element->kind != ExprKind::Spread) { + return comprehension(bracket, nullptr, std::move(element), TokenType::RightBracket); + } + list->elements.push_back(std::move(element)); if (!match(TokenType::Comma)) break; } consume(TokenType::RightBracket, "']' to close the list that starts on line " + std::to_string(bracket.line)); @@ -837,9 +1748,18 @@ ExprPtr Parser::mapLiteral(const Token& brace) { auto map = nodeAt(brace); skipNewlines(); while (!check(TokenType::RightBrace)) { + if (check(TokenType::Ellipsis)) { // {...defaults, size: 2} + map->entries.emplace_back(spreadOrExpression(), nullptr); + skipNewlines(); + if (!match(TokenType::Comma)) break; + skipNewlines(); + continue; + } ExprPtr key; + const Token* bareName = nullptr; if (check(TokenType::Identifier) && peek(1).type == TokenType::Colon) { const Token& name = advance(); + bareName = &name; auto literal = nodeAt(name); literal->value = Value::fromString(name.lexeme); key = std::move(literal); @@ -850,6 +1770,15 @@ ExprPtr Parser::mapLiteral(const Token& brace) { consume(TokenType::Colon, "':' after the map key"); skipNewlines(); ExprPtr value = expression(); + if (map->entries.empty() && peekPastNewlines().type == TokenType::For) { + // In {k: v for k, v in pairs}, a bare name is the loop variable, not a string key. + if (bareName != nullptr) { + auto variable = nodeAt(*bareName); + variable->name = bareName->lexeme; + key = std::move(variable); + } + return comprehension(brace, std::move(key), std::move(value), TokenType::RightBrace); + } map->entries.emplace_back(std::move(key), std::move(value)); skipNewlines(); if (!match(TokenType::Comma)) break; diff --git a/src/process.cpp b/src/process.cpp index 932147b..266dd03 100644 --- a/src/process.cpp +++ b/src/process.cpp @@ -3,12 +3,17 @@ #include #include #include +#include +#include #include +#include +#include #if defined(_WIN32) #include #include #include +#include #else #include #include @@ -115,6 +120,129 @@ int runProcess(const std::vector& args, bool quiet) { #endif } +namespace { + +// Three temporary files for a child's standard input, output and error. +struct TempFiles { + std::filesystem::path in; + std::filesystem::path out; + std::filesystem::path err; + + explicit TempFiles(const std::string& input) { + std::random_device device; + std::filesystem::path dir = std::filesystem::temp_directory_path(); + std::string base = "ascend-" + std::to_string(device()) + "-" + std::to_string(device()); + in = dir / (base + ".in"); + out = dir / (base + ".out"); + err = dir / (base + ".err"); + std::ofstream(in, std::ios::binary) << input; + } + ~TempFiles() { + std::error_code ignored; + std::filesystem::remove(in, ignored); + std::filesystem::remove(out, ignored); + std::filesystem::remove(err, ignored); + } + + void read(CapturedOutput& result) const { + result.out = contents(out); + result.err = contents(err); + } + + static std::string contents(const std::filesystem::path& path) { + std::ifstream file(path, std::ios::binary); + return std::string(std::istreambuf_iterator(file), std::istreambuf_iterator()); + } +}; + +} // namespace + +bool captureProcess(const std::vector& args, const std::string& input, CapturedOutput& result) { + if (args.empty()) return false; + std::cout.flush(); + std::cerr.flush(); + TempFiles files(input); +#if defined(_WIN32) + std::vector quoted; + for (const auto& arg : args) quoted.push_back(quoteArgument(arg)); + std::vector argv; + for (const auto& arg : quoted) argv.push_back(arg.c_str()); + argv.push_back(nullptr); + + std::fflush(stdout); + std::fflush(stderr); + int inFile = _open(files.in.string().c_str(), _O_RDONLY | _O_BINARY); + int outFile = _open(files.out.string().c_str(), _O_WRONLY | _O_CREAT | _O_TRUNC | _O_BINARY, _S_IREAD | _S_IWRITE); + int errFile = _open(files.err.string().c_str(), _O_WRONLY | _O_CREAT | _O_TRUNC | _O_BINARY, _S_IREAD | _S_IWRITE); + if (inFile == -1 || outFile == -1 || errFile == -1) { + if (inFile != -1) _close(inFile); + if (outFile != -1) _close(outFile); + if (errFile != -1) _close(errFile); + return false; + } + int saved[3] = {_dup(0), _dup(1), _dup(2)}; + _dup2(inFile, 0); + _dup2(outFile, 1); + _dup2(errFile, 2); + intptr_t code = _spawnvp(_P_WAIT, args[0].c_str(), argv.data()); + for (int fd = 0; fd < 3; ++fd) { + if (saved[fd] != -1) { + _dup2(saved[fd], fd); + _close(saved[fd]); + } + } + _close(inFile); + _close(outFile); + _close(errFile); + if (code == -1) return false; + result.code = static_cast(code); +#else + std::vector argv; + for (const auto& arg : args) argv.push_back(const_cast(arg.c_str())); + argv.push_back(nullptr); + std::string inName = files.in.string(); + std::string outName = files.out.string(); + std::string errName = files.err.string(); + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_addopen(&actions, 0, inName.c_str(), O_RDONLY, 0); + posix_spawn_file_actions_addopen(&actions, 1, outName.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + posix_spawn_file_actions_addopen(&actions, 2, errName.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + pid_t pid; + int spawned = posix_spawnp(&pid, argv[0], &actions, nullptr, argv.data(), environ); + posix_spawn_file_actions_destroy(&actions); + if (spawned != 0) return false; + int status = 0; + while (waitpid(pid, &status, 0) == -1) { + if (errno != EINTR) return false; + } + result.code = WIFEXITED(status) ? WEXITSTATUS(status) : 128 + (WIFSIGNALED(status) ? WTERMSIG(status) : 0); +#endif + files.read(result); + return true; +} + +bool captureShell(const std::string& command, const std::string& input, CapturedOutput& result) { + std::cout.flush(); + std::cerr.flush(); + TempFiles files(input); +#if defined(_WIN32) + std::string line = "(" + command + ") <\"" + files.in.string() + "\" >\"" + files.out.string() + "\" 2>\"" + + files.err.string() + "\""; + int status = std::system(line.c_str()); + if (status == -1) return false; + result.code = status; +#else + std::string line = "(" + command + ") <'" + files.in.string() + "' >'" + files.out.string() + "' 2>'" + + files.err.string() + "'"; + int status = std::system(line.c_str()); + if (status == -1) return false; + result.code = WIFEXITED(status) ? WEXITSTATUS(status) : 128 + (WIFSIGNALED(status) ? WTERMSIG(status) : 0); +#endif + files.read(result); + return true; +} + std::string environmentVariable(const std::string& name) { #if defined(_WIN32) char* value = nullptr; diff --git a/src/runner.cpp b/src/runner.cpp index d636ec2..27cf195 100644 --- a/src/runner.cpp +++ b/src/runner.cpp @@ -10,6 +10,44 @@ namespace ascend { +std::string formatTraceback(const std::vector& frames, const std::string& mainFile) { + if (frames.empty()) return ""; + auto describe = [&](const TraceFrame& frame) { + std::string text = " in " + frame.function; + if (frame.line > 0) { + text += ", called from " + (frame.source != nullptr ? frame.source->name : mainFile) + ":" + + std::to_string(frame.line) + ":" + std::to_string(frame.column); + } else if (frame.line == 0) { + text += ", called by a builtin"; + } + return text + "\n"; + }; + // Repeated calls (deep recursion) are shown once with a count, and very + // long tracebacks keep their start and end. + std::vector> lines; + for (const auto& frame : frames) { + std::string line = describe(frame); + if (!lines.empty() && lines.back().first == line) { + ++lines.back().second; + } else { + lines.emplace_back(line, 1); + } + } + std::string out; + const size_t head = 15; + const size_t tail = 5; + for (size_t i = 0; i < lines.size(); ++i) { + if (lines.size() > head + tail && i == head) { + out += " ... " + std::to_string(lines.size() - head - tail) + " more ...\n"; + i = lines.size() - tail - 1; + continue; + } + out += lines[i].first; + if (lines[i].second > 1) out += " (and " + std::to_string(lines[i].second - 1) + " more times)\n"; + } + return out; +} + void prepareSource(std::string& text) { if (text.compare(0, 3, "\xEF\xBB\xBF") == 0) text.erase(0, 3); // UTF-8 byte order mark if (text.compare(0, 2, "#!") == 0) text.replace(0, 2, "//"); // allow #!/usr/bin/env ascend @@ -24,7 +62,8 @@ RunResult runSource(Interpreter& interpreter, const std::string& source, const R } err.flush(); }; - // Runtime errors may come from an imported file. + // Runtime errors may come from an imported file. After the message come + // the calls the error passed through, innermost first. auto reportAt = [&](const SourceFile* file, int line, int column, const std::string& message) { interpreter.out().flush(); if (file != nullptr) { @@ -32,6 +71,7 @@ RunResult runSource(Interpreter& interpreter, const std::string& source, const R } else { err << formatDiagnostic(source, options.filename, line, column, "runtime error", message); } + err << formatTraceback(interpreter.traceback(), options.filename); err.flush(); }; @@ -54,6 +94,7 @@ RunResult runSource(Interpreter& interpreter, const std::string& source, const R } SemanticAnalyzer analyzer; + analyzer.checkTypeNames = !options.repl; std::vector problems = analyzer.analyze(*program); if (!problems.empty()) { report(problems, "error"); @@ -61,11 +102,21 @@ RunResult runSource(Interpreter& interpreter, const std::string& source, const R } if (options.checkOnly) return {RunStatus::Ok, 0}; + // Like C, a program with a main() function runs it after the top-level + // code, unless the program calls it itself. + const bool callMain = !options.repl && analyzer.shouldCallMain(); auto file = std::make_unique(); file->name = options.filename; file->text = source; try { interpreter.execute(std::move(program), std::move(file)); + if (callMain) { + int code = interpreter.runMain(); + if (code != 0) { + interpreter.out().flush(); + return {RunStatus::Exited, code}; + } + } } catch (const RuntimeError& error) { reportAt(error.source, error.line, error.column, error.what()); return {RunStatus::RuntimeError, kExitRuntimeError}; diff --git a/src/semantic_analyzer.cpp b/src/semantic_analyzer.cpp index 6b11d71..2086d29 100644 --- a/src/semantic_analyzer.cpp +++ b/src/semantic_analyzer.cpp @@ -1,5 +1,7 @@ #include "semantic_analyzer.hpp" +#include +#include #include namespace ascend { @@ -7,14 +9,45 @@ namespace ascend { std::vector SemanticAnalyzer::analyze(const Program& program) { diagnostics_.clear(); loopDepth_ = 0; + switchDepth_ = 0; functions_.clear(); + typeUses_.clear(); + declared_.clear(); + importsEverything_ = false; + hasMain_ = false; + mentionsMain_ = false; + for (const auto& stmt : program.body.statements) { + if (stmt->kind == StmtKind::Function && stmt->as().function->name == "main") hasMain_ = true; + } visitBlock(program.body); + + // Struct types can be declared after they're used, or imported, so + // unknown type names are only reported now. + if (checkTypeNames && !importsEverything_) { + for (const auto& use : typeUses_) { + if (declared_.count(use.name) != 0) continue; + report(use.line, use.column, + "unknown type '" + use.name + + "' (the types are: i8 i16 i32 i64 u8 u16 u32 u64 int byte short long f32 f64 float double bool " + "char string list map func any, and the structs you declare)"); + } + } + std::stable_sort(diagnostics_.begin(), diagnostics_.end(), [](const Diagnostic& a, const Diagnostic& b) { + return a.line != b.line ? a.line < b.line : a.column < b.column; + }); return diagnostics_; } +void SemanticAnalyzer::noteType(const TypeSpec& type, int line, int column) { + if (type.kind == TypeKind::Struct) typeUses_.push_back({type.structName, line, column}); + if (type.element) noteType(*type.element, line, column); + if (type.key) noteType(*type.key, line, column); +} + void SemanticAnalyzer::checkType(const Expr* expr, const TypeSpec& type, const std::function& describe) { - if (expr == nullptr || type.isAny()) return; + // Struct values are only made at run time (and the type may be unknown). + if (expr == nullptr || type.isAny() || type.kind == TypeKind::Struct) return; CoerceFailure failure; Value value; switch (expr->kind) { @@ -25,8 +58,37 @@ void SemanticAnalyzer::checkType(const Expr* expr, const TypeSpec& type, } return; } - case ExprKind::List: value = Value::fromList(std::make_shared()); break; - case ExprKind::Map: value = Value::fromMap(std::make_shared()); break; + case ExprKind::List: + // list xs = [1, 300]: check each element that's known now. + if (type.kind == TypeKind::List && type.element) { + for (const auto& element : expr->as().elements) { + if (element->kind == ExprKind::Spread) continue; + checkType(element.get(), *type.element, [&](const CoerceFailure& inner) { + CoerceFailure outer; + outer.what = "a list containing " + inner.what; + outer.hint = inner.hint; + return describe(outer); + }); + } + return; + } + value = Value::fromList(std::make_shared()); + break; + case ExprKind::Map: + if (type.kind == TypeKind::Map && type.element) { + for (const auto& entry : expr->as().entries) { + if (!entry.second) continue; + checkType(entry.second.get(), *type.element, [&](const CoerceFailure& inner) { + CoerceFailure outer; + outer.what = "a map containing " + inner.what; + outer.hint = inner.hint; + return describe(outer); + }); + } + return; + } + value = Value::fromMap(std::make_shared()); + break; case ExprKind::FString: value = Value::fromString(""); break; case ExprKind::Function: { if (type.kind == TypeKind::Function) return; @@ -56,7 +118,9 @@ void SemanticAnalyzer::visitStmt(const Stmt& stmt) { break; case StmtKind::Declare: { const auto& decl = stmt.as(); + declared_.insert(decl.name); visitExpr(decl.value.get()); + noteType(decl.type, stmt.line, stmt.column); if (decl.hasType) { if (!decl.value && decl.type.kind == TypeKind::Function && !decl.type.nullable) { report(stmt.line, stmt.column, @@ -99,14 +163,125 @@ void SemanticAnalyzer::visitStmt(const Stmt& stmt) { if (!loop.second.empty() && loop.first == loop.second) { report(stmt.line, stmt.column, "both loop variables are named '" + loop.first + "'"); } + noteType(loop.firstType, stmt.line, stmt.column); + noteType(loop.secondType, stmt.line, stmt.column); visitExpr(loop.iterable.get()); ++loopDepth_; visitBlock(loop.body); --loopDepth_; break; } + case StmtKind::CFor: { + const auto& loop = stmt.as(); + for (const auto& init : loop.init) visitStmt(*init); + visitExpr(loop.condition.get()); + for (const auto& step : loop.step) visitStmt(*step); + ++loopDepth_; + visitBlock(loop.body); + --loopDepth_; + break; + } + case StmtKind::Switch: { + const auto& choice = stmt.as(); + visitExpr(choice.subject.get()); + for (const auto& entry : choice.cases) { + for (const auto& value : entry.values) visitExpr(value.get()); + } + ++switchDepth_; + visitBlock(choice.body); + --switchDepth_; + break; + } + case StmtKind::Import: { + const auto& import = stmt.as(); + if (import.all) importsEverything_ = true; + for (const auto& name : import.names) declared_.insert(name.binding); + break; + } + case StmtKind::Struct: { + const auto& declaration = stmt.as(); + declared_.insert(declaration.name); + if (!declaration.parent.empty()) typeUses_.push_back({declaration.parent, stmt.line, stmt.column}); + std::set seen; + for (const auto& field : declaration.fields) { + if (!seen.insert(field.name).second) { + report(field.line, field.column, + "struct " + declaration.name + " has two fields named '" + field.name + "'"); + } + noteType(field.type, field.line, field.column); + visitExpr(field.defaultValue.get()); + if (!field.defaultValue && field.type.kind == TypeKind::Function && !field.type.nullable) { + report(field.line, field.column, + "field '" + field.name + "' is declared as func, so it needs a default value (or declare it " + "as func?)"); + } + checkType(field.defaultValue.get(), field.type, [&](const CoerceFailure& failure) { + return withHint("field '" + field.name + "' is declared as " + field.type.name() + + ", so its default can't be " + failure.what, + failure); + }); + } + for (const auto& method : declaration.methods) { + const FunctionExpr& function = *method.second; + if (!seen.insert(method.first).second) { + report(function.line, function.column, + "struct " + declaration.name + " already has a field or method named '" + method.first + "'"); + } + bool isOperator = method.first.size() > 8 && method.first.compare(0, 8, "operator") == 0 && + !std::isalnum(static_cast(method.first[8])) && method.first[8] != '_'; + if (isOperator && function.params.size() != 2) { + report(function.line, function.column, + method.first + " takes one parameter: the value on the other side (or the index, for [])"); + } + if ((method.first == "to_string" || method.first == "iter" || method.first == "len") && + function.params.size() != 1) { + report(function.line, function.column, method.first + "() is called with no arguments, so it can't have parameters"); + } + if (method.first == "init" && function.params.size() > 1) { + report(function.line, function.column, + "init() is called with no arguments when a " + declaration.name + + " is made, so it can't have parameters"); + } + visitFunction(function); + } + break; + } + case StmtKind::Enum: { + const auto& declaration = stmt.as(); + declared_.insert(declaration.name); + std::set seen; + for (const auto& member : declaration.members) { + if (!seen.insert(member.name).second) { + report(member.line, member.column, + "enum " + declaration.name + " lists '" + member.name + "' more than once"); + } + visitExpr(member.value.get()); + } + break; + } + case StmtKind::Group: + for (const auto& inner : stmt.as().statements) visitStmt(*inner); + break; + case StmtKind::MultiAssign: { + const auto& assign = stmt.as(); + std::set seen; + for (const auto& target : assign.targets) { + visitExpr(target.get()); + if (target->kind == ExprKind::Variable) { + const std::string& name = target->as().name; + if (assign.declare) declared_.insert(name); + if (!seen.insert(name).second) { + report(target->line, target->column, "'" + name + "' is listed twice"); + } + } + } + for (const auto& value : assign.values) visitExpr(value.get()); + break; + } case StmtKind::Break: - if (loopDepth_ == 0) report(stmt.line, stmt.column, "'break' can only be used inside a loop"); + if (loopDepth_ == 0 && switchDepth_ == 0) { + report(stmt.line, stmt.column, "'break' can only be used inside a loop or switch"); + } break; case StmtKind::Continue: if (loopDepth_ == 0) report(stmt.line, stmt.column, "'continue' can only be used inside a loop"); @@ -133,12 +308,14 @@ void SemanticAnalyzer::visitStmt(const Stmt& stmt) { break; } case StmtKind::Function: + declared_.insert(stmt.as().function->name); visitFunction(*stmt.as().function); break; case StmtKind::Try: { const auto& tryStmt = stmt.as(); visitBlock(tryStmt.body); visitBlock(tryStmt.handler); + visitBlock(tryStmt.finallyBody); break; } case StmtKind::Throw: @@ -160,7 +337,26 @@ void SemanticAnalyzer::visitExpr(const Expr* expr) { if (expr == nullptr) return; switch (expr->kind) { case ExprKind::Literal: + break; case ExprKind::Variable: + if (expr->as().name == "main") mentionsMain_ = true; + break; + case ExprKind::IncDec: + visitExpr(expr->as().target.get()); + break; + case ExprKind::Comprehension: { + const auto& comprehension = expr->as(); + for (const auto& clause : comprehension.clauses) visitExpr(clause.expr.get()); + visitExpr(comprehension.key.get()); + visitExpr(comprehension.value.get()); + break; + } + case ExprKind::Range: + visitExpr(expr->as().start.get()); + visitExpr(expr->as().end.get()); + break; + case ExprKind::Spread: + visitExpr(expr->as().value.get()); break; case ExprKind::Unary: visitExpr(expr->as().operand.get()); @@ -220,10 +416,12 @@ void SemanticAnalyzer::visitExpr(const Expr* expr) { void SemanticAnalyzer::visitFunction(const FunctionExpr& function) { std::unordered_set seen; + noteType(function.returnType, function.line, function.column); for (const auto& param : function.params) { if (!seen.insert(param.name).second) { report(function.line, function.column, "parameter '" + param.name + "' is listed more than once"); } + noteType(param.type, function.line, function.column); visitExpr(param.defaultValue.get()); checkType(param.defaultValue.get(), param.type, [&](const CoerceFailure& failure) { return withHint("parameter '" + param.name + "' is declared as " + param.type.name() + @@ -233,11 +431,14 @@ void SemanticAnalyzer::visitFunction(const FunctionExpr& function) { } // A loop outside the function doesn't make `break` valid inside it. int savedLoopDepth = loopDepth_; + int savedSwitchDepth = switchDepth_; loopDepth_ = 0; + switchDepth_ = 0; functions_.push_back(&function); visitBlock(function.body); functions_.pop_back(); loopDepth_ = savedLoopDepth; + switchDepth_ = savedSwitchDepth; } } // namespace ascend diff --git a/src/stack_limit.cpp b/src/stack_limit.cpp index 2933e20..129f797 100644 --- a/src/stack_limit.cpp +++ b/src/stack_limit.cpp @@ -1,5 +1,7 @@ #include "stack_limit.hpp" +#include "big_stack.hpp" + #include #if defined(_WIN32) @@ -45,6 +47,8 @@ size_t availableStack() { uintptr_t here = stackPosition(); return here > low ? static_cast(here - low) : 0; #elif defined(__unix__) || defined(__APPLE__) + // A thread started with a large stack knows its size. + if (size_t known = knownThreadStackSize()) return known; // The main thread's stack can grow up to the soft limit; the interpreter // starts near the top of it. struct rlimit limit; diff --git a/src/stdlib.cpp b/src/stdlib.cpp index 8df00c8..5fef8fd 100644 --- a/src/stdlib.cpp +++ b/src/stdlib.cpp @@ -2,6 +2,7 @@ #include "interpreter.hpp" #include "numeric.hpp" +#include "typecheck.hpp" #include "utf8.hpp" #include @@ -12,9 +13,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -243,6 +246,202 @@ std::string padText(const Args& args, const char* fn, bool left) { return left ? padding + text : text + padding; } +// Structs always have all their fields, and enums never change. +void checkResizable(const MapObj& map, const char* fn) { + if (map.structInfo()) throw RuntimeError(std::string(fn) + "() can't remove fields from a " + map.structInfo()->name); + if (!map.frozenAs.empty()) throw RuntimeError(std::string(fn) + "(): the members of " + map.frozenAs + " can't be changed"); +} + +// printf("%-10s %5.2f\n", name, price): C format strings. +// %d %i %u integer %f %e %g float (%F %E %G) %x %X %o %b hex, octal, binary +// %c char (or a character code) %s any value as text %% a percent sign +// with C flags (- + space 0 #), width and precision, which may be * (taken +// from the arguments). Length modifiers like l and ll are accepted and ignored. +std::string cFormat(const char* fn, const Args& args) { + const std::string& format = stringArg(args, 0, fn); + const std::string name = fn; + size_t next = 1; + auto nextArg = [&](const std::string& conversion) -> const Value& { + if (next >= args.size()) { + throw RuntimeError(name + "(): the format has more conversions than values (no value for " + conversion + ")"); + } + return args[next++]; + }; + + std::string out; + for (size_t i = 0; i < format.size(); ++i) { + if (format[i] != '%') { + out += format[i]; + continue; + } + size_t start = i++; + if (i < format.size() && format[i] == '%') { + out += '%'; + continue; + } + bool left = false, plus = false, space = false, zero = false, alternate = false; + for (; i < format.size(); ++i) { + char flag = format[i]; + if (flag == '-') left = true; + else if (flag == '+') plus = true; + else if (flag == ' ') space = true; + else if (flag == '0') zero = true; + else if (flag == '#') alternate = true; + else break; + } + int64_t width = 0; + if (i < format.size() && format[i] == '*') { + const Value& value = nextArg("*"); + if (!value.isInt() || !toInt64(value, width)) throw RuntimeError(name + "(): a * width needs an integer"); + if (width < 0) { + left = true; + width = -width; + } + ++i; + } else { + while (i < format.size() && std::isdigit(static_cast(format[i]))) { + width = width * 10 + (format[i++] - '0'); + if (width > 10000) throw RuntimeError(name + "(): width is too large"); + } + } + int64_t precision = -1; + if (i < format.size() && format[i] == '.') { + ++i; + precision = 0; + if (i < format.size() && format[i] == '*') { + const Value& value = nextArg("*"); + if (!value.isInt() || !toInt64(value, precision)) throw RuntimeError(name + "(): a * precision needs an integer"); + if (precision < 0) precision = -1; + ++i; + } else { + while (i < format.size() && std::isdigit(static_cast(format[i]))) { + precision = precision * 10 + (format[i++] - '0'); + if (precision > 100) throw RuntimeError(name + "(): precision is too large"); + } + } + } + while (i < format.size() && std::string("hlLjzt").find(format[i]) != std::string::npos) ++i; + if (i >= format.size()) { + throw RuntimeError(name + "(): the format ends in the middle of '" + format.substr(start) + + "' (write %% for a percent sign)"); + } + const char conversion = format[i]; + const std::string written = format.substr(start, i - start + 1); + const Value& value = nextArg(written); + + std::string text; + bool numeric = true; + std::string prefix; + switch (conversion) { + case 'd': + case 'i': + case 'u': + case 'x': + case 'X': + case 'o': + case 'b': { + Value number = value; + if (value.isChar()) number = Value::fromInt(static_cast(value.asChar())); + if (value.isBool()) number = Value::fromInt(value.asBool() ? 1 : 0); + if (!number.isInt()) { + throw RuntimeError(name + "(): " + written + " needs an integer, but got " + + (value.isFloat() ? toString(value) + " (use %f for floats, or int(...) to drop the " + "fraction)" + : aTypeName(value))); + } + char type = conversion == 'i' || conversion == 'u' ? 'd' : conversion; + text = formatValue(number, std::string(plus ? "+" : space ? " " : "") + type); + if (alternate && !(number.isInt() && number.asInt() == 0)) { + if (type == 'x') prefix = "0x"; + if (type == 'X') prefix = "0X"; + if (type == 'o') prefix = "0"; + if (type == 'b') prefix = "0b"; + } + break; + } + case 'f': + case 'F': + case 'e': + case 'E': + case 'g': + case 'G': { + if (!value.isNumber()) { + throw RuntimeError(name + "(): " + written + " needs a number, but got " + aTypeName(value)); + } + std::string spec = std::string(plus ? "+" : space ? " " : "") + "." + + std::to_string(precision >= 0 ? precision : 6) + conversion; + text = formatValue(value, spec); + break; + } + case 'c': + numeric = false; + text = formatValue(value, "c"); + break; + case 's': + numeric = false; + text = toString(value); + if (precision >= 0 && utf8Length(text) > static_cast(precision)) { + text = text.substr(0, utf8Offset(text, static_cast(precision))); + } + break; + default: + throw RuntimeError(name + "(): unknown conversion '" + written + + "' (use %d, %f, %s, %c, %x, %o, %e, %g or %%)"); + } + + // Pad to the width: spaces on the left (or right with -), or zeros + // after the sign and prefix with 0. + std::string sign; + if (numeric && !text.empty() && (text[0] == '-' || text[0] == '+' || text[0] == ' ')) { + sign = text.substr(0, 1); + text = text.substr(1); + } + size_t length = sign.size() + prefix.size() + utf8Length(text); + size_t padding = static_cast(width) > length ? static_cast(width) - length : 0; + if (left) { + out += sign + prefix + text + std::string(padding, ' '); + } else if (zero && numeric) { + out += sign + prefix + std::string(padding, '0') + text; + } else { + out += std::string(padding, ' ') + sign + prefix + text; + } + } + if (next < args.size()) { + throw RuntimeError(name + "(): " + std::to_string(args.size() - 1) + " values were given, but the format only uses " + + std::to_string(next - 1)); + } + return out; +} + +std::regex compileRegex(const Args& args, size_t i, const char* fn) { + const std::string& pattern = stringArg(args, i, fn); + try { + return std::regex(pattern, std::regex::ECMAScript); + } catch (const std::regex_error& error) { + throw RuntimeError(std::string(fn) + "(): invalid regular expression \"" + pattern + "\": " + error.what()); + } +} + +// Runs a regex operation, turning the library's errors (e.g. a pattern too +// complex for the text) into Ascend errors. +template +auto withRegex(const char* fn, F body) -> decltype(body()) { + try { + return body(); + } catch (const std::regex_error& error) { + throw RuntimeError(std::string(fn) + "(): " + error.what()); + } +} + +// The whole match and each group, as strings (null for groups that didn't match). +Value matchGroups(Interpreter& in, const std::smatch& match) { + std::vector groups; + for (size_t g = 0; g < match.size(); ++g) { + groups.push_back(match[g].matched ? Value::fromString(match[g].str()) : Value()); + } + return Value::fromList(in.newList(std::move(groups))); +} + // --------------------------------------------------------------------------- // Builtins @@ -266,14 +465,29 @@ void registerCore(Interpreter& interp) { return Value::fromString(line); }); - interp.defineNative("type", 1, 1, [](Interpreter&, Args& args) { return Value::fromString(args[0].typeName()); }); + interp.defineNative("type", 1, 1, [](Interpreter&, Args& args) { return Value::fromString(typeNameOf(args[0])); }); interp.defineNative("str", 1, 1, [](Interpreter&, Args& args) { return Value::fromString(toString(args[0])); }); interp.defineNative("string", 1, 1, [](Interpreter&, Args& args) { return Value::fromString(toString(args[0])); }); interp.defineNative("repr", 1, 1, [](Interpreter&, Args& args) { return Value::fromString(toRepr(args[0])); }); interp.defineNative("bool", 1, 1, [](Interpreter&, Args& args) { return Value::fromBool(args[0].truthy()); }); - interp.defineNative("len", 1, 1, [](Interpreter&, Args& args) { + interp.defineNative("is_a", 2, 2, [](Interpreter&, Args& args) { + // is_a(value, Point) or is_a(value, "Point"): true for the struct and the structs it extends. + std::string type; + if (args[1].isString()) { + type = args[1].asString(); + } else if (args[1].isFunction() && args[1].asFunction().kind == Callable::Kind::Struct) { + type = static_cast(args[1].asFunction()).info->name; + } else { + typeError("is_a", 1, "a struct or a type name", args[1]); + } + const StructInfo* info = args[0].isMap() ? args[0].asMap().structInfo() : nullptr; + return Value::fromBool(info != nullptr ? info->isA(type) : typeNameOf(args[0]) == type); + }); + + interp.defineNative("len", 1, 1, [](Interpreter& in, Args& args) { const Value& value = args[0]; + if (const Value* method = structMethod(value, "len")) return in.call(*method, {value}); switch (value.type()) { case Value::Type::String: return Value::fromInt(static_cast(utf8Length(value.asString()))); case Value::Type::List: return Value::fromInt(static_cast(value.asList().items.size())); @@ -310,6 +524,15 @@ void registerCore(Interpreter& interp) { return in.importModule(stringArg(args, 0, "import")); }); + interp.defineNative("printf", 1, -1, [](Interpreter& in, Args& args) { + in.out() << cFormat("printf", args); + return Value(); + }); + + interp.defineNative("sprintf", 1, -1, [](Interpreter&, Args& args) { + return Value::fromString(cFormat("sprintf", args)); + }); + interp.defineNative("format", 1, -1, [](Interpreter&, Args& args) { return Value::fromString(formatTemplate(stringArg(args, 0, "format"), args, 1)); }); @@ -323,7 +546,7 @@ void registerConversions(Interpreter& interp) { static const IntConversion intConversions[] = { {"int", NumKind::Int}, {"i8", NumKind::I8}, {"i16", NumKind::I16}, {"i32", NumKind::I32}, {"i64", NumKind::I64}, {"u8", NumKind::U8}, {"u16", NumKind::U16}, {"u32", NumKind::U32}, - {"u64", NumKind::U64}, {"byte", NumKind::U8}, + {"u64", NumKind::U64}, {"byte", NumKind::U8}, {"long", NumKind::I64}, {"short", NumKind::I16}, }; for (const auto& conversion : intConversions) { const char* name = conversion.name; @@ -336,7 +559,7 @@ void registerConversions(Interpreter& interp) { NumKind kind; }; static const FloatConversion floatConversions[] = { - {"float", NumKind::Float}, {"f32", NumKind::F32}, {"f64", NumKind::F64}}; + {"float", NumKind::Float}, {"f32", NumKind::F32}, {"f64", NumKind::F64}, {"double", NumKind::F64}}; for (const auto& conversion : floatConversions) { const char* name = conversion.name; NumKind kind = conversion.kind; @@ -377,8 +600,10 @@ void registerConversions(Interpreter& interp) { void registerCollections(Interpreter& interp) { interp.defineNative("push", 2, -1, [](Interpreter&, Args& args) { - auto& items = listArg(args, 0, "push").items; - for (size_t i = 1; i < args.size(); ++i) items.push_back(args[i]); + ListObj& list = listArg(args, 0, "push"); + std::vector added; + for (size_t i = 1; i < args.size(); ++i) added.push_back(checkListItem(list, args[i])); + list.items.insert(list.items.end(), added.begin(), added.end()); return Value(); }); @@ -399,12 +624,13 @@ void registerCollections(Interpreter& interp) { if (index < 0 || index > size) { throw RuntimeError("insert() index " + toString(args[1]) + " is out of range (length " + std::to_string(size) + ")"); } - items.insert(items.begin() + index, args[2]); + items.insert(items.begin() + index, checkListItem(args[0].asList(), args[2])); return Value(); }); interp.defineNative("remove", 2, 2, [](Interpreter&, Args& args) { if (args[0].isMap()) { + checkResizable(args[0].asMap(), "remove"); Value removed; if (!args[0].asMap().remove(args[1], &removed)) { throw RuntimeError("remove(): key " + toRepr(args[1]) + " not found in map"); @@ -419,14 +645,17 @@ void registerCollections(Interpreter& interp) { }); interp.defineNative("extend", 2, 2, [](Interpreter&, Args& args) { - auto& items = listArg(args, 0, "extend").items; + ListObj& list = listArg(args, 0, "extend"); + auto& items = list.items; std::vector more = listArg(args, 1, "extend").items; // copy: extending a list with itself is fine + for (auto& item : more) item = checkListItem(list, std::move(item)); items.insert(items.end(), more.begin(), more.end()); return Value(); }); interp.defineNative("clear", 1, 1, [](Interpreter&, Args& args) { if (args[0].isMap()) { + checkResizable(args[0].asMap(), "clear"); args[0].asMap().clearReferences(); } else { listArg(args, 0, "clear").items.clear(); @@ -500,6 +729,87 @@ void registerCollections(Interpreter& interp) { return Value::fromList(in.newList(std::move(items))); }); + interp.defineNative("count_by", 2, 2, [](Interpreter& in, Args& args) { + std::vector items = listArg(args, 0, "count_by").items; + const Value& key = functionArg(args, 1, "count_by"); + auto counts = in.newMap(); + for (const Value& item : items) { + Value group = in.call(key, {item}); + Value* existing = counts->find(group); + if (existing != nullptr) { + *existing = Value::fromInt(existing->asInt() + 1); + } else { + counts->set(group, Value::fromInt(1)); + } + } + return Value::fromMap(counts); + }); + + interp.defineNative("group_by", 2, 2, [](Interpreter& in, Args& args) { + std::vector items = listArg(args, 0, "group_by").items; + const Value& key = functionArg(args, 1, "group_by"); + auto groups = in.newMap(); + for (const Value& item : items) { + Value group = in.call(key, {item}); + Value* existing = groups->find(group); + if (existing != nullptr) { + existing->asList().items.push_back(item); + } else { + groups->set(group, Value::fromList(in.newList({item}))); + } + } + return Value::fromMap(groups); + }); + + interp.defineNative("chunk", 2, 2, [](Interpreter& in, Args& args) { + std::vector items = listArg(args, 0, "chunk").items; + int64_t size = intArg(args, 1, "chunk"); + if (size < 1) throw RuntimeError("chunk() size must be at least 1"); + std::vector chunks; + for (size_t i = 0; i < items.size(); i += static_cast(size)) { + size_t end = std::min(items.size(), i + static_cast(size)); + chunks.push_back(Value::fromList(in.newList(std::vector(items.begin() + static_cast(i), + items.begin() + static_cast(end))))); + } + return Value::fromList(in.newList(std::move(chunks))); + }); + + interp.defineNative("flat_map", 2, 2, [](Interpreter& in, Args& args) { + std::vector items = listArg(args, 0, "flat_map").items; + const Value& fn = functionArg(args, 1, "flat_map"); + std::vector result; + for (const Value& item : items) { + Value mapped = in.call(fn, {item}); + if (mapped.isList()) { + const auto& inner = mapped.asList().items; + result.insert(result.end(), inner.begin(), inner.end()); + } else { + result.push_back(std::move(mapped)); + } + } + return Value::fromList(in.newList(std::move(result))); + }); + + auto extremeBy = [](const char* name, int sign) { + return [name, sign](Interpreter& in, Args& args) { + std::vector items = listArg(args, 0, name).items; + const Value& key = functionArg(args, 1, name); + if (items.empty()) throw RuntimeError(std::string(name) + "() of an empty list"); + Value best = items[0]; + Value bestKey = in.call(key, {best}); + for (size_t i = 1; i < items.size(); ++i) { + Value candidate = in.call(key, {items[i]}); + if (compareValues(candidate, bestKey) * sign > 0) { + best = items[i]; + bestKey = std::move(candidate); + } + } + return best; + }; + }; + interp.defineNative("min_by", 2, 2, extremeBy("min_by", -1)); + interp.defineNative("max_by", 2, 2, extremeBy("max_by", 1)); + interp.defineNative("unique", 1, 1, [](Interpreter& in, Args& args) { std::vector result; for (const Value& item : listArg(args, 0, "unique").items) { @@ -587,10 +897,17 @@ void registerCollections(Interpreter& interp) { }); interp.defineNative("copy", 1, 1, [](Interpreter& in, Args& args) { - if (args[0].isList()) return Value::fromList(in.newList(args[0].asList().items)); + if (args[0].isList()) { + auto copy = in.newList(args[0].asList().items); + copy->elementType = args[0].asList().elementType; + return Value::fromList(copy); + } if (args[0].isMap()) { auto copy = in.newMap(); for (const auto& entry : args[0].asMap().entries) copy->set(entry.first, entry.second); + copy->structType = args[0].asMap().structType; // a copy of a struct is the same kind of struct + copy->keyType = args[0].asMap().keyType; + copy->valueType = args[0].asMap().valueType; return Value::fromMap(copy); } return args[0]; @@ -847,6 +1164,36 @@ void registerMath(Interpreter& interp) { return total; }); + // The numbers of a list as doubles, for mean() and median(). + auto numbers = [](const Args& args, const char* name) { + const auto& items = listArg(args, 0, name).items; + if (items.empty()) throw RuntimeError(std::string(name) + "() of an empty list"); + std::vector values; + values.reserve(items.size()); + for (const Value& item : items) { + if (!item.isNumber()) { + throw RuntimeError(std::string(name) + "() needs a list of numbers, but it contains " + article(item)); + } + values.push_back(item.toDouble()); + } + return values; + }; + + interp.defineNative("mean", 1, 1, [numbers](Interpreter&, Args& args) { + std::vector values = numbers(args, "mean"); + double total = 0; + for (double value : values) total += value; + return Value::fromFloat(total / static_cast(values.size())); + }); + + interp.defineNative("median", 1, 1, [numbers](Interpreter&, Args& args) { + std::vector values = numbers(args, "median"); + std::sort(values.begin(), values.end()); + size_t middle = values.size() / 2; + double result = values.size() % 2 == 1 ? values[middle] : (values[middle - 1] + values[middle]) / 2; + return Value::fromFloat(result); + }); + interp.defineNative("sign", 1, 1, [](Interpreter&, Args& args) { numberArg(args, 0, "sign"); if (args[0].isFloat() && std::isnan(args[0].asFloat())) return args[0]; @@ -1062,6 +1409,81 @@ void registerFiles(Interpreter& interp) { if (indent < 0 || indent > 16) throw RuntimeError("json_stringify() indent must be from 0 to 16"); return Value::fromString(jsonStringify(args[0], static_cast(indent))); }); + + interp.defineNative("is_dir", 1, 1, [](Interpreter&, Args& args) { + std::error_code ignored; + return Value::fromBool(std::filesystem::is_directory(stringArg(args, 0, "is_dir"), ignored)); + }); + + interp.defineNative("make_dir", 1, 1, [](Interpreter&, Args& args) { + const std::string& path = stringArg(args, 0, "make_dir"); + std::error_code error; + bool created = std::filesystem::create_directories(path, error); + if (error) throw RuntimeError("make_dir(): can't create '" + path + "': " + error.message()); + return Value::fromBool(created); + }); + + interp.defineNative("list_dir", 0, 1, [](Interpreter& in, Args& args) { + std::string path = args.empty() ? "." : stringArg(args, 0, "list_dir"); + std::error_code error; + std::vector names; + for (std::filesystem::directory_iterator it(path, error), end; !error && it != end; it.increment(error)) { + names.push_back(it->path().filename().string()); + } + if (error) throw RuntimeError("list_dir(): can't read the folder '" + path + "': " + error.message()); + std::sort(names.begin(), names.end()); + std::vector items; + for (auto& name : names) items.push_back(Value::fromString(std::move(name))); + return Value::fromList(in.newList(std::move(items))); + }); +} + +void registerRegex(Interpreter& interp) { + interp.defineNative("regex_match", 2, 2, [](Interpreter&, Args& args) { + std::regex pattern = compileRegex(args, 1, "regex_match"); + const std::string& text = stringArg(args, 0, "regex_match"); + return withRegex("regex_match", [&] { return Value::fromBool(std::regex_match(text, pattern)); }); + }); + + interp.defineNative("regex_search", 2, 2, [](Interpreter& in, Args& args) { + std::regex pattern = compileRegex(args, 1, "regex_search"); + const std::string& text = stringArg(args, 0, "regex_search"); + return withRegex("regex_search", [&] { + std::smatch match; + return std::regex_search(text, match, pattern) ? matchGroups(in, match) : Value(); + }); + }); + + interp.defineNative("regex_find_all", 2, 2, [](Interpreter& in, Args& args) { + std::regex pattern = compileRegex(args, 1, "regex_find_all"); + const std::string& text = stringArg(args, 0, "regex_find_all"); + return withRegex("regex_find_all", [&] { + std::vector found; + for (std::sregex_iterator it(text.begin(), text.end(), pattern), end; it != end; ++it) { + found.push_back(Value::fromString(it->str())); + } + return Value::fromList(in.newList(std::move(found))); + }); + }); + + interp.defineNative("regex_replace", 3, 3, [](Interpreter&, Args& args) { + std::regex pattern = compileRegex(args, 1, "regex_replace"); + const std::string& text = stringArg(args, 0, "regex_replace"); + const std::string& replacement = stringArg(args, 2, "regex_replace"); + return withRegex("regex_replace", [&] { return Value::fromString(std::regex_replace(text, pattern, replacement)); }); + }); + + interp.defineNative("regex_split", 2, 2, [](Interpreter& in, Args& args) { + std::regex pattern = compileRegex(args, 1, "regex_split"); + const std::string& text = stringArg(args, 0, "regex_split"); + return withRegex("regex_split", [&] { + std::vector parts; + for (std::sregex_token_iterator it(text.begin(), text.end(), pattern, -1), end; it != end; ++it) { + parts.push_back(Value::fromString(it->str())); + } + return Value::fromList(in.newList(std::move(parts))); + }); + }); } } // namespace @@ -1106,6 +1528,8 @@ void registerStandardLibrary(Interpreter& interpreter) { registerMath(interpreter); registerSystem(interpreter); registerFiles(interpreter); + registerRegex(interpreter); + registerExtraLibrary(interpreter); } } // namespace ascend diff --git a/src/stdlib_extra.cpp b/src/stdlib_extra.cpp new file mode 100644 index 0000000..833c4ca --- /dev/null +++ b/src/stdlib_extra.cpp @@ -0,0 +1,685 @@ +// More builtins: running programs, HTTP, paths and folders, text, math, +// sets, dates, CSV, base64, hashing and ids. + +#include "interpreter.hpp" +#include "numeric.hpp" +#include "process.hpp" +#include "stdlib.hpp" +#include "utf8.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace ascend { + +namespace { + +using Args = std::vector; + +[[noreturn]] void typeError(const char* fn, size_t index, const char* expected, const Value& got) { + throw RuntimeError(std::string(fn) + "() expects " + expected + " as argument " + std::to_string(index + 1) + + ", not " + aTypeName(got)); +} + +const std::string& stringArg(const Args& args, size_t i, const char* fn) { + if (!args[i].isString()) typeError(fn, i, "a string", args[i]); + return args[i].asString(); +} + +int64_t intArg(const Args& args, size_t i, const char* fn) { + int64_t value = 0; + if (!args[i].isInt() || !toInt64(args[i], value)) typeError(fn, i, "an integer", args[i]); + return value; +} + +double numberArg(const Args& args, size_t i, const char* fn) { + if (!args[i].isNumber()) typeError(fn, i, "a number", args[i]); + return args[i].toDouble(); +} + +const ListObj& listArg(const Args& args, size_t i, const char* fn) { + if (!args[i].isList()) typeError(fn, i, "a list", args[i]); + return args[i].asList(); +} + +Value stringList(Interpreter& in, const std::vector& items) { + std::vector values; + values.reserve(items.size()); + for (const auto& item : items) values.push_back(Value::fromString(item)); + return Value::fromList(in.newList(std::move(values))); +} + +void setField(MapObj& map, const char* key, Value value) { map.set(Value::fromString(key), std::move(value)); } + +std::string pathString(const fs::path& path) { return path.generic_string(); } + +[[noreturn]] void fileError(const char* fn, const std::string& path, const std::error_code& error) { + throw RuntimeError(std::string(fn) + "(): '" + path + "': " + error.message()); +} + +// --------------------------------------------------------------------------- +// Running programs + +Value capturedResult(Interpreter& in, const CapturedOutput& output) { + auto result = in.newMap(); + setField(*result, "code", Value::fromInt(output.code)); + setField(*result, "stdout", Value::fromString(output.out)); + setField(*result, "stderr", Value::fromString(output.err)); + setField(*result, "ok", Value::fromBool(output.code == 0)); + return Value::fromMap(result); +} + +void registerProcesses(Interpreter& interp) { + // run(["git", "status"]) runs a program directly; run("ls | wc -l") uses the shell. + interp.defineNative("run", 1, 2, [](Interpreter& in, Args& args) { + std::string input = args.size() > 1 ? stringArg(args, 1, "run") : ""; + CapturedOutput output; + if (args[0].isString()) { + if (!captureShell(args[0].asString(), input, output)) throw RuntimeError("run(): couldn't start the shell"); + return capturedResult(in, output); + } + if (!args[0].isList()) typeError("run", 0, "a command string or a list of arguments", args[0]); + std::vector command; + for (const Value& part : args[0].asList().items) command.push_back(toString(part)); + if (command.empty()) throw RuntimeError("run(): the command list is empty"); + if (!captureProcess(command, input, output)) { + throw RuntimeError("run(): couldn't start '" + command[0] + "' (is it installed and on your PATH?)"); + } + return capturedResult(in, output); + }); +} + +// --------------------------------------------------------------------------- +// HTTP, through the curl program + +void addHeaders(std::vector& command, const Value& headers, const char* fn, bool& hasContentType) { + if (headers.isNull()) return; + if (!headers.isMap()) throw RuntimeError(std::string(fn) + "(): headers must be a map like {\"Accept\": \"text/html\"}"); + for (const auto& entry : headers.asMap().entries) { + std::string name = toString(entry.first); + std::string lower = name; + for (char& c : lower) c = static_cast(std::tolower(static_cast(c))); + if (lower == "content-type") hasContentType = true; + command.push_back("-H"); + command.push_back(name + ": " + toString(entry.second)); + } +} + +Value httpRequest(Interpreter& in, const char* fn, const std::string& method, const std::string& url, const Value& body, + const Value& headers) { + std::vector command{"curl", "--silent", "--show-error", "--location", "--max-time", "120", + "--request", method, "--write-out", "\n%{http_code}"}; + bool hasContentType = false; + addHeaders(command, headers, fn, hasContentType); + std::string input; + if (!body.isNull()) { + if (body.isString()) { + input = body.asString(); + } else { + input = jsonStringify(body, 0); // maps and lists are sent as JSON + if (!hasContentType) { + command.push_back("-H"); + command.push_back("Content-Type: application/json"); + } + } + command.push_back("--data-binary"); + command.push_back("@-"); + } + command.push_back("--"); + command.push_back(url); + + CapturedOutput output; + if (!captureProcess(command, input, output)) { + throw RuntimeError(std::string(fn) + "() needs the curl program, which wasn't found; install curl and try again"); + } + if (output.code != 0) { + std::string why = output.err; + while (!why.empty() && (why.back() == '\n' || why.back() == '\r')) why.pop_back(); + if (why.compare(0, 5, "curl:") == 0) why = why.substr(5); + while (!why.empty() && why[0] == ' ') why.erase(0, 1); + throw RuntimeError(std::string(fn) + "(): request to " + url + " failed: " + (why.empty() ? "curl error" : why)); + } + size_t split = output.out.rfind('\n'); + std::string text = split == std::string::npos ? "" : output.out.substr(0, split); + int status = std::atoi(output.out.c_str() + (split == std::string::npos ? 0 : split + 1)); + auto result = in.newMap(); + setField(*result, "status", Value::fromInt(status)); + setField(*result, "ok", Value::fromBool(status >= 200 && status < 300)); + setField(*result, "body", Value::fromString(text)); + return Value::fromMap(result); +} + +void registerHttp(Interpreter& interp) { + interp.defineNative("http_get", 1, 2, [](Interpreter& in, Args& args) { + return httpRequest(in, "http_get", "GET", stringArg(args, 0, "http_get"), Value(), + args.size() > 1 ? args[1] : Value()); + }); + interp.defineNative("http_post", 2, 3, [](Interpreter& in, Args& args) { + return httpRequest(in, "http_post", "POST", stringArg(args, 0, "http_post"), args[1], + args.size() > 2 ? args[2] : Value()); + }); + interp.defineNative("http_request", 2, 4, [](Interpreter& in, Args& args) { + std::string method = stringArg(args, 0, "http_request"); + for (char& c : method) c = static_cast(std::toupper(static_cast(c))); + return httpRequest(in, "http_request", method, stringArg(args, 1, "http_request"), + args.size() > 2 ? args[2] : Value(), args.size() > 3 ? args[3] : Value()); + }); +} + +// --------------------------------------------------------------------------- +// Paths, folders and the environment + +void registerPaths(Interpreter& interp) { + interp.defineNative("path_join", 1, -1, [](Interpreter&, Args& args) { + fs::path path; + for (size_t i = 0; i < args.size(); ++i) path /= stringArg(args, i, "path_join"); + return Value::fromString(pathString(path)); + }); + interp.defineNative("basename", 1, 1, [](Interpreter&, Args& args) { + return Value::fromString(fs::path(stringArg(args, 0, "basename")).filename().string()); + }); + interp.defineNative("dirname", 1, 1, [](Interpreter&, Args& args) { + return Value::fromString(pathString(fs::path(stringArg(args, 0, "dirname")).parent_path())); + }); + interp.defineNative("extension", 1, 1, [](Interpreter&, Args& args) { + return Value::fromString(fs::path(stringArg(args, 0, "extension")).extension().string()); + }); + interp.defineNative("abs_path", 1, 1, [](Interpreter&, Args& args) { + std::error_code error; + fs::path path = fs::absolute(stringArg(args, 0, "abs_path"), error); + if (error) fileError("abs_path", args[0].asString(), error); + return Value::fromString(pathString(path.lexically_normal())); + }); + interp.defineNative("cwd", 0, 0, [](Interpreter&, Args&) { + std::error_code error; + fs::path path = fs::current_path(error); + if (error) throw RuntimeError("cwd(): " + error.message()); + return Value::fromString(pathString(path)); + }); + interp.defineNative("chdir", 1, 1, [](Interpreter&, Args& args) { + std::error_code error; + fs::current_path(stringArg(args, 0, "chdir"), error); + if (error) fileError("chdir", args[0].asString(), error); + return Value(); + }); + interp.defineNative("temp_dir", 0, 0, [](Interpreter&, Args&) { + std::error_code error; + fs::path path = fs::temp_directory_path(error); + if (error) throw RuntimeError("temp_dir(): " + error.message()); + return Value::fromString(pathString(path)); + }); + interp.defineNative("file_size", 1, 1, [](Interpreter&, Args& args) { + std::error_code error; + uintmax_t size = fs::file_size(stringArg(args, 0, "file_size"), error); + if (error) fileError("file_size", args[0].asString(), error); + return Value::fromInt(static_cast(size)); + }); + interp.defineNative("copy_file", 2, 2, [](Interpreter&, Args& args) { + std::error_code error; + fs::copy_file(stringArg(args, 0, "copy_file"), stringArg(args, 1, "copy_file"), + fs::copy_options::overwrite_existing, error); + if (error) fileError("copy_file", args[0].asString(), error); + return Value(); + }); + interp.defineNative("move_file", 2, 2, [](Interpreter&, Args& args) { + std::error_code error; + fs::rename(stringArg(args, 0, "move_file"), stringArg(args, 1, "move_file"), error); + if (error) fileError("move_file", args[0].asString(), error); + return Value(); + }); + interp.defineNative("remove_dir", 1, 1, [](Interpreter&, Args& args) { + const std::string& path = stringArg(args, 0, "remove_dir"); + std::error_code error; + if (!fs::is_directory(path, error)) return Value::fromBool(false); + uintmax_t removed = fs::remove_all(path, error); + if (error) fileError("remove_dir", path, error); + return Value::fromBool(removed > 0); + }); + interp.defineNative("walk_dir", 0, 1, [](Interpreter& in, Args& args) { + // Every file below a folder, as paths relative to it, sorted. + std::string root = args.empty() ? "." : stringArg(args, 0, "walk_dir"); + std::error_code error; + std::vector files; + fs::recursive_directory_iterator it(root, fs::directory_options::skip_permission_denied, error), end; + for (; !error && it != end; it.increment(error)) { + if (it->is_regular_file()) files.push_back(pathString(it->path().lexically_relative(root))); + } + if (error) fileError("walk_dir", root, error); + std::sort(files.begin(), files.end()); + return stringList(in, files); + }); + interp.defineNative("set_env", 2, 2, [](Interpreter&, Args& args) { + setEnvironmentVariable(stringArg(args, 0, "set_env"), toString(args[1])); + return Value(); + }); + interp.defineNative("platform", 0, 0, [](Interpreter&, Args&) { +#if defined(_WIN32) + return Value::fromString("windows"); +#elif defined(__APPLE__) + return Value::fromString("macos"); +#elif defined(__linux__) + return Value::fromString("linux"); +#else + return Value::fromString("unix"); +#endif + }); +} + +// --------------------------------------------------------------------------- +// Text + +void registerText(Interpreter& interp) { + interp.defineNative("capitalize", 1, 1, [](Interpreter&, Args& args) { + std::string text = stringArg(args, 0, "capitalize"); + if (!text.empty()) text[0] = static_cast(std::toupper(static_cast(text[0]))); + return Value::fromString(text); + }); + interp.defineNative("title", 1, 1, [](Interpreter&, Args& args) { + std::string text = stringArg(args, 0, "title"); + bool start = true; + for (char& c : text) { + unsigned char byte = static_cast(c); + c = static_cast(start ? std::toupper(byte) : std::tolower(byte)); + start = std::isspace(byte) != 0 || c == '-' || c == '_'; + } + return Value::fromString(text); + }); + interp.defineNative("trim_start", 1, 1, [](Interpreter&, Args& args) { + const std::string& text = stringArg(args, 0, "trim_start"); + size_t first = text.find_first_not_of(" \t\r\n\f\v"); + return Value::fromString(first == std::string::npos ? "" : text.substr(first)); + }); + interp.defineNative("trim_end", 1, 1, [](Interpreter&, Args& args) { + const std::string& text = stringArg(args, 0, "trim_end"); + size_t last = text.find_last_not_of(" \t\r\n\f\v"); + return Value::fromString(last == std::string::npos ? "" : text.substr(0, last + 1)); + }); + interp.defineNative("last_index_of", 2, 2, [](Interpreter&, Args& args) { + if (args[0].isList()) { + const auto& items = args[0].asList().items; + for (size_t i = items.size(); i > 0; --i) { + if (valuesEqual(items[i - 1], args[1])) return Value::fromInt(static_cast(i - 1)); + } + return Value::fromInt(-1); + } + const std::string& text = stringArg(args, 0, "last_index_of"); + std::string part = args[1].isChar() ? encodeUtf8(args[1].asChar()) : stringArg(args, 1, "last_index_of"); + size_t found = text.rfind(part); + if (found == std::string::npos) return Value::fromInt(-1); + return Value::fromInt(static_cast(utf8Length(text.substr(0, found)))); + }); +} + +// --------------------------------------------------------------------------- +// Math and sets + +void registerMathExtras(Interpreter& interp) { + interp.defineNative("lcm", 2, 2, [](Interpreter&, Args& args) { + int64_t a = intArg(args, 0, "lcm"); + int64_t b = intArg(args, 1, "lcm"); + if (a == 0 || b == 0) return Value::fromInt(0); + if (a == std::numeric_limits::min() || b == std::numeric_limits::min()) { + throw RuntimeError("lcm(): the numbers are too large"); + } + a = a < 0 ? -a : a; + b = b < 0 ? -b : b; + int64_t x = a; + int64_t y = b; + while (y != 0) { + int64_t t = x % y; + x = y; + y = t; + } + int64_t factor = a / x; + if (factor > std::numeric_limits::max() / b) throw RuntimeError("lcm(): the result is too large for an int"); + return Value::fromInt(factor * b); + }); + interp.defineNative("factorial", 1, 1, [](Interpreter&, Args& args) { + int64_t n = intArg(args, 0, "factorial"); + if (n < 0) throw RuntimeError("factorial() needs a number that isn't negative"); + if (n > 20) throw RuntimeError("factorial(" + std::to_string(n) + ") is too large for an int (the most is 20)"); + int64_t result = 1; + for (int64_t i = 2; i <= n; ++i) result *= i; + return Value::fromInt(result); + }); + interp.defineNative("degrees", 1, 1, [](Interpreter&, Args& args) { + return Value::fromFloat(numberArg(args, 0, "degrees") * 180.0 / 3.14159265358979323846); + }); + interp.defineNative("radians", 1, 1, [](Interpreter&, Args& args) { + return Value::fromFloat(numberArg(args, 0, "radians") * 3.14159265358979323846 / 180.0); + }); + interp.defineNative("trunc", 1, 1, [](Interpreter&, Args& args) { + if (args[0].isInt()) return args[0]; + double value = std::trunc(numberArg(args, 0, "trunc")); + if (!(value >= -9.2233720368547758e18 && value < 9.2233720368547758e18)) { + throw RuntimeError("trunc(): " + toString(args[0]) + " is too large for an int"); + } + return Value::fromInt(static_cast(value)); + }); + + // Set operations on lists. The results keep the order of the first list + // and have no duplicates. + auto setOperation = [](const char* name, int mode) { + return [name, mode](Interpreter& in, Args& args) { + const auto& a = listArg(args, 0, name).items; + const auto& b = listArg(args, 1, name).items; + auto has = [](const std::vector& items, const Value& value) { + return std::any_of(items.begin(), items.end(), [&](const Value& item) { return valuesEqual(item, value); }); + }; + std::vector result; + auto add = [&](const Value& value) { + if (!has(result, value)) result.push_back(value); + }; + for (const Value& item : a) { + if (mode == 0 || (mode == 1 && has(b, item)) || (mode == 2 && !has(b, item))) add(item); + } + if (mode == 0) { + for (const Value& item : b) add(item); + } + return Value::fromList(in.newList(std::move(result))); + }; + }; + interp.defineNative("union", 2, 2, setOperation("union", 0)); + interp.defineNative("intersect", 2, 2, setOperation("intersect", 1)); + interp.defineNative("difference", 2, 2, setOperation("difference", 2)); +} + +// --------------------------------------------------------------------------- +// Dates + +std::tm localTime(double seconds) { + std::time_t time = static_cast(std::floor(seconds)); + std::tm parts{}; +#if defined(_WIN32) + if (localtime_s(&parts, &time) != 0) throw RuntimeError("the time " + formatFloat(seconds) + " is out of range"); +#else + if (localtime_r(&time, &parts) == nullptr) throw RuntimeError("the time " + formatFloat(seconds) + " is out of range"); +#endif + return parts; +} + +void registerDates(Interpreter& interp) { + // now(): the local date and time as a map. + interp.defineNative("now", 0, 0, [](Interpreter& in, Args&) { + double seconds = std::chrono::duration(std::chrono::system_clock::now().time_since_epoch()).count(); + std::tm parts = localTime(seconds); + auto result = in.newMap(); + setField(*result, "year", Value::fromInt(parts.tm_year + 1900)); + setField(*result, "month", Value::fromInt(parts.tm_mon + 1)); + setField(*result, "day", Value::fromInt(parts.tm_mday)); + setField(*result, "hour", Value::fromInt(parts.tm_hour)); + setField(*result, "minute", Value::fromInt(parts.tm_min)); + setField(*result, "second", Value::fromInt(parts.tm_sec)); + setField(*result, "weekday", Value::fromInt(parts.tm_wday)); // 0 is Sunday + setField(*result, "yearday", Value::fromInt(parts.tm_yday + 1)); + setField(*result, "timestamp", Value::fromFloat(seconds)); + return Value::fromMap(result); + }); + // format_time(seconds, format?): a timestamp (like time()) as local time text. + interp.defineNative("format_time", 1, 2, [](Interpreter&, Args& args) { + double seconds = numberArg(args, 0, "format_time"); + std::string format = args.size() > 1 ? stringArg(args, 1, "format_time") : "%Y-%m-%d %H:%M:%S"; + if (format.empty()) return Value::fromString(""); + std::tm parts = localTime(seconds); + char buffer[512]; + size_t length = std::strftime(buffer, sizeof buffer, format.c_str(), &parts); + if (length == 0) throw RuntimeError("format_time(): the format gives no text, or too much"); + return Value::fromString(std::string(buffer, length)); + }); +} + +// --------------------------------------------------------------------------- +// CSV + +Value parseCsv(Interpreter& in, const std::string& text, char separator) { + std::vector rows; + std::vector row; + std::string field; + bool quoted = false; + bool fieldStarted = false; + auto endField = [&] { + row.push_back(Value::fromString(field)); + field.clear(); + fieldStarted = false; + }; + auto endRow = [&] { + endField(); + rows.push_back(Value::fromList(in.newList(std::move(row)))); + row.clear(); + }; + for (size_t i = 0; i < text.size(); ++i) { + char c = text[i]; + if (quoted) { + if (c == '"' && i + 1 < text.size() && text[i + 1] == '"') { + field += '"'; + ++i; + } else if (c == '"') { + quoted = false; + } else { + field += c; + } + continue; + } + if (c == '"' && !fieldStarted && field.empty()) { + quoted = true; + fieldStarted = true; + } else if (c == separator) { + endField(); + } else if (c == '\n' || c == '\r') { + if (c == '\r' && i + 1 < text.size() && text[i + 1] == '\n') ++i; + endRow(); + } else { + field += c; + fieldStarted = true; + } + } + if (quoted) throw RuntimeError("csv_parse(): a quoted field is never closed"); + if (fieldStarted || !field.empty() || !row.empty()) endRow(); + return Value::fromList(in.newList(std::move(rows))); +} + +std::string csvField(const std::string& text, char separator) { + bool needsQuotes = text.find_first_of(std::string("\"\r\n") + separator) != std::string::npos || + (!text.empty() && (text.front() == ' ' || text.back() == ' ')); + if (!needsQuotes) return text; + std::string out = "\""; + for (char c : text) { + if (c == '"') out += '"'; + out += c; + } + return out + "\""; +} + +char separatorArg(const Args& args, size_t i, const char* fn) { + if (args.size() <= i) return ','; + std::string text = args[i].isChar() ? encodeUtf8(args[i].asChar()) : stringArg(args, i, fn); + if (text.size() != 1) throw RuntimeError(std::string(fn) + "(): the separator must be one character"); + return text[0]; +} + +void registerCsv(Interpreter& interp) { + interp.defineNative("csv_parse", 1, 2, [](Interpreter& in, Args& args) { + return parseCsv(in, stringArg(args, 0, "csv_parse"), separatorArg(args, 1, "csv_parse")); + }); + interp.defineNative("csv_stringify", 1, 2, [](Interpreter&, Args& args) { + char separator = separatorArg(args, 1, "csv_stringify"); + std::string out; + for (const Value& row : listArg(args, 0, "csv_stringify").items) { + if (!row.isList()) throw RuntimeError("csv_stringify() needs a list of rows, each a list of values"); + bool first = true; + for (const Value& value : row.asList().items) { + if (!first) out += separator; + first = false; + out += csvField(value.isNull() ? "" : toString(value), separator); + } + out += '\n'; + } + return Value::fromString(out); + }); +} + +// --------------------------------------------------------------------------- +// Encodings, hashing and ids + +const char* kBase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +std::string base64Encode(const std::string& data) { + std::string out; + size_t i = 0; + for (; i + 2 < data.size(); i += 3) { + uint32_t n = (static_cast(data[i]) << 16) | (static_cast(data[i + 1]) << 8) | + static_cast(data[i + 2]); + out += kBase64[(n >> 18) & 63]; + out += kBase64[(n >> 12) & 63]; + out += kBase64[(n >> 6) & 63]; + out += kBase64[n & 63]; + } + if (i < data.size()) { + uint32_t n = static_cast(static_cast(data[i]) << 16); + if (i + 1 < data.size()) n |= static_cast(static_cast(data[i + 1]) << 8); + out += kBase64[(n >> 18) & 63]; + out += kBase64[(n >> 12) & 63]; + out += i + 1 < data.size() ? kBase64[(n >> 6) & 63] : '='; + out += '='; + } + return out; +} + +std::string base64Decode(const std::string& text) { + std::string out; + uint32_t buffer = 0; + int bits = 0; + for (char c : text) { + if (c == '=' || c == '\n' || c == '\r' || c == ' ') continue; + const char* found = std::strchr(kBase64, c); + if (found == nullptr || c == '\0') throw RuntimeError("base64_decode(): '" + std::string(1, c) + "' isn't base64"); + buffer = (buffer << 6) | static_cast(found - kBase64); + bits += 6; + if (bits >= 8) { + bits -= 8; + out += static_cast((buffer >> bits) & 0xFF); + } + } + return out; +} + +std::string sha256(const std::string& message) { + static const uint32_t k[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + uint32_t h[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + auto rotr = [](uint32_t x, int n) { return (x >> n) | (x << (32 - n)); }; + + std::string data = message; + uint64_t bitLength = static_cast(message.size()) * 8; + data += static_cast(0x80); + while (data.size() % 64 != 56) data += '\0'; + for (int i = 7; i >= 0; --i) data += static_cast((bitLength >> (i * 8)) & 0xFF); + + for (size_t chunk = 0; chunk < data.size(); chunk += 64) { + uint32_t w[64]; + for (int i = 0; i < 16; ++i) { + w[i] = (static_cast(static_cast(data[chunk + 4 * i])) << 24) | + (static_cast(static_cast(data[chunk + 4 * i + 1])) << 16) | + (static_cast(static_cast(data[chunk + 4 * i + 2])) << 8) | + static_cast(static_cast(data[chunk + 4 * i + 3])); + } + for (int i = 16; i < 64; ++i) { + uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], hh = h[7]; + for (int i = 0; i < 64; ++i) { + uint32_t s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + uint32_t ch = (e & f) ^ (~e & g); + uint32_t temp1 = hh + s1 + ch + k[i] + w[i]; + uint32_t s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + uint32_t temp2 = s0 + maj; + hh = g; + g = f; + f = e; + e = d + temp1; + d = c; + c = b; + b = a; + a = temp1 + temp2; + } + h[0] += a; + h[1] += b; + h[2] += c; + h[3] += d; + h[4] += e; + h[5] += f; + h[6] += g; + h[7] += hh; + } + static const char* digits = "0123456789abcdef"; + std::string out; + for (uint32_t word : h) { + for (int shift = 28; shift >= 0; shift -= 4) out += digits[(word >> shift) & 0xF]; + } + return out; +} + +void registerEncodings(Interpreter& interp) { + interp.defineNative("base64_encode", 1, 1, [](Interpreter&, Args& args) { + return Value::fromString(base64Encode(stringArg(args, 0, "base64_encode"))); + }); + interp.defineNative("base64_decode", 1, 1, [](Interpreter&, Args& args) { + return Value::fromString(base64Decode(stringArg(args, 0, "base64_decode"))); + }); + interp.defineNative("sha256", 1, 1, [](Interpreter&, Args& args) { + return Value::fromString(sha256(stringArg(args, 0, "sha256"))); + }); + interp.defineNative("uuid", 0, 0, [](Interpreter& in, Args&) { + // A random (version 4) UUID. + std::uniform_int_distribution byte(0, 255); + unsigned char bytes[16]; + for (auto& b : bytes) b = static_cast(byte(in.random())); + bytes[6] = static_cast((bytes[6] & 0x0F) | 0x40); + bytes[8] = static_cast((bytes[8] & 0x3F) | 0x80); + static const char* digits = "0123456789abcdef"; + std::string out; + for (int i = 0; i < 16; ++i) { + if (i == 4 || i == 6 || i == 8 || i == 10) out += '-'; + out += digits[bytes[i] >> 4]; + out += digits[bytes[i] & 0xF]; + } + return Value::fromString(out); + }); +} + +} // namespace + +void registerExtraLibrary(Interpreter& interpreter) { + registerProcesses(interpreter); + registerHttp(interpreter); + registerPaths(interpreter); + registerText(interpreter); + registerMathExtras(interpreter); + registerDates(interpreter); + registerCsv(interpreter); + registerEncodings(interpreter); +} + +} // namespace ascend diff --git a/src/typecheck.cpp b/src/typecheck.cpp index aead71c..b3ae2b6 100644 --- a/src/typecheck.cpp +++ b/src/typecheck.cpp @@ -21,6 +21,49 @@ std::string withHint(const std::string& message, const CoerceFailure& failure) { return failure.hint.empty() ? message : message + " (" + failure.hint + ")"; } +namespace { + +// Guards against lists that contain themselves while checking element types. +int& checkDepth() { + static thread_local int depth = 0; + return depth; +} + +struct DepthGuard { + DepthGuard() { ++checkDepth(); } + ~DepthGuard() { --checkDepth(); } +}; + +const char* kKeepsTypeHint = "a list or map keeps the element types it was first used with; use copy() for another type"; + +} // namespace + +Value checkListItem(const ListObj& list, Value value) { + if (!list.elementType) return value; + Value converted; + CoerceFailure failure; + if (!coerceToType(value, *list.elementType, converted, failure)) { + throw RuntimeError(withHint("this is a list<" + list.elementType->name() + ">, so it can't hold " + failure.what, + failure)); + } + return converted; +} + +void checkMapEntry(const MapObj& map, const Value& key, Value& value) { + Value converted; + CoerceFailure failure; + std::string type = map.keyType ? "map<" + map.keyType->name() + ", " + map.valueType->name() + ">" : ""; + if (map.keyType && !coerceToType(key, *map.keyType, converted, failure)) { + throw RuntimeError(withHint("this is a " + type + ", so its keys can't be " + failure.what, failure)); + } + if (map.valueType) { + if (!coerceToType(value, *map.valueType, converted, failure)) { + throw RuntimeError(withHint("this is a " + type + ", so its values can't be " + failure.what, failure)); + } + value = std::move(converted); + } +} + bool coerceToType(const Value& value, const TypeSpec& type, Value& out, CoerceFailure& failure) { if (type.kind == TypeKind::Any) { out = value; @@ -83,21 +126,102 @@ bool coerceToType(const Value& value, const TypeSpec& type, Value& out, CoerceFa failure.hint = "use str(...) to convert it to a string"; break; - case TypeKind::List: + case TypeKind::List: { if (!value.isList()) break; + ListObj& list = value.asList(); + if (type.element) { + if (list.elementType) { + if (!sameType(*list.elementType, *type.element)) { + failure.what = "a list<" + list.elementType->name() + ">"; + failure.hint = kKeepsTypeHint; + return false; + } + } else { + // Check (and convert) every item, then mark the list as a list. + DepthGuard guard; + if (checkDepth() > 200) { + failure.what = "a list nested too deeply"; + return false; + } + std::vector items; + items.reserve(list.items.size()); + for (const Value& item : list.items) { + Value converted; + CoerceFailure itemFailure; + if (!coerceToType(item, *type.element, converted, itemFailure)) { + failure.what = "a list containing " + itemFailure.what; + failure.hint = itemFailure.hint; + return false; + } + items.push_back(std::move(converted)); + } + list.items = std::move(items); + list.elementType = type.element; + } + } out = value; return true; + } - case TypeKind::Map: + case TypeKind::Map: { if (!value.isMap()) break; + MapObj& map = value.asMap(); + if (type.element && (map.structType || !map.frozenAs.empty())) { + failure.what = map.structType ? aTypeName(value) : "an enum"; // these keep their own rules + return false; + } + if (type.element) { + if (map.valueType) { + if (!sameType(*map.valueType, *type.element) || !sameType(*map.keyType, *type.key)) { + failure.what = "a map<" + map.keyType->name() + ", " + map.valueType->name() + ">"; + failure.hint = kKeepsTypeHint; + return false; + } + } else { + DepthGuard guard; + if (checkDepth() > 200) { + failure.what = "a map nested too deeply"; + return false; + } + std::vector values; + values.reserve(map.entries.size()); + for (const auto& entry : map.entries) { + Value converted; + CoerceFailure itemFailure; + if (!coerceToType(entry.first, *type.key, converted, itemFailure)) { + failure.what = "a map with the key " + itemFailure.what; + failure.hint = itemFailure.hint; + return false; + } + if (!coerceToType(entry.second, *type.element, converted, itemFailure)) { + failure.what = "a map containing " + itemFailure.what; + failure.hint = itemFailure.hint; + return false; + } + values.push_back(std::move(converted)); + } + for (size_t i = 0; i < values.size(); ++i) map.entries[i].second = std::move(values[i]); + map.keyType = type.key; + map.valueType = type.element; + } + } out = value; return true; + } case TypeKind::Function: if (!value.isFunction()) break; out = value; return true; + case TypeKind::Struct: + if (value.isMap() && value.asMap().structInfo() && value.asMap().structInfo()->isA(type.structName)) { + out = value; + return true; + } + if (value.isMap() && !value.asMap().structInfo()) failure.hint = "use " + type.structName + "(...) to make one"; + break; + default: break; } diff --git a/src/types.cpp b/src/types.cpp index f96bc01..a53f79f 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -77,9 +77,10 @@ std::string TypeSpec::name() const { case TypeKind::Number: text = numKindName(num); break; case TypeKind::Char: text = "char"; break; case TypeKind::String: text = "string"; break; - case TypeKind::List: text = "list"; break; - case TypeKind::Map: text = "map"; break; + case TypeKind::List: text = element ? "list<" + element->name() + ">" : "list"; break; + case TypeKind::Map: text = element ? "map<" + key->name() + ", " + element->name() + ">" : "map"; break; case TypeKind::Function: text = "func"; break; + case TypeKind::Struct: text = structName; break; } if (nullable) text += '?'; return text; @@ -110,6 +111,9 @@ bool lookupType(const std::string& name, TypeSpec& out) { {"list", simple(TypeKind::List)}, {"map", simple(TypeKind::Map)}, {"func", simple(TypeKind::Function)}, {"any", simple(TypeKind::Any)}, {"void", simple(TypeKind::Void)}, + // C names for some of the types above. + {"double", number(NumKind::F64)}, {"long", number(NumKind::I64)}, + {"short", number(NumKind::I16)}, }; }(); auto it = types.find(name); @@ -118,4 +122,21 @@ bool lookupType(const std::string& name, TypeSpec& out) { return true; } +bool sameType(const TypeSpec& a, const TypeSpec& b) { + if (a.kind != b.kind || a.nullable != b.nullable) return false; + if (a.kind == TypeKind::Number && a.num != b.num) return false; + if (a.kind == TypeKind::Struct && a.structName != b.structName) return false; + if ((a.element == nullptr) != (b.element == nullptr) || (a.key == nullptr) != (b.key == nullptr)) return false; + if (a.element && !sameType(*a.element, *b.element)) return false; + if (a.key && !sameType(*a.key, *b.key)) return false; + return true; +} + +TypeSpec structType(const std::string& name) { + TypeSpec spec; + spec.kind = TypeKind::Struct; + spec.structName = name; + return spec; +} + } // namespace ascend diff --git a/src/value.cpp b/src/value.cpp index 470e442..b68b56a 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -5,6 +5,7 @@ #include "utf8.hpp" #include +#include #include #include #include @@ -68,14 +69,26 @@ void write(std::string& out, const Value& value, bool repr, std::vectorstructType && activeValueHooks() != nullptr) { + std::string custom; // a to_string() method + if (activeValueHooks()->customString(value, custom)) { + out += custom; + return; + } + } checkNesting(static_cast(active.size())); active.push_back(map); + if (map->structInfo()) out += map->structInfo()->name; // Point{x: 1, y: 2} out += '{'; bool first = true; for (const auto& entry : map->entries) { if (!first) out += ", "; first = false; - write(out, entry.first, true, active); + if (map->structInfo() && entry.first.isString()) { + out += entry.first.asString(); + } else { + write(out, entry.first, true, active); + } out += ": "; write(out, entry.second, true, active); } @@ -88,6 +101,8 @@ void write(std::string& out, const Value& value, bool repr, std::vector"; + } else if (fn.kind == Callable::Kind::Struct) { + out += ""; } else { out += name.empty() ? "" : ""; } @@ -128,7 +143,12 @@ bool equalAt(const Value& a, const Value& b, int depth) { checkNesting(depth); const MapObj& x = a.asMap(); const MapObj& y = b.asMap(); + bool custom = false; // an operator== method + if (x.structType && activeValueHooks() != nullptr && activeValueHooks()->customEqual(a, b, custom)) { + return custom; + } if (x.size() != y.size()) return false; + if ((x.structInfo() ? x.structInfo()->name : "") != (y.structInfo() ? y.structInfo()->name : "")) return false; for (const auto& entry : x.entries) { const Value* other = y.find(entry.first); if (other == nullptr || !equalAt(entry.second, *other, depth + 1)) return false; @@ -158,7 +178,13 @@ int compareAt(const Value& a, const Value& b, int depth) { } return x.size() < y.size() ? -1 : (x.size() > y.size() ? 1 : 0); } - throw RuntimeError(std::string("can't compare ") + a.typeName() + " with " + b.typeName()); + int custom = 0; // an operator< method + if (a.isMap() && a.asMap().structType && activeValueHooks() != nullptr && + activeValueHooks()->customCompare(a, b, custom)) { + return custom; + } + throw RuntimeError("can't compare " + aTypeName(a) + " with " + aTypeName(b) + + (a.isMap() && a.asMap().structType ? " (give the struct an operator< method to sort it)" : "")); } void appendEscaped(std::string& out, char c, char quote) { @@ -373,6 +399,7 @@ bool MapObj::remove(const Value& key, Value* removed) { } void MapObj::forEachChild(const std::function& visit) const { + if (structType) visit(structType.get()); for (const auto& entry : entries) { if (GcObject* child = entry.first.gcObject()) visit(child); if (GcObject* child = entry.second.gcObject()) visit(child); @@ -382,28 +409,60 @@ void MapObj::forEachChild(const std::function& visit) const { void MapObj::clearReferences() { entries.clear(); index_.clear(); + structType.reset(); } void FunctionObj::forEachChild(const std::function& visit) const { if (closure) visit(closure.get()); } +ValueHooks*& activeValueHooks() { + static thread_local ValueHooks* hooks = nullptr; + return hooks; +} + +const Value* structMethod(const Value& value, const std::string& name) { + if (!value.isMap() || !value.asMap().structType) return nullptr; + return value.asMap().structType->method(name); +} + +void StructType::forEachChild(const std::function& visit) const { + if (closure) visit(closure.get()); + if (parent) visit(parent.get()); + for (const auto& entry : methods) { + if (GcObject* child = entry.second.gcObject()) visit(child); + } +} + // --------------------------------------------------------------------------- // Formatting and comparison std::string formatFloat(double value, bool singlePrecision) { if (std::isnan(value)) return "nan"; if (std::isinf(value)) return value > 0 ? "inf" : "-inf"; - // Use the shortest representation that reads back as the same number. - char buffer[32]; - const int maxPrecision = singlePrecision ? 9 : 17; - for (int precision = 1; precision <= maxPrecision; ++precision) { - std::snprintf(buffer, sizeof buffer, "%.*g", precision, value); + // Use the fewest significant digits that read back as the same number. + char buffer[400]; + const int maxDigits = singlePrecision ? 9 : 17; + int digits = 1; + for (; digits <= maxDigits; ++digits) { + std::snprintf(buffer, sizeof buffer, "%.*e", digits - 1, value); double parsed = std::strtod(buffer, nullptr); if ((singlePrecision ? roundToF32(parsed) : parsed) == value) break; } - std::string text = buffer; - if (text.find_first_of(".eE") == std::string::npos) text += ".0"; // keep floats looking like floats + if (digits > maxDigits) digits = maxDigits; + // Like Python: plain notation unless the exponent is very small or large + // (20.0, not 2e+01; 1e+16 and 1e-05 stay as they are). + const char* e = std::strchr(buffer, 'e'); + int exponent = e != nullptr ? std::atoi(e + 1) : 0; + std::string text; + if (exponent >= -4 && exponent < 16) { + std::snprintf(buffer, sizeof buffer, "%.*f", std::max(0, digits - 1 - exponent), value); + text = buffer; + } else { + std::snprintf(buffer, sizeof buffer, "%.*g", digits, value); + text = buffer; + } + if (text.find_first_of(".eEn") == std::string::npos) text += ".0"; // keep floats looking like floats return text; } @@ -425,9 +484,19 @@ std::string quoteChar(char32_t c) { return out; } +std::string typeNameOf(const Value& value) { + if (value.isMap() && value.asMap().structInfo()) return value.asMap().structInfo()->name; + return value.typeName(); +} + std::string aTypeName(const Value& value) { - std::string name = value.typeName(); + std::string name = typeNameOf(value); if (value.isNull()) return "null"; + if (value.isMap() && value.asMap().structInfo()) { + char first = static_cast(std::tolower(static_cast(name[0]))); + bool vowel = first == 'a' || first == 'e' || first == 'i' || first == 'o' || first == 'u'; + return (vowel ? "an " : "a ") + name; + } bool vowelSound = name[0] == 'a' || name[0] == 'e' || name[0] == 'i' || name[0] == 'o' || (name[0] == 'f' && name.size() > 1 && name[1] >= '0' && name[1] <= '9'); // "an f32" return (vowelSound ? "an " : "a ") + name; @@ -435,6 +504,7 @@ std::string aTypeName(const Value& value) { std::string functionName(const Callable& callable) { if (callable.kind == Callable::Kind::Native) return static_cast(callable).name; + if (callable.kind == Callable::Kind::Struct) return static_cast(callable).info->name; return static_cast(callable).name; } diff --git a/tests/interpreter_tests.cpp b/tests/interpreter_tests.cpp index d91311a..de6dece 100644 --- a/tests/interpreter_tests.cpp +++ b/tests/interpreter_tests.cpp @@ -303,7 +303,7 @@ TEST(interp_list_builtins) { " reduce(nums, func(a, b) { return a + b }), reduce([], func(a, b) { return a }, 0))", "[1, 4, 9, 16] [2, 4] 10 0\n"); CHECK_ERROR("pop([])", "pop() from an empty list"); - CHECK_ERROR("sort([1, \"a\"])", "can't compare string with int"); + CHECK_ERROR("sort([1, \"a\"])", "can't compare a string with an int"); } TEST(interp_maps) { diff --git a/tests/language_tests.cpp b/tests/language_tests.cpp new file mode 100644 index 0000000..6acdc73 --- /dev/null +++ b/tests/language_tests.cpp @@ -0,0 +1,673 @@ +// Tests for C-style syntax, import statements, structs, enums and the +// builtins added with them. + +#include "interpreter.hpp" +#include "runner.hpp" +#include "test_framework.hpp" + +#include +#include +#include +#include +#include +#include + +using namespace ascend; +namespace fs = std::filesystem; + +namespace { + +struct Result { + RunStatus status; + int exitCode; + std::string out; + std::string err; +}; + +Result run(const std::string& code, const std::vector& programArgs = {}) { + std::ostringstream out; + std::ostringstream err; + std::istringstream in; + RunResult result; + { + Interpreter interpreter(out, in); + std::vector args; + for (const auto& arg : programArgs) args.push_back(Value::fromString(arg)); + interpreter.defineGlobal("args", Value::fromList(interpreter.newList(std::move(args)))); + RunOptions options; + options.filename = "test.asc"; + result = runSource(interpreter, code, options, err); + } + return {result.status, result.exitCode, out.str(), err.str()}; +} + +std::string output(const std::string& code) { + Result result = run(code); + if (result.status != RunStatus::Ok) return "UNEXPECTED ERROR:\n" + result.err; + return result.out; +} + +std::string error(const std::string& code) { + Result result = run(code); + return result.status == RunStatus::Ok ? "" : result.err; +} + +bool contains(const std::string& text, const std::string& fragment) { return text.find(fragment) != std::string::npos; } + +// Files in the working directory that are deleted when the test ends. +struct TempFiles { + void add(const std::string& path, const std::string& contents) { + fs::path file(path); + if (file.has_parent_path()) { + fs::create_directories(file.parent_path()); + folders.push_back(file.parent_path().string()); + } + std::ofstream(file, std::ios::binary) << contents; + files.push_back(path); + } + ~TempFiles() { + std::error_code ignored; + for (const auto& file : files) fs::remove(file, ignored); + for (const auto& folder : folders) fs::remove_all(folder, ignored); + } + std::vector files; + std::vector folders; +}; + +} // namespace + +#define CHECK_OUTPUT(code, expected) CHECK_EQ(output(code), std::string(expected)) +#define CHECK_ERROR(code, fragment) \ + do { \ + std::string message_ = error(code); \ + if (!contains(message_, fragment)) { \ + throw ::asctest::Failure{std::string(__FILE__) + ":" + std::to_string(__LINE__) + \ + ": expected an error containing \"" + (fragment) + \ + "\"\n got: " + (message_.empty() ? "(no error)" : message_)}; \ + } \ + } while (0) + +// --------------------------------------------------------------------------- +// C-style statements + +TEST(lang_c_style_for_loops) { + CHECK_OUTPUT("for (i32 i = 0; i < 3; i++) { print(i) }", "0\n1\n2\n"); + CHECK_OUTPUT("for i := 3; i > 0; i-- { print(i) }", "3\n2\n1\n"); + CHECK_OUTPUT("for (i32 a = 0, b = 6; a < b; a += 2, b -= 2) print(a, b)", "0 6\n2 4\n"); + CHECK_OUTPUT("n := 0\nfor (;;) { n++\n if (n == 4) break }\nprint(n)", "4\n"); + CHECK_OUTPUT("n := 0\nfor { n += 2\n if (n > 5) break }\nprint(n)", "6\n"); + // continue still runs the step. + CHECK_OUTPUT("for (i32 i = 0; i < 5; i++) {\n if (i % 2 == 0) continue\n print(i)\n}", "1\n3\n"); + // The loop variable belongs to the loop... + CHECK_ERROR("for (i32 i = 0; i < 1; i++) {}\nprint(i)", "undefined variable 'i'"); + // ...unless it was declared before it. + CHECK_OUTPUT("i := 0\nfor (i = 0; i < 3; i++) {}\nprint(i)", "3\n"); + // Each iteration's functions see that iteration's value, like `let` in JavaScript. + CHECK_OUTPUT("fns := []\nfor (i32 k = 0; k < 3; k++) fns.push(func() { return k })\nprint(map(fns, func(f) { return f() }))", + "[0, 1, 2]\n"); + CHECK_ERROR("for (u8 i = 250; i < 300; i++) {}", "integer overflow: 255 + 1 doesn't fit in u8"); + CHECK_OUTPUT("for (string s : [\"a\", 'b']) print(s, type(s))", "a string\nb string\n"); + CHECK_ERROR("for (i32 n in [1, \"x\"]) {}", "'n' is declared as i32, so it can't hold a string"); +} + +TEST(lang_increment_and_decrement) { + CHECK_OUTPUT("i32 x = 5\nprint(x++, x, ++x, x--, --x, x)", "5 6 7 7 5 5\n"); + CHECK_OUTPUT("items := [1, 2]\nitems[0]++\n--items[1]\nprint(items)", "[2, 1]\n"); + CHECK_OUTPUT("m := {count: 1}\nm.count++\nm[\"count\"] += 1\nprint(m.count)", "3\n"); + CHECK_OUTPUT("i := 0\nitems := [10, 20, 30]\nprint(items[i++], items[i++], i)", "10 20 2\n"); + CHECK_OUTPUT("char c = 'a'\nc++\nprint(c)\nf64 f = 1.5\nf--\nprint(f)", "b\n0.5\n"); + CHECK_ERROR("u8 b = 255\nb++", "integer overflow: 255 + 1 doesn't fit in u8"); + CHECK_ERROR("const N = 1\nN++", "can't change 'N': it's a constant"); + CHECK_ERROR("s := \"a\"\ns++", "can't use '++' on a string value"); + CHECK_ERROR("missing++", "undefined variable 'missing'"); +} + +TEST(lang_bodies_without_braces) { + CHECK_OUTPUT("x := 3\nif (x > 2) print(\"big\")\nelse print(\"small\")", "big\n"); + CHECK_OUTPUT("x := 1\nif (x > 2) print(\"big\") else print(\"small\")", "small\n"); + CHECK_OUTPUT("x := 0\nif (x > 0)\n print(\"positive\")\nelse if (x < 0)\n print(\"negative\")\nelse\n print(\"zero\")", + "zero\n"); + CHECK_OUTPUT("i := 0\nwhile (i < 3) i++\nprint(i)", "3\n"); + CHECK_OUTPUT("total := 0\nfor (x in [1, 2, 3]) total += x\nprint(total)", "6\n"); + CHECK_OUTPUT("i := 0\nwhile (i++ < 5);\nprint(i)", "6\n"); + CHECK_OUTPUT("done := true\ncount := 0\nif (done) ++count\nprint(count)", "1\n"); + CHECK_OUTPUT("if (1 < 2)\n{\n print(\"allman\")\n}\nelse\n{\n print(\"no\")\n}", "allman\n"); +} + +TEST(lang_do_while) { + CHECK_OUTPUT("i := 10\ndo {\n print(i)\n i++\n} while (i < 3)", "10\n"); + CHECK_OUTPUT("i := 0\ndo { i++ } while (i < 3)\nprint(i)", "3\n"); + CHECK_OUTPUT("i := 0\ndo i += 5; while (i < 12)\nprint(i)", "15\n"); + CHECK_OUTPUT("i := 0\ndo {\n i++\n if (i == 2) continue\n if (i == 4) break\n print(i)\n} while (true)", "1\n3\n"); +} + +TEST(lang_switch) { + const char* program = + "func describe(n) {\n" + " switch (n) {\n" + " case 1:\n" + " return \"one\"\n" + " case 2, 3:\n" + " return \"two or three\"\n" + " default:\n" + " return \"many\"\n" + " }\n" + "}\n" + "print(describe(1), describe(3), describe(9))"; + CHECK_OUTPUT(program, "one two or three many\n"); + // Like C, cases fall through until a break. + CHECK_OUTPUT("switch (2) {\ncase 1:\n print(1)\ncase 2:\n print(2)\ncase 3:\n print(3)\n break\ncase 4:\n print(4)\n}", "2\n3\n"); + CHECK_OUTPUT("switch (9) {\ncase 1:\n print(1)\n}\nprint(\"none\")", "none\n"); + CHECK_OUTPUT("switch \"b\" { case \"a\": print(\"A\"); break; case \"b\": print(\"B\"); break }", "B\n"); + CHECK_OUTPUT("switch 5 {\ndefault:\n print(\"default\")\ncase 1:\n print(\"one\")\n}", "default\none\n"); + // break leaves the switch, continue goes to the loop around it. + CHECK_OUTPUT("for i in range(4) {\n switch (i) {\n case 1: continue\n case 3: break\n }\n print(i)\n}", "0\n2\n3\n"); + CHECK_OUTPUT("switch 1 {\ncase 1:\n x := 5\n print(x)\n}", "5\n"); + CHECK_ERROR("switch 1 { case 1: continue }", "'continue' can only be used inside a loop"); + CHECK_ERROR("break", "'break' can only be used inside a loop or switch"); +} + +TEST(lang_casts_and_type_aliases) { + CHECK_OUTPUT("print((f64) 7 / 2, (i32) 3.9, (string) 5 + \"!\", (char) 65, (bool) 0)", "3.5 3 5! A false\n"); + CHECK_OUTPUT("double d = 2\nlong l = 5\nshort s = 3\nprint(type(d), type(l), type(s), d)", "f64 i64 i16 2.0\n"); + CHECK_OUTPUT("print(double(\"1.5\"), long(2.0), short(-7))", "1.5 2 -7\n"); + CHECK_ERROR("short s = 40000", "can't hold 40000 (i16 is -32768 to 32767)"); +} + +TEST(lang_auto) { + CHECK_OUTPUT("auto n = 5\nauto f = 2.5f32\nauto s = \"x\"\nprint(type(n), type(f), type(s))", "int f32 string\n"); + CHECK_ERROR("auto n = 5\nn = \"five\"", "'n' is declared as int, so it can't hold a string"); + CHECK_ERROR("auto n = 5\nn = 2.5", "'n' is declared as int, so it can't hold 2.5"); + CHECK_OUTPUT("const auto LIMIT = 10u8\nprint(type(LIMIT))", "u8\n"); + CHECK_OUTPUT("auto anything = null\nanything = 5\nprint(anything)", "5\n"); +} + +TEST(lang_main_function) { + Result result = run("int main() {\n print(\"in main\")\n return 3\n}\nprint(\"top level first\")"); + CHECK(result.status == RunStatus::Exited); + CHECK_EQ(result.exitCode, 3); + CHECK_EQ(result.out, "top level first\nin main\n"); + + Result withArgs = run("void main(list args) { print(args) }", {"a", "b"}); + CHECK_EQ(withArgs.out, "[\"a\", \"b\"]\n"); + CHECK(withArgs.status == RunStatus::Ok); + + Result cStyle = run("int main(int argc, list argv) {\n printf(\"%d %s\\n\", argc, argv[0])\n return 0\n}", {"x"}); + CHECK_EQ(cStyle.out, "1 x\n"); + CHECK(cStyle.status == RunStatus::Ok); + + // A program that calls main() itself isn't run twice. + CHECK_OUTPUT("func main() { print(\"once\") }\nmain()", "once\n"); + CHECK_ERROR("int main() { return 1 / 0 }", "division by zero"); +} + +// --------------------------------------------------------------------------- +// Import statements + +TEST(lang_import_statements) { + TempFiles files; + files.add("ascend_test_geo.asc", "f64 square(f64 s) { return s * s }\nVERSION := \"1.0\"\n_hidden := 1\n"); + files.add("ascend_test_lib/shapes.asc", "struct Box { f64 w; f64 h = 1 }\nf64 area(Box b) { return b.w * b.h }\n"); + + CHECK_OUTPUT("import ascend_test_geo\nprint(ascend_test_geo.square(3))", "9.0\n"); + CHECK_OUTPUT("import ascend_test_geo as geo, \"ascend_test_geo.asc\" as again\nprint(geo.VERSION, geo == again)", + "1.0 true\n"); + CHECK_OUTPUT("import ascend_test_lib.shapes\nprint(shapes.area(shapes.Box(2, 3)))", "6.0\n"); + CHECK_OUTPUT("from ascend_test_lib.shapes import Box, area as box_area\nBox b = Box(4)\nprint(box_area(b), b)", + "4.0 Box{w: 4.0, h: 1.0}\n"); + CHECK_OUTPUT("from ascend_test_geo import *\nprint(square(2), VERSION)", "4.0 1.0\n"); + CHECK_ERROR("from ascend_test_geo import _hidden", "\"ascend_test_geo\" has no '_hidden' (it has: VERSION, square)"); + CHECK_ERROR("import ascend_test_nothing", "no installed package named 'ascend_test_nothing'"); + CHECK_OUTPUT("func load() {\n import ascend_test_geo as g\n return g.square(5)\n}\nprint(load())", "25.0\n"); + CHECK_ERROR("const geo = 1\nimport ascend_test_geo as geo", "'geo' is a constant and can't be declared again"); +} + +// --------------------------------------------------------------------------- +// Structs and enums + +TEST(lang_structs) { + const char* point = "struct Point {\n f64 x\n f64 y = 0\n string label = \"p\"\n}\n"; + CHECK_OUTPUT(std::string(point) + "p := Point(1, 2)\nprint(p, p.x, type(p))", "Point{x: 1.0, y: 2.0, label: \"p\"} 1.0 Point\n"); + CHECK_OUTPUT(std::string(point) + "print(Point(), Point(5).y)", "Point{x: 0.0, y: 0.0, label: \"p\"} 0.0\n"); + CHECK_OUTPUT(std::string(point) + "Point p\np.x = 3\np.y += 1\np.x++\nprint(p)", "Point{x: 4.0, y: 1.0, label: \"p\"}\n"); + CHECK_ERROR(std::string(point) + "p := Point()\np.x = \"far\"", "field 'x' of Point is f64, so it can't hold a string"); + CHECK_ERROR(std::string(point) + "p := Point()\np.z = 1", "Point has no field 'z' (its fields are: x, y, label)"); + CHECK_ERROR(std::string(point) + "p := Point()\np[\"z\"] = 1", "Point has no field 'z'"); + CHECK_ERROR(std::string(point) + "print(Point().z)", "Point has no field 'z'"); + CHECK_ERROR(std::string(point) + "Point(1, 2, \"a\", 4)", "Point() takes at most 3 arguments (x, y, label), but got 4"); + CHECK_ERROR(std::string(point) + "Point(\"left\")", "Point(): field 'x' is f64, so it can't hold a string"); + CHECK_ERROR(std::string(point) + "remove(Point(), \"x\")", "remove() can't remove fields from a Point"); + + // Struct types in declarations, parameters and return values. + const char* typed = "struct Point { f64 x; f64 y }\n" + "struct Size { f64 w; f64 h }\n" + "f64 distance(Point a, Point b) { return hypot(b.x - a.x, b.y - a.y) }\n" + "Point? find(list points, f64 x) {\n" + " for (Point p : points) if (p.x == x) return p\n" + " return null\n" + "}\n"; + CHECK_OUTPUT(std::string(typed) + "print(distance(Point(0, 0), Point(3, 4)))", "5.0\n"); + CHECK_OUTPUT(std::string(typed) + "print(find([Point(1, 1), Point(2, 5)], 2), find([], 1))", "Point{x: 2.0, y: 5.0} null\n"); + CHECK_ERROR(std::string(typed) + "distance(Size(1, 1), Point(0, 0))", + "distance() expects 'a' to be Point, but got a Size"); + CHECK_ERROR(std::string(typed) + "distance({x: 1, y: 1}, Point(0, 0))", "but got a map (use Point(...) to make one)"); + CHECK_ERROR(std::string(typed) + "Point p = Size(1, 2)", "'p' is declared as Point, so it can't hold a Size"); + + // Method-style calls find functions that take the struct first. + CHECK_OUTPUT(std::string(typed) + "print(Point(0, 0).distance(Point(0, 2)))", "2.0\n"); + // Nested structs start with their own defaults; copies and equality work. + CHECK_OUTPUT("struct A { i32 n = 7 }\nstruct B { A inner; A? other }\nB b\nprint(b, copy(b) == b, type(copy(b)))", + "B{inner: A{n: 7}, other: null} true B\n"); + CHECK_OUTPUT("struct A { i32 n }\nstruct B { i32 n }\nprint(A(1) == A(1), A(1) == B(1), A(1) == {n: 1})", "true false false\n"); + // Structs can be used before they're declared, like functions. + CHECK_OUTPUT("print(Later(2))\nstruct Later { i32 n }", "Later{n: 2}\n"); + // Each value gets its own fresh default. + CHECK_OUTPUT("struct Bag { list items = [] }\na := Bag()\nb := Bag()\na.items.push(1)\nprint(b.items)", "[]\n"); + + CHECK_ERROR("struct P { f64 x; f64 x }", "struct P has two fields named 'x'"); + CHECK_ERROR("Pointt p", "unknown type 'Pointt'"); + CHECK_ERROR("struct T { u8 small = 300 }", "field 'small' is declared as u8, so its default can't be 300"); +} + +TEST(lang_enums) { + CHECK_OUTPUT("enum Color { Red, Green = 5, Blue }\nprint(Color.Red, Color.Green, Color.Blue, Color)", + "0 5 6 {\"Red\": 0, \"Green\": 5, \"Blue\": 6}\n"); + CHECK_OUTPUT("enum Level {\n Low\n High\n}\nswitch (Level.High) {\ncase Level.Low: print(\"low\"); break\ncase Level.High: print(\"high\")\n}", + "high\n"); + CHECK_ERROR("enum Color { Red }\nColor.Red = 2", "the members of enum Color can't be changed"); + CHECK_ERROR("enum Color { Red }\nprint(Color.Pink)", "enum Color has no member 'Pink'"); + CHECK_ERROR("enum Color { Red }\nColor = 5", "can't change 'Color': it's a constant"); + CHECK_ERROR("enum E { A, A }", "enum E lists 'A' more than once"); +} + +// --------------------------------------------------------------------------- +// Builtins + +TEST(lang_printf) { + CHECK_OUTPUT("printf(\"%d|%5d|%-5d|%05d|%+d|% d\\n\", 1, 2, 3, 4, 5, 6)", "1| 2|3 |00004|+5| 6\n"); + CHECK_OUTPUT("printf(\"%x %X %#x %o %#o %b %c %c\\n\", 255, 255, 255, 8, 8, 5, 65, 'z')", "ff FF 0xff 10 010 101 A z\n"); + CHECK_OUTPUT("printf(\"%f|%.2f|%8.3f|%-8.1f|%e|%g\\n\", 3.14159, 3.14159, 2.5, 2.5, 12345.678, 0.0001)", + "3.141590|3.14| 2.500|2.5 |1.234568e+04|0.0001\n"); + CHECK_OUTPUT("printf(\"%s|%10s|%-6s|%.3s|%s\\n\", \"hi\", \"right\", \"left\", \"abcdef\", [1, 2])", + "hi| right|left |abc|[1, 2]\n"); + CHECK_OUTPUT("print(sprintf(\"%*d|%-*d|%.*f|%%|%ld|%lld|%zu\", 4, 7, 3, 8, 1, 2.25, 1, 2, 3))", " 7|8 |2.2|%|1|2|3\n"); + CHECK_OUTPUT("printf(\"%05.1f|%-+5d|%d\\n\", -2.5, 3, 'a')", "-02.5|+3 |97\n"); + CHECK_ERROR("printf(\"%d\", 2.5)", "printf(): %d needs an integer, but got 2.5 (use %f for floats"); + CHECK_ERROR("printf(\"%d %d\", 1)", "no value for %d"); + CHECK_ERROR("printf(\"%d\", 1, 2)", "2 values were given, but the format only uses 1"); + CHECK_ERROR("printf(\"%q\", 1)", "unknown conversion '%q'"); + CHECK_ERROR("printf(\"50%\")", "the format ends in the middle of '%'"); +} + +TEST(lang_regex) { + CHECK_OUTPUT("print(regex_match(\"2024-01-05\", \"\\\\d{4}-\\\\d{2}-\\\\d{2}\"), regex_match(\"x2024\", \"\\\\d+\"))", + "true false\n"); + CHECK_OUTPUT("print(regex_search(\"id=42; id=7\", \"id=(\\\\d+)\"), regex_search(\"x\", \"y\"))", "[\"id=42\", \"42\"] null\n"); + CHECK_OUTPUT("print(regex_search(\"ab\", \"a(x)?b\"))", "[\"ab\", null]\n"); + CHECK_OUTPUT("print(regex_find_all(\"a1b22c333\", \"\\\\d+\"))", "[\"1\", \"22\", \"333\"]\n"); + CHECK_OUTPUT("print(regex_replace(\"John Smith\", \"(\\\\w+) (\\\\w+)\", \"$2, $1\"))", "Smith, John\n"); + CHECK_OUTPUT("print(regex_split(\"a, b,c\", \",\\\\s*\"))", "[\"a\", \"b\", \"c\"]\n"); + CHECK_ERROR("regex_match(\"x\", \"(\")", "regex_match(): invalid regular expression \"(\""); +} + +TEST(lang_folders) { + TempFiles files; + files.add("ascend_test_folder/b.txt", "b"); + files.add("ascend_test_folder/a.txt", "a"); + CHECK_OUTPUT("print(is_dir(\"ascend_test_folder\"), is_dir(\"ascend_test_folder/a.txt\"), list_dir(\"ascend_test_folder\"))", + "true false [\"a.txt\", \"b.txt\"]\n"); + CHECK_OUTPUT("print(make_dir(\"ascend_test_folder/x/y\"), make_dir(\"ascend_test_folder/x/y\"), is_dir(\"ascend_test_folder/x/y\"))", + "true false true\n"); + CHECK_ERROR("list_dir(\"ascend_test_no_such_folder\")", "list_dir(): can't read the folder 'ascend_test_no_such_folder'"); +} + +TEST(lang_float_printing) { + CHECK_OUTPUT("print(20.0, 100.0, 1e15, 1e16, 0.0001, 0.00001, 123456.789, -20.0)", + "20.0 100.0 1000000000000000.0 1e+16 0.0001 1e-05 123456.789 -20.0\n"); +} + +// --------------------------------------------------------------------------- +// Methods, lambdas, comprehensions and more + +TEST(lang_struct_methods) { + const char* vec = + "struct Vec {\n" + " f64 x\n" + " f64 y = 0\n" + " f64 length() { return hypot(this.x, this.y) }\n" + " void scale(f64 k) {\n" + " this.x *= k\n" + " this.y *= k\n" + " }\n" + " func plus(other) { return Vec(this.x + other.x, this.y + other.y) }\n" + " label() { return f\"({this.x}, {this.y})\" }\n" + "}\n"; + CHECK_OUTPUT(std::string(vec) + "v := Vec(3, 4)\nprint(v.length(), v.plus(Vec(1)).label())", "5.0 (4.0, 4.0)\n"); + CHECK_OUTPUT(std::string(vec) + "v := Vec(1, 2)\nv.scale(3)\nprint(v)", "Vec{x: 3.0, y: 6.0}\n"); + CHECK_ERROR(std::string(vec) + "print(Vec(1).length)", "'length' is a method of Vec; call it: .length()"); + CHECK_ERROR(std::string(vec) + "Vec(1).size()", "Vec has no method or field 'size'"); + CHECK_ERROR(std::string(vec) + "Vec(1).scale(\"big\")", "Vec.scale() expects 'k' to be f64, but got a string"); + // Methods win over functions with the same name, and fields hold their own functions. + CHECK_OUTPUT("func size(x) { return \"global\" }\nstruct S { size() { return \"method\" } }\nprint(S().size())", "method\n"); + // init() runs when a value is made. + const char* account = + "struct Account {\n" + " string owner\n" + " f64 balance = 0\n" + " void init() {\n" + " assert(this.balance >= 0, \"balance can't be negative\")\n" + " this.owner = upper(this.owner)\n" + " }\n" + "}\n"; + CHECK_OUTPUT(std::string(account) + "print(Account(\"ada\", 5))", "Account{owner: \"ADA\", balance: 5.0}\n"); + CHECK_ERROR(std::string(account) + "Account(\"bob\", -1)", "balance can't be negative"); + CHECK_ERROR("struct P { i32 a; f64 a() { return 1 } }", "struct P already has a field or method named 'a'"); + CHECK_ERROR("struct P { void init(i32 x) { } }", "init() is called with no arguments when a P is made"); +} + +TEST(lang_lambdas) { + CHECK_OUTPUT("double := x => x * 2\nadd := (a, b) => a + b\nprint(double(4), add(2, 3))", "8 5\n"); + CHECK_OUTPUT("print(map([1, 2, 3], x => x * x), filter(range(10), n => n % 4 == 0))", "[1, 4, 9] [0, 4, 8]\n"); + CHECK_OUTPUT("print(sort([\"ccc\", \"a\", \"bb\"], s => len(s)))", "[\"a\", \"bb\", \"ccc\"]\n"); + CHECK_OUTPUT("square := (i32 n) => { return n * n }\nprint(square(7))", "49\n"); + CHECK_ERROR("square := (i32 n) => n * n\nsquare(\"x\")", "expects 'n' to be i32"); + CHECK_OUTPUT("make := n => () => n + 1\nprint(make(5)())", "6\n"); + CHECK_OUTPUT("x := 3\nmatch x {\n 3 => print(\"three\")\n else => print(\"other\")\n}", "three\n"); +} + +TEST(lang_comprehensions) { + CHECK_OUTPUT("print([x * x for x in 1..=5])", "[1, 4, 9, 16, 25]\n"); + CHECK_OUTPUT("print([x for x in range(20) if x % 7 == 0])", "[0, 7, 14]\n"); + CHECK_OUTPUT("print([[a, b] for a in 1..3 for b in \"xy\"])", "[[1, 'x'], [1, 'y'], [2, 'x'], [2, 'y']]\n"); + CHECK_OUTPUT("print({k: v * 10 for k, v in {a: 1, b: 2}})", "{\"a\": 10, \"b\": 20}\n"); + CHECK_OUTPUT("print({s: len(s) for s in [\"hi\", \"hey\"] if len(s) > 2})", "{\"hey\": 3}\n"); + CHECK_OUTPUT("print([i for i, c in \"abc\" if c != 'b'], [k for k in {x: 1, y: 2}])", "[0, 2] [\"x\", \"y\"]\n"); + // The loop variables don't leak out. + CHECK_ERROR("[x for x in [1]]\nprint(x)", "undefined variable 'x'"); + CHECK_OUTPUT("fns := [() => i for i in 0..3]\nprint([f() for f in fns])", "[0, 1, 2]\n"); +} + +TEST(lang_destructuring) { + CHECK_OUTPUT("a, b := 1, 2\na, b = b, a\nprint(a, b)", "2 1\n"); + CHECK_OUTPUT("func minmax(list xs) { return min(xs), max(xs) }\nlo, hi := minmax([5, 3, 9])\nprint(lo, hi)", "3 9\n"); + CHECK_OUTPUT("struct P { i32 x; i32 y }\nx, y := P(7, 8)\nprint(x, y)", "7 8\n"); + CHECK_OUTPUT("items := [0, 0]\nm := {}\nitems[0], m.key = \"a\", \"b\"\nprint(items, m)", "[\"a\", 0] {\"key\": \"b\"}\n"); + CHECK_ERROR("a, b := [1]", "can't unpack 1 value into 2 variables"); + CHECK_ERROR("a, b := 5", "can't unpack an int value into 2 variables (it needs a list or a struct)"); + CHECK_ERROR("i32 a = 0\nb := 0\na, b = \"x\", 1", "'a' is declared as i32, so it can't hold a string"); +} + +TEST(lang_null_operators) { + CHECK_OUTPUT("string? name = null\nprint(name ?? \"anonymous\", \"set\" ?? \"other\", 0 ?? 5, false ?? true)", + "anonymous set 0 false\n"); + CHECK_OUTPUT("user := {address: null}\nprint(user.address?.city, user?.address, null?.length())", "null null null\n"); + CHECK_OUTPUT("user := {address: {city: \"Oslo\"}}\nprint(user.address?.city ?? \"?\")", "Oslo\n"); + CHECK_OUTPUT("calls := 0\nfunc f() { calls++\n return 1 }\nx := 5 ?? f()\nprint(x, calls)", "5 0\n"); +} + +TEST(lang_ranges) { + CHECK_OUTPUT("print(0..4, 1..=3, 'a'..='d', 5..2)", "[0, 1, 2, 3] [1, 2, 3] ['a', 'b', 'c', 'd'] []\n"); + CHECK_OUTPUT("total := 0\nfor i in 1..=100 { total += i }\nprint(total)", "5050\n"); + CHECK_OUTPUT("for i in 0..1000000000 { if (i == 3) break }\nprint(\"no list was built\")", "no list was built\n"); + CHECK_OUTPUT("print(5 in 1..10, 10 in 1..10, 10 in 1..=10, 'q' in 'a'..='z', 2.5 in 0..3, \"x\" in 0..3)", + "true false true true true false\n"); + CHECK_OUTPUT("func grade(i32 score) {\n switch (score) {\n case 90..=100: return \"A\"\n case 80..90: return \"B\"\n" + " default: return \"C\"\n }\n}\nprint(grade(95), grade(85), grade(10))", + "A B C\n"); + CHECK_OUTPUT("match 15 {\n 0..10 => print(\"small\")\n 10..100 => print(\"medium\")\n}", "medium\n"); + CHECK_ERROR("print(\"a\"..\"b\")", "a range needs two integers or two chars, not a string and a string"); + CHECK_ERROR("x := 1..=100000000", "the range is too large to make a list from"); +} + +TEST(lang_variadic_and_spread) { + const char* sum = "i32 sum(i32 ...nums) {\n i32 total = 0\n for (n in nums) total += n\n return total\n}\n"; + CHECK_OUTPUT(std::string(sum) + "print(sum(), sum(1, 2, 3), sum(...[4, 5], 6))", "0 6 15\n"); + CHECK_ERROR(std::string(sum) + "sum(1, \"x\")", "sum() expects each of 'nums' to be i32, but got a string"); + CHECK_OUTPUT("func log(string level, ...parts) { print(level + \":\", join(parts, \" \")) }\nlog(\"info\", \"a\", 1, true)\nlog(\"x\")", + "info: a 1 true\nx: \n"); + CHECK_ERROR("func f(a, ...rest) { }\nf()", "f() expects at least 1 argument but got 0"); + CHECK_OUTPUT("print([0, ...[1, 2], 3, ...(4..6)])", "[0, 1, 2, 3, 4, 5]\n"); + CHECK_OUTPUT("defaults := {color: \"red\", size: 1}\nprint({...defaults, size: 2})", "{\"color\": \"red\", \"size\": 2}\n"); + CHECK_OUTPUT("xs := [1, 2]\nprint([...xs, ...xs], max(...xs))", "[1, 2, 1, 2] 2\n"); + CHECK_ERROR("print(...5)", "'...' needs a list to spread into the arguments, not an int"); +} + +TEST(lang_named_arguments) { + const char* greet = "func greet(string name, string greeting = \"Hello\", string mark = \"!\") {\n" + " return greeting + \", \" + name + mark\n}\n"; + CHECK_OUTPUT(std::string(greet) + "print(greet(\"Ada\", mark: \"?\"), greet(greeting: \"Hi\", name: \"Bob\"))", + "Hello, Ada? Hi, Bob!\n"); + CHECK_ERROR(std::string(greet) + "greet(\"x\", nope: 1)", + "greet() has no parameter named 'nope' (its parameters are: name, greeting, mark)"); + CHECK_ERROR(std::string(greet) + "greet(mark: \"!\")", "greet() needs a value for 'name'"); + CHECK_ERROR(std::string(greet) + "greet(\"x\", name: \"y\")", "greet() got two values for 'name'"); + CHECK_ERROR("print(1, sep: \"\")", "print() doesn't take arguments by name"); + CHECK_OUTPUT("struct Box { f64 w = 1; f64 h = 1; string label = \"box\" }\nprint(Box(h: 3, label: \"tall\"))", + "Box{w: 1.0, h: 3.0, label: \"tall\"}\n"); + CHECK_ERROR("struct Box { f64 w }\nBox(depth: 2)", "Box() has no field named 'depth' (its fields are: w)"); +} + +TEST(lang_finally) { + const char* risky = "func risky(n) {\n try {\n if (n == 0) throw \"zero!\"\n return \"ok \" + str(n)\n" + " } catch e {\n return \"caught \" + e\n } finally {\n print(\"cleanup\", n)\n }\n}\n"; + CHECK_OUTPUT(std::string(risky) + "print(risky(1))\nprint(risky(0))", "cleanup 1\nok 1\ncleanup 0\ncaught zero!\n"); + CHECK_OUTPUT("for i in range(3) {\n try { if (i == 1) break } finally { print(\"finally\", i) }\n}", "finally 0\nfinally 1\n"); + CHECK_OUTPUT("try {\n try { x := 1 / 0 } finally { print(\"inner\") }\n} catch e { print(\"outer:\", e) }", + "inner\nouter: division by zero\n"); + // The function's result survives calls made in finally. + CHECK_OUTPUT("func helper() { return \"helper\" }\nfunc f() {\n try { return \"result\" } finally { helper() }\n}\nprint(f())", + "result\n"); + CHECK_OUTPUT("func f() { try { throw \"x\" } finally { return \"replaced\" } }\nprint(f())", "replaced\n"); + // An error that escapes still points at where it happened. + CHECK_ERROR("try {\n x := 1 / 0\n} finally {\n y := 2\n}", "test.asc:2:12: runtime error: division by zero"); +} + +TEST(lang_strings_raw_and_multiline) { + CHECK_OUTPUT("print(r\"C:\\new\\d+\")", "C:\\new\\d+\n"); + CHECK_OUTPUT("print(regex_find_all(\"1-2 and 33-44\", r\"\\d+-\\d+\"))", "[\"1-2\", \"33-44\"]\n"); + CHECK_OUTPUT("text := \"\"\"\nLine one\n Line \"two\"\n\"\"\"\nprint(repr(text))", "\"Line one\\n Line \\\"two\\\"\\n\"\n"); +} + +TEST(lang_list_helpers) { + CHECK_OUTPUT("print(group_by([\"apple\", \"bob\", \"avocado\"], w => str(w[0])))", + "{\"a\": [\"apple\", \"avocado\"], \"b\": [\"bob\"]}\n"); + CHECK_OUTPUT("print(count_by([1, 2, 3, 4, 5], n => n % 2 == 0))", "{false: 3, true: 2}\n"); + CHECK_OUTPUT("print(chunk(1..=7, 3), flat_map([1, 2], x => [x, x * 10]))", "[[1, 2, 3], [4, 5, 6], [7]] [1, 10, 2, 20]\n"); + CHECK_OUTPUT("print(mean([1, 2, 3, 4]), median([5, 1, 3]), median([4, 1, 3, 2]))", "2.5 3.0 2.5\n"); + CHECK_OUTPUT("words := [\"bob\", \"avocado\", \"cat\"]\nprint(min_by(words, len), max_by(words, len))", "bob avocado\n"); + CHECK_ERROR("mean([])", "mean() of an empty list"); + CHECK_ERROR("median([1, \"x\"])", "median() needs a list of numbers, but it contains a string"); + CHECK_ERROR("chunk([1], 0)", "chunk() size must be at least 1"); +} + +// --------------------------------------------------------------------------- +// Tracebacks, inheritance, operators, typed collections + +TEST(lang_tracebacks) { + const char* program = + "f64 divide(f64 a, f64 b) {\n" + " if (b == 0) throw \"can't divide by zero\"\n" + " return a / b\n" + "}\n" + "func average(list xs) { return divide(sum(xs), len(xs)) }\n" + "print(map([[1, 2], []], g => average(g)))\n"; + std::string err = error(program); + CHECK(contains(err, "test.asc:2:17: runtime error: uncaught error: can't divide by zero")); + CHECK(contains(err, " in divide(), called from test.asc:5:32\n")); + CHECK(contains(err, " in average(), called from test.asc:6:30\n")); + CHECK(contains(err, " in an anonymous function, called by a builtin\n")); + CHECK(contains(err, " in map(), called from test.asc:6:7\n")); + // Caught errors leave no traceback behind. + std::string caught = error("func f() { return [][0] }\ntry { f() } catch e { }\nx := 1 / 0"); + CHECK(contains(caught, "division by zero")); + CHECK(!contains(caught, " in f()")); + // Deep recursion is summarized. + std::string deep = error("func f(n) { return n == 0 ? 1 / 0 : f(n - 1) }\nf(50)"); + CHECK(contains(deep, " in f(), called from test.asc:1:37\n (and 49 more times)\n in f(), called from test.asc:2:1\n")); +} + +TEST(lang_inheritance) { + const char* animals = + "struct Animal {\n" + " string name\n" + " i32 legs = 4\n" + " string sound() { return \"...\" }\n" + " string describe() { return f\"{this.name} says {this.sound()}\" }\n" + "}\n" + "struct Dog : Animal {\n" + " string breed = \"mutt\"\n" + " string sound() { return \"Woof\" }\n" + " string describe() { return super.describe() + \" (\" + this.breed + \")\" }\n" + "}\n" + "struct Bird : Animal {\n" + " void init() { this.legs = 2 }\n" + " string sound() { return \"Tweet\" }\n" + "}\n"; + std::string base = animals; + CHECK_OUTPUT(base + "d := Dog(\"Rex\", breed: \"beagle\")\nprint(d.describe(), d.legs)", "Rex says Woof (beagle) 4\n"); + CHECK_OUTPUT(base + "b := Bird(\"Tweety\")\nprint(b.describe(), b.legs, b)", + "Tweety says Tweet 2 Bird{name: \"Tweety\", legs: 2}\n"); + CHECK_OUTPUT(base + "Animal a = Dog(\"Rex\")\nfunc greet(Animal x) { return x.name }\nprint(type(a), greet(Bird(\"B\")))", + "Dog B\n"); + CHECK_OUTPUT(base + "d := Dog(\"Rex\")\nprint(is_a(d, Animal), is_a(d, Dog), is_a(d, Bird), is_a(d, \"Animal\"), is_a(5, \"int\"))", + "true true false true true\n"); + CHECK_ERROR(base + "Dog d = Bird(\"B\")", "'d' is declared as Dog, so it can't hold a Bird"); + CHECK_ERROR("struct A { i32 x }\nstruct B : A { i32 x }", "struct B can't declare 'x' again: it already has it from A"); + CHECK_ERROR("struct B : Missing { }", "unknown type 'Missing'"); + CHECK_ERROR("struct A { f() { return super.f() } }\nA().f()", "'super' can only be used in the methods of a struct that extends another"); +} + +TEST(lang_operator_overloading) { + const char* vec = + "struct Vec {\n" + " f64 x\n" + " f64 y\n" + " Vec operator+(Vec o) { return Vec(this.x + o.x, this.y + o.y) }\n" + " Vec operator*(f64 k) { return Vec(this.x * k, this.y * k) }\n" + " bool operator==(Vec o) { return this.x == o.x and this.y == o.y }\n" + " bool operator<(Vec o) { return this.length() < o.length() }\n" + " f64 operator[](i32 i) { return i == 0 ? this.x : this.y }\n" + " f64 length() { return hypot(this.x, this.y) }\n" + " string to_string() { return f\"({this.x}, {this.y})\" }\n" + "}\n"; + std::string base = vec; + CHECK_OUTPUT(base + "v := Vec(1, 2) + Vec(3, 4) * 2\nprint(v, v[0], v[1])", "(7.0, 10.0) 7.0 10.0\n"); + CHECK_OUTPUT(base + "v := Vec(3, 4)\nprint(v == Vec(3, 4), v != Vec(3, 4), Vec(1, 1) < v, v > Vec(1, 1), v <= v, v >= Vec(9, 9))", + "true false true true true false\n"); + CHECK_OUTPUT(base + "v := Vec(1, 1)\nv += Vec(1, 2)\nprint(v, [v], f\"{v}\", \"v=\" + v, str(v))", + "(2.0, 3.0) [(2.0, 3.0)] (2.0, 3.0) v=(2.0, 3.0) (2.0, 3.0)\n"); + CHECK_OUTPUT(base + "print(sort([Vec(3, 3), Vec(1, 0), Vec(2, 2)]), max([Vec(3, 3), Vec(1, 0)]), Vec(1, 0) in [Vec(1, 0)])", + "[(1.0, 0.0), (2.0, 2.0), (3.0, 3.0)] (3.0, 3.0) true\n"); + CHECK_ERROR("struct P { i32 x }\nsort([P(1), P(2)])", "can't compare a P with a P (give the struct an operator< method to sort it)"); + CHECK_ERROR("struct P { to_string() { return 5 } }\nprint(P())", "P.to_string() must return a string, not an int"); + CHECK_ERROR("struct P { P operator+() { return this } }", "operator+ takes one parameter"); +} + +TEST(lang_custom_iteration) { + const char* countdown = + "struct Countdown {\n" + " i32 from\n" + " list iter() { return [this.from - i for i in 0..this.from] }\n" + " i32 len() { return this.from }\n" + " bool contains(i32 n) { return n >= 1 and n <= this.from }\n" + "}\n"; + std::string base = countdown; + CHECK_OUTPUT(base + "c := Countdown(3)\nfor n in c print(n)\nprint([n * 10 for n in c], len(c), 2 in c, 5 in c)", + "3\n2\n1\n[30, 20, 10] 3 true false\n"); +} + +TEST(lang_typed_collections) { + CHECK_OUTPUT("list xs = [1, 2]\nxs.push(3)\nxs[0] = 10\nxs.insert(0, 0)\nxs.extend([4])\nprint(xs)", "[0, 10, 2, 3, 4]\n"); + CHECK_ERROR("list xs = [1]\nxs.push(\"two\")", "this is a list, so it can't hold a string"); + CHECK_ERROR("list xs = [1]\nxs[0] = 2.5", "this is a list, so it can't hold 2.5"); + CHECK_OUTPUT("list fs = [1, 2]\nprint(fs)", "[1.0, 2.0]\n"); + CHECK_OUTPUT("map ages = {ada: 36}\nages[\"bob\"] = 40\nages.cy = 22\nprint(ages)", "{\"ada\": 36, \"bob\": 40, \"cy\": 22}\n"); + CHECK_ERROR("map ages = {}\nages[\"x\"] = \"old\"", "this is a map, so its values can't be a string"); + CHECK_ERROR("map ages = {}\nages[5] = 1", "this is a map, so its keys can't be 5"); + CHECK_OUTPUT("list> grid = [[1, 2], [3]]\ngrid[1].push(4)\nprint(grid)", "[[1, 2], [3, 4]]\n"); + CHECK_ERROR("list> grid = [[1]]\ngrid[0].push(\"x\")", "this is a list, so it can't hold a string"); + // A list used as a list stays one, whatever variable it's reached through. + CHECK_ERROR("plain := [1]\nlist typed = plain\nplain.push(\"x\")", "this is a list"); + CHECK_ERROR("plain := [1]\nlist a = plain\nlist b = plain", "can't hold a list (a list or map keeps the element types"); + CHECK_OUTPUT("list names\nnames.push(\"a\")\nprint(names, len(names))", "[\"a\"] 1\n"); + CHECK_OUTPUT("func total(list values) { return sum(values) }\nprint(total([1, 2.5]))", "3.5\n"); + CHECK_ERROR("func total(list values) { return sum(values) }\ntotal([\"x\"])", + "total() expects 'values' to be list, but got a list containing a string"); + CHECK_OUTPUT("struct Team { list members = [] }\nt := Team()\nt.members.push(\"ann\")\nprint(t)", "Team{members: [\"ann\"]}\n"); + CHECK_ERROR("struct Team { list members = [] }\nTeam().members.push(5)", "this is a list, so it can't hold 5"); + CHECK_ERROR("list small = [1, 300]", "'small' is declared as list, so it can't hold a list containing 300"); + CHECK_ERROR("map m", "expected ',' between the key and value types"); + CHECK_OUTPUT("list? maybe = null\nprint(maybe)\nx := 3\nprint(x < 5, x > 2)", "null\ntrue true\n"); +} + +TEST(lang_deep_recursion_is_an_error) { + // Where the stack runs out depends on the machine, so only the error and + // the recursive call in the traceback are checked. + const char* program = "func forever(n) { return forever(n + 1) }\nforever(0)"; + CHECK_ERROR(program, "runtime error:"); + CHECK_ERROR(program, " in forever(), called from test.asc:1:26\n (and "); + CHECK_ERROR(program, " in forever(), called from test.asc:2:1\n"); +} + +// --------------------------------------------------------------------------- +// The scripting library + +TEST(lang_run_programs) { +#if defined(_WIN32) + CHECK_OUTPUT("r := run(\"echo hello\")\nprint(r.code, trim(r.stdout), r.ok)", "0 hello true\n"); + CHECK_OUTPUT("r := run(\"exit 3\")\nprint(r.code, r.ok)", "3 false\n"); +#else + CHECK_OUTPUT("r := run([\"echo\", \"hello\", \"world\"])\nprint(r.code, r.stdout, r.ok)", "0 hello world\n true\n"); + CHECK_OUTPUT("r := run(\"echo out; echo err >&2; exit 3\")\nprint(r.code, repr(r.stdout), repr(r.stderr), r.ok)", + "3 \"out\\n\" \"err\\n\" false\n"); + CHECK_OUTPUT("print(run([\"cat\"], \"fed in\").stdout)", "fed in\n"); +#endif + CHECK_ERROR("run([\"ascend-no-such-program\"])", "run(): couldn't start 'ascend-no-such-program'"); + CHECK_ERROR("run([])", "run(): the command list is empty"); +} + +TEST(lang_paths_and_folders) { + TempFiles files; + files.add("ascend_test_tree/inner/a.txt", "abc"); + files.add("ascend_test_tree/b.txt", "hello"); + CHECK_OUTPUT("print(path_join(\"a\", \"b\", \"c.txt\"), basename(\"/x/y/z.tar.gz\"), dirname(\"/x/y/z.txt\"), extension(\"p.JPG\"))", + "a/b/c.txt z.tar.gz /x/y .JPG\n"); + CHECK_OUTPUT("print(walk_dir(\"ascend_test_tree\"), file_size(\"ascend_test_tree/b.txt\"))", "[\"b.txt\", \"inner/a.txt\"] 5\n"); + CHECK_OUTPUT("copy_file(\"ascend_test_tree/b.txt\", \"ascend_test_tree/c.txt\")\n" + "move_file(\"ascend_test_tree/c.txt\", \"ascend_test_tree/d.txt\")\n" + "print(list_dir(\"ascend_test_tree\"), read_file(\"ascend_test_tree/d.txt\"))", + "[\"b.txt\", \"d.txt\", \"inner\"] hello\n"); + CHECK_OUTPUT("print(remove_dir(\"ascend_test_tree/inner\"), is_dir(\"ascend_test_tree/inner\"), remove_dir(\"nope\"))", + "true false false\n"); + CHECK_OUTPUT("print(len(cwd()) > 0, is_dir(temp_dir()), platform() in [\"linux\", \"macos\", \"windows\", \"unix\"])", + "true true true\n"); + CHECK_OUTPUT("set_env(\"ASCEND_TEST_SETTING\", 42)\nprint(env(\"ASCEND_TEST_SETTING\"))", "42\n"); + CHECK_ERROR("file_size(\"ascend_no_such_file\")", "file_size(): 'ascend_no_such_file':"); +} + +TEST(lang_text_math_and_sets) { + CHECK_OUTPUT("print(capitalize(\"hello world\"), title(\"hello big-world\"), \"[\" + trim_start(\" x \") + \"]\", \"[\" + trim_end(\" x \") + \"]\")", + "Hello world Hello Big-World [x ] [ x]\n"); + CHECK_OUTPUT("print(last_index_of(\"banana\", \"an\"), last_index_of([1, 2, 1], 1), last_index_of(\"héllo\", 'l'), last_index_of(\"x\", \"y\"))", + "3 2 3 -1\n"); + CHECK_OUTPUT("print(lcm(4, 6), factorial(10), degrees(PI), radians(180), trunc(-3.7), trunc(5))", + "12 3628800 180.0 3.141592653589793 -3 5\n"); + CHECK_ERROR("factorial(21)", "factorial(21) is too large for an int"); + CHECK_OUTPUT("print(union([1, 2, 2], [2, 3]), intersect([1, 2, 3, 2], [2, 3, 4]), difference([1, 2, 3], [2]))", + "[1, 2, 3] [2, 3] [1, 3]\n"); +} + +TEST(lang_dates_csv_and_encodings) { + CHECK_OUTPUT("n := now()\nprint(n.year > 2000, n.month in 1..=12, n.weekday in 0..7, type(n.timestamp))", "true true true float\n"); + CHECK_OUTPUT("print(format_time(86400 * 365, \"%Y\") in [\"1970\", \"1971\"])", "true\n"); + CHECK_OUTPUT("rows := csv_parse(\"name,age\\n\\\"Smith, John\\\",42\\n\\\"say \\\"\\\"hi\\\"\\\"\\\",7\\n\")\nprint(rows)", + "[[\"name\", \"age\"], [\"Smith, John\", \"42\"], [\"say \\\"hi\\\"\", \"7\"]]\n"); + CHECK_OUTPUT("print(repr(csv_stringify([[\"a\", \"b, c\"], [1, \"say \\\"hi\\\"\"]])))", "\"a,\\\"b, c\\\"\\n1,\\\"say \\\"\\\"hi\\\"\\\"\\\"\\n\"\n"); + CHECK_OUTPUT("print(csv_parse(\"a;b\\r\\nc;d\", ';'))", "[[\"a\", \"b\"], [\"c\", \"d\"]]\n"); + CHECK_ERROR("csv_parse(\"\\\"open\")", "csv_parse(): a quoted field is never closed"); + CHECK_OUTPUT("print(base64_encode(\"Hello, Ascend!\"), base64_decode(\"SGVsbG8sIEFzY2VuZCE=\"), base64_encode(\"a\"), base64_encode(\"ab\"))", + "SGVsbG8sIEFzY2VuZCE= Hello, Ascend! YQ== YWI=\n"); + CHECK_ERROR("base64_decode(\"*\")", "base64_decode(): '*' isn't base64"); + CHECK_OUTPUT("print(sha256(\"abc\"))", "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad\n"); + CHECK_OUTPUT("id := uuid()\nprint(len(id), id[14], id[8], uuid() != id)", "36 4 - true\n"); +} diff --git a/tests/lexer_tests.cpp b/tests/lexer_tests.cpp index b283569..c04e4c1 100644 --- a/tests/lexer_tests.cpp +++ b/tests/lexer_tests.cpp @@ -179,3 +179,34 @@ TEST(lexer_f_strings) { CHECK_EQ(firstError("f\"{x\""), std::string("unterminated string (missing closing \")")); CHECK_EQ(firstError("f\"oops }\""), std::string("single '}' in f-string (write '}}' for a literal brace)")); } + +TEST(lexer_raw_and_long_strings) { + std::vector raw = Lexer("r\"C:\\new\\d+\"").tokenize(); + CHECK(raw[0].type == TokenType::String); + CHECK_EQ(raw[0].text, "C:\\new\\d+"); + + std::vector longString = Lexer("x := \"\"\"\nfirst \"quoted\"\n second\\t\"\"\"\ny").tokenize(); + CHECK(longString[2].type == TokenType::String); + CHECK_EQ(longString[2].text, "first \"quoted\"\n second\t"); + CHECK_EQ(longString[4].lexeme, "y"); + CHECK_EQ(longString[4].line, 4); + + std::vector rawLong = Lexer("r\"\"\"a\\nb\"\"\"").tokenize(); + CHECK_EQ(rawLong[0].text, "a\\nb"); + + Lexer unterminated("\"\"\"never closed"); + unterminated.tokenize(); + CHECK_EQ(unterminated.errors().size(), size_t(1)); +} + +TEST(lexer_new_operators) { + std::vector tokens = Lexer("a ?? b?.c 0..5 1..=2 ...xs x ? .5 : y").tokenize(); + std::vector expected = {TokenType::Identifier, TokenType::QuestionQuestion, TokenType::Identifier, + TokenType::QuestionDot, TokenType::Identifier, TokenType::Int, + TokenType::DotDot, TokenType::Int, TokenType::Int, TokenType::DotDotEqual, + TokenType::Int, TokenType::Ellipsis, TokenType::Identifier, + TokenType::Identifier, TokenType::Question, TokenType::Dot}; + for (size_t i = 0; i < expected.size(); ++i) CHECK(tokens[i].type == expected[i]); + CHECK(Lexer("1.5").tokenize()[0].type == TokenType::Float); + CHECK(Lexer("x++ --y").tokenize()[1].type == TokenType::PlusPlus); +} diff --git a/tests/parser_tests.cpp b/tests/parser_tests.cpp index 20563f0..505efd9 100644 --- a/tests/parser_tests.cpp +++ b/tests/parser_tests.cpp @@ -43,6 +43,7 @@ std::string op(TokenType type) { case TokenType::In: return "in"; case TokenType::And: return "and"; case TokenType::Or: return "or"; + case TokenType::QuestionQuestion: return "??"; case TokenType::Not: return "not"; case TokenType::Tilde: return "~"; case TokenType::Ampersand: return "&"; @@ -119,6 +120,19 @@ std::string dump(const Expr* expr) { if (!function.returnType.isAny()) out += " -> " + function.returnType.name(); return out + ")"; } + case ExprKind::Range: { + const auto& range = expr->as(); + return "(" + std::string(range.inclusive ? "..=" : "..") + " " + dump(range.start.get()) + " " + + dump(range.end.get()) + ")"; + } + case ExprKind::Spread: return "(... " + dump(expr->as().value.get()) + ")"; + case ExprKind::Comprehension: return "(comprehension " + dump(expr->as().value.get()) + ")"; + case ExprKind::IncDec: { + const auto& incdec = expr->as(); + std::string symbol = incdec.increment ? "++" : "--"; + return incdec.prefix ? "(" + symbol + dump(incdec.target.get()) + ")" + : "(" + dump(incdec.target.get()) + symbol + ")"; + } case ExprKind::FString: { std::string out = "(f"; for (const auto& part : expr->as().parts) { @@ -335,7 +349,8 @@ TEST(parser_typed_declarations) { CHECK(onlyStatement(constant).as().constant); CHECK(parse("const LIMIT := 3").errors.empty()); - CHECK(firstError("int32 x = 1").find("unknown type 'int32'") == 0); + // Other names may be structs, so unknown types are reported by the semantic analyzer. + CHECK(parse("int32 x = 1").errors.empty()); CHECK_EQ(firstError("void x"), std::string("'void' can only be used as a return type")); CHECK_EQ(firstError("i32 x := 5"), std::string("use '=' in a typed declaration, like 'i32 x = ...'")); CHECK_EQ(firstError("const X"), std::string("a constant needs a value: 'const X = ...'")); @@ -404,3 +419,233 @@ TEST(parser_f_strings) { CHECK(firstError("f\"{a ? b : c}\"").find("put a conditional expression in parentheses") != std::string::npos); CHECK(parse("f\"{(a ? b : c)}\"").errors.empty()); } + +TEST(parser_increment_and_decrement) { + CHECK_EQ(expr("i++"), "(i++)"); + CHECK_EQ(expr("--i"), "(--i)"); + CHECK_EQ(expr("items[i++]"), "(index items (i++))"); + CHECK_EQ(expr("p.count--"), "((. p count)--)"); + CHECK_EQ(expr("x + ++y"), "(+ x (++y))"); + CHECK_EQ(expr("a - -b"), "(- a (- b))"); + CHECK_EQ(firstError("5++"), "expected a newline or ';' after the statement, found '++'"); + CHECK_EQ(firstError("++f()"), "'++' needs a variable, element or field to change"); +} + +TEST(parser_casts) { + CHECK_EQ(expr("(f64) total / count"), "(/ (call f64 total) count)"); + CHECK_EQ(expr("(i32) -x"), "(call i32 (- x))"); + CHECK_EQ(expr("(double) 1"), "(call double 1)"); + CHECK_EQ(expr("(x) + 1"), "(+ x 1)"); // not a type: just parentheses + CHECK_EQ(expr("(list) + 1"), "(+ list 1)"); // only number, bool, char and string casts +} + +TEST(parser_c_style_for_loops) { + Parsed classic = parse("for (i32 i = 0; i < 10; i++) { print(i) }"); + CHECK(classic.errors.empty()); + const auto& loop = onlyStatement(classic).as(); + CHECK_EQ(loop.init.size(), size_t(1)); + CHECK(loop.init[0]->kind == StmtKind::Declare); + CHECK_EQ(dump(loop.condition.get()), "(< i 10)"); + CHECK_EQ(loop.step.size(), size_t(1)); + CHECK(!loop.copyPerIteration); + + Parsed several = parse("for (i32 a = 0, b = 9; a < b; a++, b--) {}"); + CHECK(several.errors.empty()); + const auto& pair = onlyStatement(several).as(); + CHECK(pair.init[0]->kind == StmtKind::Group); + CHECK_EQ(pair.step.size(), size_t(2)); + + CHECK(onlyStatement(parse("for (;;) { break }")).as().condition == nullptr); + CHECK(onlyStatement(parse("for { break }")).kind == StmtKind::CFor); + CHECK(onlyStatement(parse("for i := 0; i < 3; i++ { }")).kind == StmtKind::CFor); + CHECK(onlyStatement(parse("for (i := 0; i < 3; i++) fns.push(func() { return i })")).as().copyPerIteration); + + Parsed typed = parse("for (string name : names) print(name)"); + CHECK(typed.errors.empty()); + const auto& each = onlyStatement(typed).as(); + CHECK_EQ(each.firstType.name(), "string"); + CHECK(onlyStatement(parse("for (i, x in items) {}")).kind == StmtKind::For); + CHECK(onlyStatement(parse("for x in (a) {}")).kind == StmtKind::For); + CHECK_EQ(firstError("for (i = 0; i < 3) {}"), "expected ';' after the loop condition, found ')'"); +} + +TEST(parser_bodies_without_braces) { + Parsed parsed = parse("if (x > 0) print(x)\nelse if (x < 0)\n print(-x)\nelse print(0)"); + CHECK(parsed.errors.empty()); + const auto& branch = onlyStatement(parsed).as(); + CHECK_EQ(branch.thenBody.statements.size(), size_t(1)); + CHECK(branch.elseBranch->kind == StmtKind::If); + + CHECK(parse("if (a) b() else c()").errors.empty()); + CHECK(parse("while (i < 10) i++").errors.empty()); + CHECK(parse("while (busy());").errors.empty()); + CHECK(parse("for (x in xs) total += x").errors.empty()); + CHECK(parse("if (done) ++count").errors.empty()); + // When something follows the ')', the parentheses are part of the condition. + Parsed longer = parse("if (a + b) * 2 > 3 { }"); + CHECK(longer.errors.empty()); + CHECK_EQ(dump(onlyStatement(longer).as().condition.get()), "(> (* (+ a b) 2) 3)"); + CHECK_EQ(firstError("if (x)\n"), "expected '{' to start the 'if' body, found end of input"); +} + +TEST(parser_do_while_and_switch) { + Parsed loop = parse("do {\n i++\n} while (i < 3)"); + CHECK(loop.errors.empty()); + CHECK(onlyStatement(loop).as().doWhile); + CHECK(parse("do x++; while (x < 3)").errors.empty()); + CHECK_EQ(firstError("do { }"), "expected 'while' and a condition after the 'do' body, found end of input"); + + Parsed choice = parse("switch (x) {\ncase 1, 2:\n a()\n break\ncase 3:\ndefault:\n b()\n}"); + CHECK(choice.errors.empty()); + const auto& cases = onlyStatement(choice).as().cases; + CHECK_EQ(cases.size(), size_t(3)); + CHECK_EQ(cases[0].values.size(), size_t(2)); + CHECK_EQ(cases[1].start, size_t(2)); + CHECK(cases[2].values.empty()); + CHECK_EQ(cases[2].start, size_t(2)); + CHECK(parse("switch x { case \"a\": print(1); break }").errors.empty()); + CHECK_EQ(firstError("switch x { print(1) }"), "expected 'case' or 'default' in the switch, found identifier 'print'"); + CHECK_EQ(firstError("switch x { default: a()\n default: b() }"), "a switch can only have one 'default'"); + CHECK_EQ(firstError("case 1: x()"), "'case' can only be used inside a switch"); + // These words are still ordinary names elsewhere. + CHECK(parse("switch := 1\ndo := 2\ncase := 3\ndefault := 4\nprint(switch, do, case, default)").errors.empty()); +} + +TEST(parser_import_statements) { + Parsed simple = parse("import geometry, lib.shapes as shapes, \"../x.asc\""); + CHECK(simple.errors.empty()); + const auto& names = onlyStatement(simple).as().names; + CHECK_EQ(names.size(), size_t(3)); + CHECK_EQ(names[0].name, "geometry"); + CHECK_EQ(names[0].binding, "geometry"); + CHECK_EQ(names[1].name, "lib/shapes"); + CHECK_EQ(names[1].binding, "shapes"); + CHECK_EQ(names[2].name, "../x.asc"); + CHECK_EQ(names[2].binding, "x"); + + Parsed from = parse("from lib.shapes import area, Point as P"); + CHECK(from.errors.empty()); + const auto& fromImport = onlyStatement(from).as(); + CHECK(fromImport.from); + CHECK_EQ(fromImport.fromPath, "lib/shapes"); + CHECK_EQ(fromImport.names[1].binding, "P"); + CHECK(onlyStatement(parse("from \"./x\" import *")).as().all); + CHECK(parse("from x import (a, b,)").errors.empty()); + + CHECK(onlyStatement(parse("m := import(\"x\")")).kind == StmtKind::Declare); // the function still works + CHECK(parse("from := 1\nimport := 2").errors.empty()); + CHECK(firstError("import \"my-lib\"").find("give the module a name with 'as'") != std::string::npos); + CHECK_EQ(firstError("from x import"), "expected a name to import, found end of input"); +} + +TEST(parser_structs_enums_and_auto) { + Parsed point = parse("struct Point {\n f64 x, y\n string label = \"p\"\n any extra\n}"); + CHECK(point.errors.empty()); + const auto& fields = onlyStatement(point).as().fields; + CHECK_EQ(fields.size(), size_t(4)); + CHECK_EQ(fields[1].name, "y"); + CHECK_EQ(fields[1].type.name(), "f64"); + CHECK(fields[2].defaultValue != nullptr); + CHECK(parse("struct P { i32 a; i32 b };").errors.empty()); + Parsed methods = parse("struct P {\n f64 w\n f64 area() { return this.w }\n func grow() { }\n reset() { }\n}"); + CHECK(methods.errors.empty()); + const auto& methodList = onlyStatement(methods).as().methods; + CHECK_EQ(methodList.size(), size_t(3)); + CHECK_EQ(methodList[0].first, "area"); + CHECK_EQ(methodList[0].second->params[0].name, "this"); + CHECK_EQ(methodList[0].second->returnType.name(), "f64"); + CHECK(methodList[1].second->returnType.isAny()); + + Parsed typed = parse("Point p = Point(1, 2)"); + CHECK(typed.errors.empty()); + CHECK_EQ(onlyStatement(typed).as().type.name(), "Point"); + CHECK(onlyStatement(parse("Point? find(list items) { return null }")).kind == StmtKind::Function); + CHECK(onlyStatement(parse("ready ? go() : stop()")).kind == StmtKind::Expression); + + Parsed colors = parse("enum Color {\n Red,\n Green = 5\n Blue\n}"); + CHECK(colors.errors.empty()); + CHECK_EQ(onlyStatement(colors).as().members.size(), size_t(3)); + + Parsed inferred = parse("auto n = 5"); + CHECK(inferred.errors.empty()); + CHECK(onlyStatement(inferred).as().inferType); + CHECK_EQ(firstError("auto n"), "'auto n' needs a value to take its type from, like 'auto n = 0'"); + CHECK_EQ(firstError("i32 f(auto x) { }"), "'auto' can only be used to declare a variable with a value"); + CHECK(onlyStatement(parse("i32 a = 1, b = 2")).kind == StmtKind::Group); +} + +TEST(parser_lambdas_and_comprehensions) { + CHECK_EQ(expr("x => x * 2"), "(func x)"); + CHECK_EQ(expr("(a, i32 b) => a + b"), "(func a b:i32)"); + CHECK_EQ(expr("map(xs, x => x + 1)"), "(call map xs (func x))"); + CHECK_EQ(expr("() => { print(1) }"), "(func)"); + // In a match, `name => ...` is still a pattern and its arm. + Parsed arms = parse("match x {\n y => print(1)\n}"); + CHECK(arms.errors.empty()); + CHECK_EQ(dump(onlyStatement(arms).as().arms[0].patterns[0].get()), "y"); + + Parsed list = parse("[x * x for x in items if x > 0]"); + CHECK(list.errors.empty()); + const auto& comprehension = onlyStatement(list).as().expr->as(); + CHECK(!comprehension.isMap); + CHECK_EQ(comprehension.clauses.size(), size_t(2)); + CHECK(comprehension.clauses[0].isFor); + CHECK(!comprehension.clauses[1].isFor); + Parsed map = parse("m := {k: v for k, v in pairs}"); + CHECK(map.errors.empty()); + const auto& mapComprehension = onlyStatement(map).as().value->as(); + CHECK(mapComprehension.isMap); + CHECK_EQ(dump(mapComprehension.key.get()), "k"); // the variable, not the string "k" +} + +TEST(parser_ranges_spreads_and_named_arguments) { + const auto rangeStmt = parse("0..n + 1"); + const auto& range = onlyStatement(rangeStmt).as().expr->as(); + CHECK_EQ(dump(range.end.get()), "(+ n 1)"); + CHECK(!range.inclusive); + CHECK(onlyStatement(parse("1..=10")).as().expr->as().inclusive); + CHECK_EQ(expr("x in 0..10"), "(in x (.. 0 10))"); + CHECK_EQ(expr("f(...xs)"), "(call f (... xs))"); + + Parsed call = parse("f(1, ...rest, size: 2)"); + CHECK_EQ(call.errors.size(), size_t(0)); + const auto& args = onlyStatement(call).as().expr->as(); + CHECK_EQ(args.args.size(), size_t(3)); + CHECK(args.args[1]->kind == ExprKind::Spread); + CHECK_EQ(args.argNames[2], "size"); + CHECK_EQ(firstError("f(a: 1, 2)"), "positional arguments must come before named ones"); + CHECK_EQ(firstError("f(a: 1, a: 2)"), "the argument 'a' is given twice"); + + CHECK(parse("[...a, 1, ...b]").errors.empty()); + CHECK(parse("m := {...defaults, size: 2}").errors.empty()); + CHECK(onlyStatement(parse("func f(a, ...rest) { }")).as().function->variadic); + CHECK(onlyStatement(parse("i32 sum(i32 ...nums) { return 0 }")).as().function->variadic); + CHECK_EQ(firstError("func f(...a, b) { }"), "'...a' collects the remaining arguments, so it must be the last parameter"); +} + +TEST(parser_null_operators_destructuring_and_finally) { + CHECK_EQ(expr("a ?? b ?? c"), "(?? (?? a b) c)"); + CHECK_EQ(expr("a ?? b or c"), "(?? a (or b c))"); + CHECK(parse("x := a?.b?.c").errors.empty()); + CHECK(parse("a?.f()").errors.empty()); + CHECK_EQ(firstError("a?.b = 1"), "can't assign to this expression"); + + Parsed swap = parse("a, b = b, a"); + CHECK(swap.errors.empty()); + const auto& multi = onlyStatement(swap).as(); + CHECK(!multi.declare); + CHECK_EQ(multi.targets.size(), size_t(2)); + CHECK_EQ(multi.values.size(), size_t(2)); + CHECK(onlyStatement(parse("x, y := point")).as().declare); + CHECK_EQ(firstError("a, b := 1, 2, 3"), "there are 2 variables but 3 values"); + CHECK_EQ(firstError("a.b, c := 1, 2"), "':=' can only declare plain variable names (use '=' to change elements or fields)"); + CHECK(onlyStatement(parse("func f() { return 1, 2 }")).kind == StmtKind::Function); + + Parsed tryFinally = parse("try {\n a()\n} finally {\n b()\n}"); + CHECK(tryFinally.errors.empty()); + const auto& tryStmt = onlyStatement(tryFinally).as(); + CHECK(!tryStmt.hasCatch); + CHECK(tryStmt.hasFinally); + CHECK(parse("try { a() } catch e { b() } finally { c() }").errors.empty()); + CHECK_EQ(firstError("try { a() }"), "expected 'catch' or 'finally' after the 'try' block, found end of input"); +}