Add apkg, a package manager for Ascend

.github/workflows/build.yml

@@ -48,14 +48,16 @@ jobs:
       - name: Test
         run: ctest --test-dir build --build-config Release --output-on-failure
 
-      - name: Upload the interpreter
+      - name: Upload the interpreter and package manager
         if: ${{ matrix.artifact }}
         uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.artifact }}
           path: |
             build/ascend
+            build/apkg
             build/Release/ascend.exe
+            build/Release/apkg.exe
           if-no-files-found: error
 
   sanitizers:

.gitignore

@@ -1,3 +1,5 @@
-# Build output (build/deleteme.txt keeps the folder in git)
-build/*
-!build/deleteme.txt
+# Build output
+build/
+
+# Packages installed by apkg
+ascend_packages/

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 0.4.0
+
+### New
+
+- **`apkg`, the Ascend package manager**, built and installed next to
+  `ascend`. `apkg install`, `uninstall`, `list`, `show`, `search` and `init`
+  work like pip and npm. Packages can come from the registry (`colors`,
+  `colors@1.0.0`), GitHub (`github:user/repo@v1.0`), any git repository
+  (`git+https://...`) or a folder (`./my-lib`), and their dependencies are
+  installed with them. See [docs/packages.md](docs/packages.md).
+- Projects: `apkg init` writes an `ascend.json`; inside a project packages go
+  into `ascend_packages/` and are recorded in `ascend.json`, and `apkg install`
+  with no names installs everything listed there.
+- `import("name")` finds installed packages when there's no `name.asc` next to
+  the importing file, and `import("name/file")` imports a file inside one.
+- A package registry (`registry/index.json`) with three official packages:
+  `colors` (terminal colors), `strings` (case conversion, word wrapping,
+  edit distance and more) and `testing` (unit tests).
+
+### Changed
+
+- Importing a file that doesn't exist suggests `apkg install` when the name
+  could be a package.
+
 ## 0.3.0
 
 ### New

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.10)
 
-project(ascend VERSION 0.3.0 LANGUAGES CXX)
+project(ascend VERSION 0.4.0 LANGUAGES CXX)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -33,9 +33,17 @@ add_library(ascend_core STATIC
     src/json.cpp
     src/runner.cpp
     src/repl.cpp
+    src/process.cpp
+    src/packages.cpp
+    src/package_manager.cpp
 )
 target_include_directories(ascend_core PUBLIC src/include)
 
+# GCC 8 keeps std::filesystem in a separate library.
+if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)
+    target_link_libraries(ascend_core PUBLIC stdc++fs)
+endif()
+
 if(MSVC)
     target_compile_options(ascend_core PRIVATE /W4 /permissive-)
 else()
@@ -54,7 +62,12 @@ endfunction()
 add_executable(ascend src/main.cpp)
 target_link_libraries(ascend PRIVATE ascend_core)
 ascend_set_stack_size(ascend)
-install(TARGETS ascend RUNTIME DESTINATION bin)
+
+# The package manager.
+add_executable(apkg src/apkg_main.cpp)
+target_link_libraries(apkg PRIVATE ascend_core)
+
+install(TARGETS ascend apkg RUNTIME DESTINATION bin)
 
 option(ASCEND_BUILD_TESTS "Build the Ascend test suite" ON)
 if(ASCEND_BUILD_TESTS)
@@ -65,8 +78,11 @@ if(ASCEND_BUILD_TESTS)
         tests/lexer_tests.cpp
         tests/parser_tests.cpp
         tests/interpreter_tests.cpp
+        tests/package_tests.cpp
     )
     target_link_libraries(ascend_tests PRIVATE ascend_core)
+    # The package tests install the packages in this repository's registry.
+    target_compile_definitions(ascend_tests PRIVATE ASCEND_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}")
     ascend_set_stack_size(ascend_tests)
     add_test(NAME unit_tests COMMAND ascend_tests)
 

README.md

@@ -43,7 +43,8 @@ for name, score in scores {
 - f-strings with format specs: `f"{price:>8.2f}"`
 - Hex, binary and octal literals, type suffixes (`255u8`) and bitwise operators
 - Unicode strings and a `char` type
-- Modules with `import("file")`
+- Modules with `import("file")`, and **`apkg`, a package manager** like pip:
+  `apkg install colors`, then `import("colors")`
 - `try` / `catch` / `throw` error handling
 - 100+ builtins for strings, lists, maps, math, files, JSON and the system
 - Clear error messages that point at the problem, many caught before running:
@@ -65,10 +66,18 @@ cmake --build build
 ./build/ascend          # interactive REPL
 ```
 
+Install a package and use it:
+
+```sh
+./build/apkg install colors
+./build/ascend -e 'print(import("colors").green("Hello!"))'
+```
+
 ## Documentation
 
 - [Language guide](docs/syntax.md)
 - [Standard library](docs/standard_library.md)
+- [Packages and apkg](docs/packages.md)
 - [Building, installing, testing and editor support](docs/installation.md)
 - [Example programs](examples/)
 
@@ -83,6 +92,7 @@ cmake --build build
 | Values, numeric types and checked arithmetic | `src/value.cpp`, `src/numeric.cpp`, `src/types.cpp` |
 | Builtin functions, formatting and JSON | `src/stdlib.cpp`, `src/format.cpp`, `src/json.cpp` |
 | Command line and REPL | `src/main.cpp`, `src/repl.cpp`, `src/runner.cpp` |
+| Package manager (`apkg`) and finding packages for `import` | `src/apkg_main.cpp`, `src/package_manager.cpp`, `src/packages.cpp` |
 
 ## License
 

docs/installation.md

@@ -5,7 +5,9 @@
 - A C++17 compiler: GCC 8+, Clang 7+ or Visual Studio 2019+
 - CMake 3.10 or newer
 
-Ascend has no other dependencies.
+Ascend has no other dependencies. The package manager, `apkg`, uses
+[git](https://git-scm.com) to download packages, so install git too if you
+want to use packages from the internet.
 
 ## Build
 
@@ -14,8 +16,9 @@ cmake -S . -B build
 cmake --build build
 ```
 
-This produces the interpreter, `build/ascend` (`build\Release\ascend.exe`
-with Visual Studio). Builds default to the optimized `Release` configuration.
+This produces the interpreter, `build/ascend`, and the package manager,
+`build/apkg` (`build\Release\ascend.exe` and `build\Release\apkg.exe` with
+Visual Studio). Builds default to the optimized `Release` configuration.
 
 ## Run
 
@@ -26,9 +29,14 @@ with Visual Studio). Builds default to the optimized `Release` configuration.
 ./build/ascend --check my_program.asc   # find errors without running
 ```
 
-To run `ascend` from anywhere, copy `build/ascend` into a directory on your
-`PATH`, or run `cmake --install build` (add `--prefix ~/.local` to install
-without administrator rights).
+To run `ascend` and `apkg` from anywhere, copy them from `build/` into a
+directory on your `PATH`, or run `cmake --install build` (add
+`--prefix ~/.local` to install without administrator rights).
+
+```sh
+apkg install colors                     # see docs/packages.md
+ascend -e 'print(import("colors").bold("ready"))'
+```
 
 ## Test
 
@@ -37,9 +45,10 @@ cmake --build build
 cd build && ctest --output-on-failure
 ```
 
-This runs the unit tests in `tests/` and checks that every program in
-`examples/` runs without errors. You can also run the unit tests directly and
-filter them by name:
+This runs the unit tests in `tests/` (including tests that install the
+packages in `packages/` with `apkg`, and from temporary git repositories when
+git is available) and checks that every program in `examples/` runs without
+errors. You can also run the unit tests directly and filter them by name:
 
 ```sh
 ./build/ascend_tests            # all tests
@@ -53,9 +62,9 @@ To skip building the tests, configure with `-DASCEND_BUILD_TESTS=OFF`.
 Every push to `main` and every pull request is built and tested by GitHub
 Actions (`.github/workflows/build.yml`) on Linux (GCC and Clang), macOS and
 Windows (MSVC), plus a run with AddressSanitizer and UndefinedBehaviorSanitizer.
-Each run also uploads ready-to-use interpreters for Linux, macOS and Windows:
-open the run on the repository's **Actions** tab and download them under
-**Artifacts**.
+Each run also uploads ready-to-use builds of `ascend` and `apkg` for Linux,
+macOS and Windows: open the run on the repository's **Actions** tab and
+download them under **Artifacts**.
 
 ## Editor support
 

docs/packages.md

@@ -0,0 +1,173 @@
+# Packages and apkg
+
+`apkg` is Ascend's package manager. It's built and installed next to the
+`ascend` interpreter, and works much like `pip` or `npm`:
+
+```sh
+apkg install colors        # install a package
+apkg list                  # see what's installed
+apkg uninstall colors      # remove it again
+```
+
+Programs use installed packages with `import`:
+
+```
+colors := import("colors")
+print(colors.green("it works!"))
+```
+
+`apkg` downloads packages with [git](https://git-scm.com), so git must be
+installed and on your `PATH` (folders on your computer can be installed
+without it).
+
+## Commands
+
+| Command | What it does |
+|---------|--------------|
+| `apkg install <package>...` | Installs packages and everything they depend on. Aliases: `add`, `i`. |
+| `apkg install` | In a project, installs everything its `ascend.json` lists. |
+| `apkg uninstall <package>...` | Removes packages. Aliases: `remove`, `rm`. |
+| `apkg list` | Lists the installed packages. Alias: `ls`. |
+| `apkg show <package>` | Shows an installed package's details: version, source, what it needs and what needs it. Alias: `info`. |
+| `apkg search [text]` | Searches the registry by name and description. Without text, lists every package. |
+| `apkg init [name]` | Makes the current folder a project by writing an `ascend.json` (and a `main.asc`, if there isn't one). |
+
+Options (they can go anywhere on the command line):
+
+| Option | |
+|--------|--|
+| `-g`, `--global` | Use your own packages folder even inside a project. |
+| `-U`, `--upgrade` | Get the newest version of packages that are already installed. Without it, installing a package that's already there does nothing. |
+| `-h`, `--help` | Show help. |
+| `-v`, `--version` | Show the version. |
+
+## Where packages come from
+
+```sh
+apkg install colors                                     # the registry, newest version
+apkg install colors@1.0.0                               # the registry, a particular version
+apkg install github:someone/their-package               # a GitHub repository
+apkg install github:someone/their-package@v2.1          # ...at a tag or branch
+apkg install github:someone/monorepo/packages/thing     # ...a folder inside it
+apkg install git+https://gitlab.com/someone/pkg.git     # any git repository
+apkg install git+https://example.com/repo.git@v1#sub    # ...at a tag, in a folder
+apkg install git@github.com:someone/private-pkg.git     # git over SSH
+apkg install ./my-library                               # a folder on this computer
+```
+
+The **registry** is a list of known packages: [`registry/index.json`](../registry/index.json)
+in the Ascend repository. To use a different one, set `ASCEND_REGISTRY` to
+another git repository or to a folder with the same layout.
+
+## Projects and your own packages
+
+**Inside a project** (a folder with an `ascend.json`, or any folder below
+one), `apkg install` puts packages into the project's `ascend_packages` folder
+and records them in `ascend.json`:
+
+```sh
+mkdir weather && cd weather
+apkg init                  # writes ascend.json and main.asc
+apkg install colors        # installs into ./ascend_packages, adds "colors" to ascend.json
+```
+
+```json
+{
+  "name": "weather",
+  "version": "0.1.0",
+  "description": "",
+  "main": "main.asc",
+  "dependencies": {
+    "colors": "*"
+  }
+}
+```
+
+Commit `ascend.json` but not `ascend_packages` (`apkg init` adds it to
+`.gitignore`). Anyone who checks out the project runs `apkg install` with no
+package names to install everything it lists.
+
+**Anywhere else**, packages are installed just for you, in
+`~/.ascend/packages` (`%USERPROFILE%\.ascend\packages` on Windows). Use `-g`
+to install there from inside a project. Set `ASCEND_HOME` to keep them
+somewhere else.
+
+## How import finds packages
+
+`import("name")` first looks for a file `name.asc` next to the importing file,
+as always. If there isn't one, it looks for an installed package called
+`name`:
+
+1. in `ascend_packages/name` next to the importing file, then in the
+   `ascend_packages` folder of each folder above it;
+2. in your own packages (`~/.ascend/packages/name`).
+
+It runs the package's main file (`main` in its `ascend.json`). To import a
+different file from a package, add its path: `import("strings/strings")`.
+
+A package that depends on another one imports it the same way, so packages
+can be used together without knowing where they're installed.
+
+## Making a package
+
+A package is a folder with an `ascend.json`:
+
+```json
+{
+  "name": "shapes",
+  "version": "1.0.0",
+  "description": "Areas and perimeters of common shapes",
+  "main": "shapes.asc",
+  "dependencies": {
+    "strings": "*",
+    "units": "1.2.0",
+    "geometry": "github:someone/geometry@v3.0"
+  }
+}
+```
+
+| Field | |
+|-------|--|
+| `name` | Required. Lowercase letters, digits, `-` and `_`. This is the name used to `import` it. |
+| `version` | Required. Use `major.minor.patch` numbers. |
+| `description` | A one-line summary, shown by `apkg show` and `apkg search`. |
+| `main` | The file `import("name")` runs. Defaults to `main.asc`. |
+| `dependencies` | Other packages it needs: `"*"` for any version from the registry, a version number like `"1.2.0"`, or any source `apkg install` accepts, like `"github:user/repo@v1.0"` or `"file:../other-folder"`. |
+
+Everything at the top level of the main file that doesn't start with `_`
+becomes part of the package's map when it's imported.
+
+To share a package, put the folder in a git repository and others can install
+it with `apkg install github:you/your-package`. Tag each release `v` + its
+version (`v1.0.0`), so `name@1.0.0` can find it once it's in the registry.
+
+### Adding a package to the registry
+
+Open a pull request that adds an entry to [`registry/index.json`](../registry/index.json):
+
+```json
+"shapes": {
+  "description": "Areas and perimeters of common shapes",
+  "source": "git+https://github.com/you/shapes.git"
+}
+```
+
+`source` can be any git source (as above, but without a version: `apkg`
+adds `@v<version>` when someone asks for a particular one), or
+`registry:packages/<name>` for packages kept in the Ascend repository's
+[`packages`](../packages) folder.
+
+## Official packages
+
+| Package | |
+|---------|--|
+| [`colors`](../packages/colors) | Color and style text in the terminal. |
+| [`strings`](../packages/strings) | Case conversion, word wrapping, padding, edit distance and other text tools. |
+| [`testing`](../packages/testing) | Write and run unit tests for Ascend code. |
+
+## Environment variables
+
+| Variable | |
+|----------|--|
+| `ASCEND_HOME` | Where your own packages (`packages/`) and downloaded data (`cache/`) are kept. Default: `~/.ascend`. |
+| `ASCEND_REGISTRY` | The registry: a git repository address or a folder containing `registry/index.json`. Default: `https://github.com/csubbdev/Ascend.git`. |

docs/standard_library.md

@@ -145,7 +145,7 @@ Lengths and positions count characters, not bytes.
 
 | Function | Description |
 |----------|-------------|
-| `import(path)` | Runs another Ascend file once and returns its top-level names as a map. Relative paths start from the importing file; `.asc` is optional. |
+| `import(path)` | Runs another Ascend file once and returns its top-level names as a map. Relative paths start from the importing file; `.asc` is optional. If there's no such file, imports the installed package with that name (see [packages.md](packages.md)). |
 | `assert(condition, message?)` | Throws an error if `condition` is falsy. |
 | `exit(code?)` | Ends the program with exit code `code` (default 0). |
 | `args` | List of the extra command line arguments given after the script name. |