more py, build as user,...

.github/workflows/pre-commit.yaml

@@ -0,0 +1,10 @@
+name: pre-commit
+on: [push, pull_request]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
+    - uses: pre-commit/action@v3.0.1

.github/workflows/shellcheck.yml → .github/workflows/shellcheck.yaml

@@ -6,6 +6,6 @@ jobs:
     name: ShellCheck
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Run ShellCheck
       run: find ./scripts -type f -name "*.sh" -exec shellcheck --enable require-double-brackets,require-variable-braces,avoid-nullary-conditions {} +

.pre-commit-config.yaml

@@ -0,0 +1,34 @@
+---
+repos:
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+   rev: v6.0.0
+   hooks:
+    - id: trailing-whitespace
+      exclude: |
+          (?x)(
+             README.md
+          )$
+    - id: check-merge-conflict
+    - id: check-yaml
+    - id: check-ast
+    - id: detect-private-key
+    - id: end-of-file-fixer
+
+ - repo: https://github.com/asottile/pyupgrade
+   rev: v3.21.2
+   hooks:
+    - id: pyupgrade
+      args: [--py36-plus]
+
+ - repo: https://github.com/psf/black
+   rev: 26.5.1
+   hooks:
+    - id: black
+
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+   rev: v0.16.5
+   hooks:
+    # Run the linter.
+    - id: ruff-check
+    # Run the formatter.
+    - id: ruff-format

Dockerfile

@@ -16,7 +16,7 @@ USER root
 # Create dirs and copy scripts
 RUN mkdir --parents "${SCRIPT_DIR}"
 COPY scripts/ "${SCRIPT_DIR}"/
-COPY py-utils/ "${BIN_DIR}"/
+COPY py-utils/ "${SCRIPT_DIR}"/
 
 # Set up user and work directories
 RUN chown --recursive "${userid}":"${groupid}" "${ROOT_DIR}" && chmod --recursive u+srw "${ROOT_DIR}"
@@ -35,6 +35,8 @@ ENV BUILD_TYPE=''
 ENV ROM_MANIFEST=''
 ENV ROM_BRANCH=''
 ENV ROM_BUILD_FLAGS=''
+ENV BL_RELOCK=''
+ENV FETCH_MUPPETS=''
 
 # Fallbacks (required for non-standard naming and conflicts)
 ENV LUNCH_PREFIX_FALLBACK=''
@@ -43,14 +45,16 @@ ENV ROM_VERSION_FALLBACK=''
 ENV ROM_OTA_BRANCH_FALLBACK=''
 ENV RELEASETOOL_EXTRA_FLAGS=''
 
-# Extra variables
+# Repopick
+ENV REPOPICK_PICKS=''
+ENV REPOPICK_TOPICS=''
+ENV REPOPICK_PULLS=''
+
+# Build env variables
 ENV CCACHE_SIZE=40
 ENV OTA_REPO_URL=''
 ENV KEYS_SUBJECT='/C=US/ST=California/L=Mountain View/O=Android/OU=Android/CN=Android/emailAddress=android@android.com'
 ENV TIME_ZONE='UTC'
-ENV REPOPICK_PICKS=''
-ENV REPOPICK_TOPICS=''
-ENV REPOPICK_PULLS=''
 ENV DEBUG=''
 
 # Authentification
@@ -62,4 +66,4 @@ ENV SSH_USER=''
 ENV SSH_UPLOAD_URL=''
 ENV SSH_DOWNLOAD_URL=''
 
-ENTRYPOINT ["/bin/bash", "-c", "${SCRIPT_DIR}/init.sh"]
+ENTRYPOINT ["/bin/bash", "-c", "${SCRIPT_DIR}/00_init.sh"]

README.md

@@ -11,30 +11,21 @@ This project targets Android 7 up to the most recent version.
 ## Prerequisites
 
 - [Podman](https://podman.io/docs/installation)
-  - [Podman compose](https://github.com/containers/podman-compose?tab=readme-ov-file#pip)
+  - podman-docker or alias
 - or [Docker](https://docs.docker.com/engine/install)
-  - [Docker Compose](https://docs.docker.com/compose/install)
-  - [Docker Rootless](https://docs.docker.com/engine/security/rootless/)
 - [SSH](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent)
 - [GitConfig](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup)
 - ZRam (highly recommended): [Debian](https://wiki.debian.org/ZRam), [Fedora](https://github.com/systemd/zram-generator), [Ubuntu, Arch and others](https://wiki.archlinux.org/title/Zram)
 
 
-### Setting up permissions
-
-First we need to find the UID, that is used for Docker/Podman.  
-For Debian, Ubuntu and Gentoo this seems to be 100999 and on Fedora 52587,  
-which should be $subUID+$containerUID-1 according to the [docker forums](https://forums.docker.com/t/map-more-uid-on-rootless-docker-and-mount-volume/102928/8).
-
-
 We need to manually create the required folders for the respective volumes
 ```
-mkdir -p ~/docker_droid/src ~/docker_droid/dotfiles ~/docker_droid/ccache ~/docker_droid/logs ~/docker_droid/keys
+mkdir --parents ~/docker_droid/src ~/docker_droid/dotfiles ~/docker_droid/ccache ~/docker_droid/logs ~/docker_droid/keys
 ```
 Copy the required dotfiles from the host machines
 ```
 cp ~/.gitconfig ~/docker_droid/dotfiles/
-cp -r ~/.ssh ~/docker_droid/dotfiles/
+cp --recursive ~/.ssh ~/docker_droid/dotfiles/
 ```
 and clone this repo
 ```
@@ -43,44 +34,6 @@ git clone https://github.com/SirRGB/dockdroid ~/docker_droid/minideb
 
 Then we need to chown that directory to the Docker user:
 
-#### Debian/Ubuntu
-```
-sudo chown --recursive 100999:"${UID}" ~/docker_droid/src ~/docker_droid/dotfiles ~/docker_droid/ccache ~/docker_droid/logs ~/docker_droid/keys
-```
-
-#### Fedora
-```
-sudo chown --recursive 52587:"${UID}" ~/docker_droid/src ~/docker_droid/dotfiles ~/docker_droid/ccache ~/docker_droid/logs ~/docker_droid/keys
-```
-
-#### Other
-(If you know a smarter way to do this please tell me,  
-I know the available subuids can be found with `grep $USER < /etc/subuid | cut --delimiter=":" --fields=2`  
-I just do not know if the container uid is predictable,  
-it seems to be 1000 for debian/ubuntu and 100 for fedora)
-
-Let other users read the directory
-```
-sudo chmod --recursive 507 ~/docker_droid/src ~/docker_droid/dotfiles ~/docker_droid/ccache ~/docker_droid/logs ~/docker_droid/keys
-```
-Run the first docker build
-```
-docker compose up --force-recreate --build
-```
-Wait until it starts syncing and stop using ctrl + c  
-Find out the uid by running:
-```
-ls -n ~/docker_droid/src/Los15/.repo
-```
-Give ownership to the uid you found out:  
-(replace the 1st UID)
-```
-sudo chown --recursive UID:"${UID}" ~/docker_droid/src ~/docker_droid/dotfiles ~/docker_droid/ccache ~/docker_droid/logs ~/docker_droid/keys
-```
-And remove the incomplete sync
-```
-sudo rm -rf ~/docker_droid/src/Los15/
-```
 </details>
 
 
@@ -93,9 +46,6 @@ sudo rm -rf ~/docker_droid/src/Los15/
 - ```ROM_DIR```: Only change the last part after src/. Defines the source path within the container
 - ```ROM_MANIFEST```: URL of the rom manifest you want to sync
 - ```ROM_BRANCH```: Branch of the rom you want to sync
-- ```LOCAL_MANIFEST```: Direct link to the local manifest(s)
-or
-- ```CLONE_REPOS```: Links to the repo(s) to clone. Repo name MUST have the following pattern https://github.com/user/android_dir1_dir2_dir3/tree/branch, https://github.com/user/dir1_dir2_dir3/tree/branch or https://github.com/user/proprietary_dir1_dir2_dir3/tree/branch. Not recommended.
 
 These variables should be defined in the target.env.
 
@@ -108,34 +58,60 @@ DEVICE=cheeseburger,dumpling,TP1803
 ROM_DIR=/droid_workdir/src/Los15
 ROM_MANIFEST=https://github.com/LineageOS/android.git
 ROM_BRANCH=lineage-22.2
-LOCAL_MANIFEST=https://raw.githubusercontent.com/SirRGB/local_manifests/refs/heads/main/cheeseburgerdumpling/A15Lineage.xml,https://raw.githubusercontent.com/SirRGB/local_manifests/refs/heads/main/TP1803/A15Lineage.xml
 ```
 
 ### optional
-
+- Device tree sources for unofficial support
+  - ```LOCAL_MANIFEST```: Direct link to the local manifest(s)
+  or
+  - ```CLONE_REPOS```: Links to the repo(s) to clone. Repo name MUST have the following pattern https://github.com/user/android_dir1_dir2_dir3/tree/branch, https://github.com/user/dir1_dir2_dir3/tree/branch or https://github.com/user/proprietary_dir1_dir2_dir3/tree/branch. Not recommended.
+  - ```FETCH_MUPPETS```: Whether to fetch the official TheMuppets vendor repos. Only works on LineageOS 21 and above due to the respective [manifest](https://github.com/TheMuppets/manifests/blob/lineage-21.0/muppets.xml) groups. Any non-empty value is true. Only works with CLONE_REPOS.
+  - ```BL_RELOCK```: Signs vbmeta and uploads pkmd custom key. Untested, use at your own risk.
+- Fallbacks
+  - ```LUNCH_PREFIX_FALLBACK```: Prefix for lunching, i.e. lineage_ for LineageOS. Needed when neither ```LOCAL_MANIFEST``` or ```CLONE_REPOS``` are specified.
+  - ```ROM_PREFIX_FALLBACK```: Prefix for naming, i.e. lineage will result in a package name like lineage-extraversion(if set)-version-date-device-signed.zip. Needed when *_TARGET_PACKAGE is not defined in vendor/*/build/tasks/* or build/core/Makefile.
+  - ```ROM_VERSION_FALLBACK```: Version fallback for package name. Needed when rom does not specify PRODUCT_VERSION_MINOR/PRODUCT_VERSION_MAJOR in vendor/*/config/.
+  - ```ROM_OTA_BRANCH_FALLBACK```: Ota branch fallback. Needed when roms have conflicting branch names and ota updates are set up.
+  - ```RELEASETOOL_EXTRA_FLAGS```: Releasetool flags for setting up legacy quirks i.e. skip certain vintf checks.
 - GitHub Upload
-  - ```[GITHUB_TOKEN](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens)```
+  - ```GITHUB_TOKEN```: [Github Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) for upload (and/or push) authentification
   - ```OTA_REPO_URL```: for example git@github.com:user/ota_config, will also be used for uploading
   - Requires ```GITHUB_TOKEN``` or passwordless ssh keys [added to your GitHub account](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/adding-a-new-ssh-key-to-your-github-account)
 - SourceForge Upload
   - ```SF_USER```: Username of your account
   - ```SF_RELEASES_REPO```: Project name
   - Requires passwordless ssh keys added to your [SourceForge account](https://sourceforge.net/p/forge/documentation/SSH%20Keys)
+- SSH Upload
+  - ```SSH_USER```: Username on the ssh server
+  - ```SSH_UPLOAD_URL```: ssh server domain or ip
+  - ```SSH_DOWNLOAD_URL```: Direct download url excluding filename to ssh server
+  - Requires passwordless ssh keys with respective public keys authorized on your ssh server
 - Telegram logging
-  - ```[TELEGRAM_TOKEN](https://core.telegram.org/bots/features#botfather)```
+  - ```TELEGRAM_TOKEN```: [Telegram Token](https://core.telegram.org/bots/features#botfather). Needed when telegram logging is set up.
   - ```TELEGRAM_CHAT```: either as @xyz or the id
 - ```TIME_ZONE```: either as in the format UTC+2 or CET
 - ```ROM_BUILD_FLAGS```: if you want to define values like ```WITH_GMS=true``` you can do this in here, even with multiple of them separated by comma for separate builds or space for the same build
 - OTA Updates
   - At least one file provider mentioned above
   - ```OTA_REPO_URL```: Any git hoster using ssh authentification i.e. GitHub, GitLab, CodeBerg,...
+- Repopick
+  - ```REPOPICK_PICKS```: Fetch changes from respective Gerrit instance using short-form change-id
+  - ```REPOPICK_TOPICS```: Fetch changes from respective Gerrit instance using topics
+  - ```REPOPICK_PULLS```: Pull changes from respective Gerrit instance using short-form change-id
 
 These variables should be defined in config.env.
 
 ```
 GITHUB_TOKEN=thing1234
 OTA_REPO_URL=git@github.com:user/ota_config
 ```
+You can use config.env for all variables, that span across more than one device. To include this run:
+
+```
+bash dockdroid -r target.env -c config.env
+```
+`--config` can also be used. Your config file can be named however you please.
+
 </details>
 
 
@@ -151,18 +127,14 @@ OTA_REPO_URL=git@github.com:user/ota_config
 
 - After setting everything up you should do a test build with the default variables for testing. (Be sure to be in ~/docker_droid/minideb)
 ```
-podman compose up --force-recreate --build
-```
-or
-```
-docker compose up --force-recreate --build
+bash dockdroid -r target.env
 ```
 
 
 ## Debugging
 
 - Look up known issues in [TODO.md](TODO.md)
-- If the error is undocumented you can uncomment [this](https://github.com/SirRGB/dockdroid/blob/da9593dca1ced7932f7f4295d20fab7077104b73/scripts/init.sh#L3) and send the part of the logs, where things go overboard via the [issues](https://github.com/SirRGB/dockdroid/issues) or debug it on your own and send a pull request.
+- If the error is undocumented you can set DEBUG=true in the target.env and send the part of the logs, where things go overboard via the [issues](https://github.com/SirRGB/dockdroid/issues) or debug it on your own and send a pull request.
 
 
 ## Too much RAM
@@ -187,3 +159,4 @@ You can further speed up build times by using tmpfs as described [here](https://
 - [LineageOS4MicroG](https://github.com/lineageos4microg/docker-lineage-cicd)
 - [amyROM](https://github.com/amyROM/vendor_amy/blob/207d5e32c3fba38b9fe1ab9cd12c71ca6b81d653/scripts/generate_json_build_info.sh)
 - [LineageOS Infra](https://github.com/lineageos-infra/build-config/tree/main/android)
+- [Halogen OS](https://github.com/halogenOS/android_external_xos/blob/fb9a58362b930807766100d1288ff809df6b7c51/xostools/xostools.sh)

TODO.md

@@ -1,2 +1,8 @@
 ## ToDo
 - make vendor pulling work for multiple devices
+- GSI/patch handling
+- project linkfile handling
+- progress message telegram
+- incremental updates
+- move to py (apart from env vars?, devices as objects?)
+- only log errors (docker/podman logs work otherwise)

base/Dockerfile

@@ -55,6 +55,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninte
     libreadline-dev \
 # Automation
     file \
+    python3-requests \
     unzip
 
 RUN rm --recursive /var/lib/apt/lists /var/cache/apt/archives

base/Makefile

@@ -1,13 +1,15 @@
 all: build run
 tag ?= $$(date +%Y%m%d%H%M)
+user ?= sirrgb
+registry ?= docker.io
 
 build:
-	docker build . --tag docker.io/sirrgb/dockdroid-base:latest
+	docker build . --tag $(registry)/$(user)/dockdroid-base:latest --no-cache
 
 run:
-	docker run --interactive --tty docker.io/sirrgb/dockdroid-base:latest
+	docker run --interactive --tty $(registry)/$(user)/dockdroid-base:latest
 
 publish:
-	docker tag docker.io/sirrgb/dockdroid-base:latest docker.io/sirrgb/dockdroid-base:$(tag)
-	docker image push docker.io/sirrgb/dockdroid-base:$(tag)
-	docker image push docker.io/sirrgb/dockdroid-base:latest
+	docker tag $(registry)/$(user)/dockdroid-base:latest $(registry)/$(user)/dockdroid-base:$(tag)
+	docker image push $(registry)/$(user)/dockdroid-base:$(tag)
+	docker image push $(registry)/$(user)/dockdroid-base:latest

base/README.md

@@ -0,0 +1,10 @@
+Build the base image
+---
+To use this you need to have ```make``` installed on your system. Alternatively you can also reference the Makefile itself and run the commands manually.  
+You only want to alter the base image if you need to alter the dependencies, which you can also do in the Dockerfile in the root of this repo, but at the cost of build-time.
+
+You can simply build this base image by running `make build`  
+If you wish to configure the registry, user or tag you can do so by adding the parameters to the command like `make build user=exampleuser registry=example.com tag=exampletag`  
+
+To use your newly made image you have to adapt the Dockerfile in the above directory and replace the ```FROM docker.io/sirrgb/dockdroid-base:latest``` statement with ```FROM example.com/exampleuser/dockdroid-base:exampletag```  
+If you want to make your changes public you can run ```docker login example.com``` where example.com is your chosen registry and follow the given instructions. The same works for podman. ```make publish``` will push the image to that registry with your chosen tag and latest.