Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,22 @@ jobs:
go-version: "1.23"

- name: Build
run: go build -o /dev/null ./cmd/coginfer
run: CGO_ENABLED=0 go build -o /dev/null ./cmd/coginfer

- name: Lint
uses: golangci/golangci-lint-action@v6
uses: golangci/golangci-lint-action@v9
env:
CGO_ENABLED: 0
with:
version: latest
args: --timeout=3m

- name: Test
env:
CGO_ENABLED: 0
run: go test ./... -v -count=1

- name: Vet
env:
CGO_ENABLED: 0
run: go vet ./...
23 changes: 14 additions & 9 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ internal/llm/ — Backend interface + implementations
├── cgobackend.go — CgoBackend: Backend impl calling bridge functions (build tag: cgo)
└── loadopts.go — LoadOptions{NumCtx, GPULayers, Threads}
internal/model/ — Model scanning and metadata (.gguf file discovery)
vendor/llama.cpp/ — Git submodule, pinned to b9842
vendor/llama.cpp/ — Manually cloned (NOT a git submodule — no `.gitmodules` file)
```

### coginfer (Wide Model)
Expand All @@ -30,18 +30,27 @@ LLM inference server linking llama.cpp via a vendored CGo bridge, exposing an Ol
#### Build

```bash
# With CGo (production — requires cmake + gcc + submodule)
# With CGo (production — requires cmake + gcc + llama.cpp cloned into vendor/)
cd vendor/llama.cpp && cmake -B build -DLLAMA_NO_ACCELERATE=1 \
-DLLAMA_STATIC=1 -DLLAMA_NATIVE=0 \
-DBUILD_SHARED_LIBS=0 -DLLAMA_BUILD_TESTS=0 \
-DLLAMA_BUILD_EXAMPLES=0 -DLLAMA_BUILD_SERVER=0
cmake --build build --config Release -j$(nproc)
-DLLAMA_BUILD_EXAMPLES=0 -DLLAMA_BUILD_SERVER=0 \
-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY="$PWD/build"
cmake --build build --target llama --config Release -j$(nproc)
cd ../.. && CGO_ENABLED=1 go build -tags=cgo -o bin/coginfer ./cmd/coginfer

# Without CGo (CI, development — mock backend only)
CGO_ENABLED=0 go build -o bin/coginfer ./cmd/coginfer
```

**CGo flags** (in `internal/llm/bridge.go`):
- `-lllama` — NOT `-llama` (cmake target `llama` → `libllama.a`)
- `-L.../build/src` — modern llama.cpp puts static archives in `build/src/`
(workaround: `CMAKE_ARCHIVE_OUTPUT_DIRECTORY` puts it in `build/`)
- `-I.../ggml/include` — `llama.h` includes `ggml.h`, `ggml-cpu.h`, etc.
- `CGO_LDFLAGS` is appended **before** `#cgo LDFLAGS` by Go toolchain;
`-L` in env var establishes search path before `-l` in source

#### Usage

```bash
Expand All @@ -51,9 +60,6 @@ CGO_ENABLED=0 go build -o bin/coginfer ./cmd/coginfer
# Start with CGo llama.cpp bridge (production)
./bin/coginfer --backend cgo --models /cognitiveos/models

# Start with CGo llama.cpp bridge (production)
./bin/coginfer --backend cgo --models /cognitiveos/models

# Custom port and log file
./bin/coginfer --addr 127.0.0.1:11434 --log /cognitiveos/logs/inference.log
```
Expand All @@ -77,13 +83,12 @@ CGO_ENABLED=0 go build -o bin/coginfer ./cmd/coginfer

- **mock**: Simulated token generation with delays, no external dependencies. Default for development. Always available.
- **cgo**: In-process llama.cpp via CGo bridge. Requires `CGO_ENABLED=1` and `vendor/llama.cpp/build/libllama.a`. Production default.
- **cgo**: In-process llama.cpp via CGo bridge. Requires `CGO_ENABLED=1` and `vendor/llama.cpp/build/libllama.a`. Production default.

#### Build Tags

- `bridge.go`, `cgobackend.go`, `backend_cgo.go` — `//go:build cgo`
- `backend_stub.go` — `//go:build !cgo`
- CI runs `CGO_ENABLED=0`, excludes all CGo files automatically
- CI runs `CGO_ENABLED=0` (Ubuntu defaults to `CGO_ENABLED=1`, which includes `cgo` tag — must be explicit)
- `MockBackend` has no build tag — always available

### cograw (Raw Model — Firmware Guardrail)
Expand Down
2 changes: 1 addition & 1 deletion cmd/coginfer/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ func main() {
if err != nil {
log.Fatal(err)
}
defer f.Close()
defer func() { _ = f.Close() }()
log.SetOutput(f)
}

Expand Down
23 changes: 10 additions & 13 deletions cmd/cograw/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ var rawLog *log.Logger

func initRawLog(path string) {
dir := filepath.Dir(path)
os.MkdirAll(dir, 0755)
_ = os.MkdirAll(dir, 0755)
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640)
if err != nil {
log.Printf("WARN: cannot open raw audit log %s: %v", path, err)
Expand Down Expand Up @@ -215,7 +215,7 @@ func main() {
if err != nil {
log.Fatal(err)
}
defer f.Close()
defer func() { _ = f.Close() }()
log.SetOutput(f)
}

Expand All @@ -227,7 +227,7 @@ func main() {
}
logAudit("startup", fmt.Sprintf("raw model loaded: %s", *modelPath))

os.Remove(*socketPath)
_ = os.Remove(*socketPath)
addr, err := net.ResolveUnixAddr("unix", *socketPath)
if err != nil {
log.Fatalf("resolve addr: %v", err)
Expand All @@ -240,7 +240,7 @@ func main() {
if err := os.Chmod(*socketPath, 0600); err != nil {
log.Fatalf("chmod: %v", err)
}
defer os.Remove(*socketPath)
defer func() { _ = os.Remove(*socketPath) }()

log.Printf("cograw ready on %s (model: %s)", *socketPath, rm.model)

Expand All @@ -250,7 +250,7 @@ func main() {
go func() {
<-sigCh
log.Println("shutting down")
listener.Close()
_ = listener.Close()
}()

for {
Expand Down Expand Up @@ -283,7 +283,7 @@ func (r *RawModel) verifyModel(path string) error {
}

func handleConn(conn *net.UnixConn, rm *RawModel) {
defer conn.Close()
defer func() { _ = conn.Close() }()

decoder := json.NewDecoder(conn)
encoder := json.NewEncoder(conn)
Expand All @@ -295,7 +295,7 @@ func handleConn(conn *net.UnixConn, rm *RawModel) {
}

resp := dispatch(call, rm)
encoder.Encode(resp)
_ = encoder.Encode(resp)
}
}

Expand Down Expand Up @@ -499,12 +499,12 @@ func readMemAvailableMB() (int64, int64) {
for _, line := range lines {
if strings.HasPrefix(line, "MemTotal:") {
var kb int64
fmt.Sscanf(line, "MemTotal: %d kB", &kb)
_, _ = fmt.Sscanf(line, "MemTotal: %d kB", &kb)
totalMB = kb / 1024
}
if strings.HasPrefix(line, "MemAvailable:") {
var kb int64
fmt.Sscanf(line, "MemAvailable: %d kB", &kb)
_, _ = fmt.Sscanf(line, "MemAvailable: %d kB", &kb)
availableMB = kb / 1024
}
}
Expand All @@ -519,10 +519,7 @@ func handleAudit(call RPCCall, rm *RawModel) RPCResp {

totalMB, freeMB := readMemAvailableMB()

allowed := true
if params.RequestedMB > 0 && params.RequestedMB > freeMB {
allowed = false
}
allowed := params.RequestedMB <= 0 || params.RequestedMB <= freeMB

return RPCResp{
JSONRPC: "2.0",
Expand Down
4 changes: 2 additions & 2 deletions internal/llm/bridge.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
package llm

/*
#cgo LDFLAGS: -L${SRCDIR}/../../vendor/llama.cpp/build -llama -lm -lstdc++
#cgo CFLAGS: -I${SRCDIR}/../../vendor/llama.cpp/include
#cgo LDFLAGS: -L${SRCDIR}/../../vendor/llama.cpp/build -L${SRCDIR}/../../vendor/llama.cpp/build/src -lllama -lm -lstdc++
#cgo CFLAGS: -I${SRCDIR}/../../vendor/llama.cpp/include -I${SRCDIR}/../../vendor/llama.cpp/ggml/include

#include <stdlib.h>
#include "llama.h"
Expand Down
4 changes: 2 additions & 2 deletions internal/llm/cgobackend.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ func (c *CgoBackend) Load(modelPath string, opts *LoadOptions) (*ModelInfo, erro
return nil, fmt.Errorf("E_MODEL_NOT_FOUND: %s: %w", modelPath, err)
}
magic := make([]byte, 4)
f.Read(magic)
f.Close()
_, _ = f.Read(magic)
_ = f.Close()
if string(magic) != "GGUF" {
return nil, fmt.Errorf("E_MODEL_LOAD_FAILED: invalid GGUF magic bytes in %s", modelPath)
}
Expand Down
8 changes: 4 additions & 4 deletions internal/llm/llm_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ func TestMockBackendLoad(t *testing.T) {

func TestMockBackendUnload(t *testing.T) {
b := NewMockBackend()
b.Load("/cognitiveos/models/test.gguf", nil)
_, _ = b.Load("/cognitiveos/models/test.gguf", nil)
if err := b.Unload(); err != nil {
t.Fatalf("unload: %v", err)
}
Expand All @@ -38,7 +38,7 @@ func TestMockBackendUnload(t *testing.T) {

func TestMockBackendClose(t *testing.T) {
b := NewMockBackend()
b.Load("/cognitiveos/models/test.gguf", nil)
_, _ = b.Load("/cognitiveos/models/test.gguf", nil)
if err := b.Close(); err != nil {
t.Fatalf("close: %v", err)
}
Expand All @@ -57,7 +57,7 @@ func TestMockBackendGenerateWithoutLoad(t *testing.T) {

func TestMockBackendGenerate(t *testing.T) {
b := NewMockBackend()
b.Load("/cognitiveos/models/test.gguf", nil)
_, _ = b.Load("/cognitiveos/models/test.gguf", nil)

resp, err := b.Generate(GenerateReq{Prompt: "What is 2+2?"}, nil)
if err != nil {
Expand All @@ -79,7 +79,7 @@ func TestMockBackendGenerate(t *testing.T) {

func TestMockBackendGenerateStreaming(t *testing.T) {
b := NewMockBackend()
b.Load("/cognitiveos/models/test.gguf", nil)
_, _ = b.Load("/cognitiveos/models/test.gguf", nil)

var tokens []string
onToken := func(tok string) {
Expand Down
2 changes: 1 addition & 1 deletion internal/model/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func (m *Manager) Resolve(name string) (string, error) {

// Search recursively
var found string
filepath.Walk(m.ModelDir, func(path string, info os.FileInfo, err error) error {
_ = filepath.Walk(m.ModelDir, func(path string, info os.FileInfo, err error) error {
if err != nil || found != "" {
return nil
}
Expand Down
8 changes: 4 additions & 4 deletions internal/model/model_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
func TestManagerResolveAbsPath(t *testing.T) {
dir := t.TempDir()
modelPath := filepath.Join(dir, "test.gguf")
os.WriteFile(modelPath, []byte("dummy"), 0644)
_ = os.WriteFile(modelPath, []byte("dummy"), 0644)

m := NewManager(dir)
got, err := m.Resolve(modelPath)
Expand All @@ -31,9 +31,9 @@ func TestManagerResolveNotFound(t *testing.T) {

func TestManagerList(t *testing.T) {
dir := t.TempDir()
os.MkdirAll(filepath.Join(dir, "wide", "active"), 0755)
os.WriteFile(filepath.Join(dir, "raw-model.gguf"), []byte("raw"), 0644)
os.WriteFile(filepath.Join(dir, "wide", "active", "gemma.gguf"), []byte("gemma"), 0644)
_ = os.MkdirAll(filepath.Join(dir, "wide", "active"), 0755)
_ = os.WriteFile(filepath.Join(dir, "raw-model.gguf"), []byte("raw"), 0644)
_ = os.WriteFile(filepath.Join(dir, "wide", "active", "gemma.gguf"), []byte("gemma"), 0644)

m := NewManager(dir)
models, err := m.List()
Expand Down
12 changes: 6 additions & 6 deletions internal/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ func (s *Server) handleGenerate(w http.ResponseWriter, r *http.Request) {

onToken := func(token string) {
line, _ := json.Marshal(llm.GenerateResp{Response: token, Done: false})
fmt.Fprintf(w, "%s\n", line)
_, _ = fmt.Fprintf(w, "%s\n", line)
flusher.Flush()
}

Expand All @@ -242,7 +242,7 @@ func (s *Server) handleGenerate(w http.ResponseWriter, r *http.Request) {
}
resp.Done = true
line, _ := json.Marshal(resp)
fmt.Fprintf(w, "%s\n", line)
_, _ = fmt.Fprintf(w, "%s\n", line)
flusher.Flush()
return
}
Expand Down Expand Up @@ -273,7 +273,7 @@ func (s *Server) handleChat(w http.ResponseWriter, r *http.Request) {

var promptBuilder strings.Builder
for _, msg := range req.Messages {
promptBuilder.WriteString(fmt.Sprintf("%s: %s\n", msg.Role, msg.Content))
fmt.Fprintf(&promptBuilder, "%s: %s\n", msg.Role, msg.Content)
}
prompt := promptBuilder.String()

Expand Down Expand Up @@ -312,7 +312,7 @@ func (s *Server) handleChat(w http.ResponseWriter, r *http.Request) {

onToken := func(token string) {
line, _ := json.Marshal(llm.GenerateResp{Response: token, Done: false})
fmt.Fprintf(w, "%s\n", line)
_, _ = fmt.Fprintf(w, "%s\n", line)
flusher.Flush()
}

Expand All @@ -323,7 +323,7 @@ func (s *Server) handleChat(w http.ResponseWriter, r *http.Request) {
}
resp.Done = true
line, _ := json.Marshal(resp)
fmt.Fprintf(w, "%s\n", line)
_, _ = fmt.Fprintf(w, "%s\n", line)
flusher.Flush()
return
}
Expand Down Expand Up @@ -444,7 +444,7 @@ func (s *Server) handleDelete(w http.ResponseWriter, r *http.Request) {

var req deleteRequest
if r.Method == http.MethodPost {
json.NewDecoder(r.Body).Decode(&req)
_ = json.NewDecoder(r.Body).Decode(&req)
}

s.mu.Lock()
Expand Down
Loading
Loading