Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion check_files.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,16 @@ func (c *checker) undeployed(f *FileFacts, repo *Repo, paths []string) map[strin
return units
}

// release compares every PHP file outside uploads with the local install:
// release compares every PHP file outside uploads with the local install,
// and one that differs in a git-sourced plugin with composer.lock's ref:
// one flag per package or theme, its ID covering every differing file, so
// accepting it holds until any of them changes again.
func (c *checker) release(f *FileFacts, repo *Repo, paths []string, undeployed map[string]bool) {
content := f.Content
type diff struct{ changed, extra []string }
diffs := map[string]*diff{}
skipped := map[string]int{}
edited := map[string]int{} // files production has as composer.lock's ref has them, the checkout not
for _, p := range paths {
if strings.HasPrefix(p, content+"/uploads/") {
continue
Expand Down Expand Up @@ -130,6 +132,9 @@ func (c *checker) release(f *FileFacts, repo *Repo, paths []string, undeployed m
case !ok && (strings.Contains(p, "/build/") || strings.Contains(p, "/dist/") && strings.HasSuffix(p, ".asset.php")):
skipped["built files the checkout hasn't built (build/, dist/*.asset.php)"]++
continue
case repo.AtLock(p, f.Files[p]):
edited[u]++
continue
}
d := diffs[u]
if d == nil {
Expand Down Expand Up @@ -160,6 +165,9 @@ func (c *checker) release(f *FileFacts, repo *Repo, paths []string, undeployed m
for _, what := range slices.Sorted(maps.Keys(skipped)) {
c.info("%d PHP file(s) not compared: %s", skipped[what], what)
}
for _, u := range slices.Sorted(maps.Keys(edited)) {
c.info("%s: %d PHP file(s) match composer.lock's ref but not the checkout's install, which was edited: compared with the ref", u, edited[u])
}
// The collector runs core, mu-plugins, drop-ins and the site's wp-cli; the
// hasher doesn't. When those differ, the collector's answers are suspect.
var ran []string
Expand Down
53 changes: 49 additions & 4 deletions check_files_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,45 @@ func TestReleaseComparedWithDeployedCommit(t *testing.T) {
}
}

// A git-sourced plugin whose checkout copy was edited by hand: production's
// file as composer.lock's ref has it isn't flagged (a note says so); one the
// ref doesn't have, or that GitHub can't serve, still is.
func TestReleaseEditedCheckoutComparedWithLockRef(t *testing.T) {
sum := func(s string) string { b := sha256.Sum256([]byte(s)); return hex.EncodeToString(b[:]) }
atRef := map[string]string{"inc/a.php": "<?php // a", "inc/b.php": "<?php // b"}
orig := fileAtRef
t.Cleanup(func() { fileAtRef = orig })
var asked []string
fileAtRef = func(repo, ref, path string) ([]byte, error) {
asked = append(asked, repo+"@"+ref+":"+path)
if s, ok := atRef[path]; ok {
return []byte(s), nil
}
return nil, os.ErrNotExist
}
local := map[string]string{
"web/app/plugins/fork/inc/a.php": sum("<?php // a, edited"),
"web/app/plugins/fork/inc/b.php": sum("<?php // b, edited"),
}
server := map[string]string{
"web/app/plugins/fork/inc/a.php": sum("<?php // a"), // the ref's
"web/app/plugins/fork/inc/b.php": sum("<?php // b, bad"), // neither
}
repo := &Repo{Plugins: map[string]bool{"fork": true}, MuPlugins: map[string]bool{}, Dropins: map[string]bool{}, Revision: "abc123def", Local: local,
Sources: map[string]Source{"web/app/plugins/fork": {Repo: "acme/fork", Ref: "fc60653"}}}
r := Check(Input{Facts: Facts{At: now, Files: files("abc123", server)}, Repo: repo}).Report
rel := kind(r, KindRelease)
if len(rel) != 1 || !strings.Contains(rel[0].Message, "1 changed, 0 not in the release") || !strings.Contains(rel[0].Untrusted, "inc/b.php") || strings.Contains(rel[0].Untrusted, "inc/a.php") {
t.Fatalf("release flags %+v", rel)
}
if n := noteTexts(r); !strings.Contains(n, "web/app/plugins/fork: 1 PHP file(s) match composer.lock's ref") {
t.Fatalf("notes %s", n)
}
if !slices.Contains(asked, "acme/fork@fc60653:inc/a.php") {
t.Fatalf("asked %v", asked)
}
}

func TestExposed(t *testing.T) {
f := &FileFacts{Content: "web/app", Data: []DataFile{
{Path: "web/app/uploads/wpallimport/files/asiakkaat_2019.csv", Size: 10, Mtime: 1551900000},
Expand Down Expand Up @@ -364,14 +403,17 @@ func TestReadRepo(t *testing.T) {
"web/app/mu-plugins/{$name}/":["type:wordpress-muplugin","wp-plugin/moved"],
"web/app/plugins/{$name}/":["type:wordpress-plugin"]},
"dropin-paths":{"web/app/":["package:wp-plugin/cache:wp-cache-phase1.php"],"web/app/mu-plugins":["package:vendor/cli:plugin/login-server.php"]},
"patches":{"wp-plugin/fixed":{},"roots/wordpress-no-content":{}}}}`)
"patches":{"wp-plugin/fixed":{},"roots/wordpress-no-content":{},"acme/patched-fork":{}}}}`)
write("data-audit.json", `{"public_roles":{"members.example":["subscriber"]},"client_domains":["client.example"]}`)
write("composer.lock", `{"packages":[
{"name":"wp-plugin/fixed","type":"wordpress-plugin"},
{"name":"wp-plugin/moved","type":"wordpress-plugin"},
{"name":"vendor/mu","type":"wordpress-muplugin"},
{"name":"vendor/renamed-package","type":"wordpress-plugin","extra":{"installer-name":"renamed"}},
{"name":"vendor/lib","type":"library"}],
{"name":"vendor/lib","type":"library","source":{"type":"git","url":"https://github.com/vendor/lib.git","reference":"1111"}},
{"name":"acme/fork","type":"wordpress-plugin","source":{"type":"git","url":"git@github.com:acme/fork.git","reference":"fc60653"}},
{"name":"acme/patched-fork","type":"wordpress-plugin","source":{"type":"git","url":"https://github.com/acme/patched-fork","reference":"2222"}},
{"name":"acme/zip","type":"wordpress-plugin","source":{"type":"zip","url":"https://example.com/zip.zip","reference":"3333"}}],
"packages-dev":[{"name":"wp-plugin/dev-only","type":"wordpress-plugin"}]}`)
write("web/app/mu-plugins/local.php", "<?php")
write("web/app/plugins/custom/custom.php", "<?php")
Expand Down Expand Up @@ -405,10 +447,13 @@ func TestReadRepo(t *testing.T) {
}
}
}
check("plugins", repo.Plugins, "fixed", "custom", "renamed", "woocommerce")
check("plugins", repo.Plugins, "fixed", "custom", "renamed", "woocommerce", "fork", "patched-fork", "zip")
if len(repo.Sources) != 1 || repo.Sources["web/app/plugins/fork"] != (Source{Repo: "acme/fork", Ref: "fc60653"}) {
t.Errorf("sources %v: only the unpatched plugin from GitHub", repo.Sources)
}
check("mu-plugins", repo.MuPlugins, "moved", "mu", "local.php", "login-server.php")
check("dropins", repo.Dropins, "object-cache.php", "wp-cache-phase1.php")
check("patched", repo.PatchedPlugins, "fixed")
check("patched", repo.PatchedPlugins, "fixed", "patched-fork")
if !repo.PatchedCore {
t.Error("core patch not seen")
}
Expand Down
73 changes: 68 additions & 5 deletions repo.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"hash"
"io"
"io/fs"
"net/url"
"os"
"os/exec"
"path/filepath"
Expand Down Expand Up @@ -64,9 +65,52 @@ type Repo struct {
// ships from the repo itself, whatever the working tree holds.
Tracked map[string]bool
TrackedAt string
dir string // the checkout, install or not
// Sources are the unpatched plugins and mu-plugins it installs from a
// GitHub repo, by folder ("web/app/plugins/x"): composer.lock's ref of
// each, for a file the checkout's install doesn't match (AtLock).
Sources map[string]Source
dir string // the checkout, install or not
}

// Source is a package's GitHub repo ("owner/name") and composer.lock's ref.
type Source struct{ Repo, Ref string }

// fileAtRef is path's content in a GitHub repo at ref (gh, read only).
var fileAtRef = func(repo, ref, path string) ([]byte, error) {
segs := strings.Split(path, "/")
for i, s := range segs {
segs[i] = url.PathEscape(s)
}
return exec.Command("gh", "api", "-H", "Accept: application/vnd.github.raw+json",
"repos/"+repo+"/contents/"+strings.Join(segs, "/")+"?ref="+url.QueryEscape(ref)).Output()
}

// AtLock reports whether p, a file of one of the Sources, has sha256 sum at
// its package's composer.lock ref. The checkout's install can be a copy that
// was edited by hand (worktrees clone it from the main checkout, and Composer
// doesn't reinstall a version it already has); the ref is what deploys ship.
func (r *Repo) AtLock(p, sum string) bool {
if r == nil {
return false
}
for dir, s := range r.Sources {
rest, ok := strings.CutPrefix(p, dir+"/")
if !ok {
continue
}
b, err := fileAtRef(s.Repo, s.Ref, rest)
if err != nil {
return false
}
got := sha256.Sum256(b)
return hex.EncodeToString(got[:]) == sum
}
return false
}

// githubRepo matches a git URL on github.com; its group is "owner/name".
var githubRepo = regexp.MustCompile(`^(?:git@github\.com:|https://github\.com/)([^/]+/[^/]+?)(?:\.git)?/?$`)

// LoadTracked reads the files git tracks at commit rev (one ls-tree), for a
// checkout that isn't at it, else GitHub's tree of that commit (gh, read
// only) when the checkout hasn't fetched it. Neither leaves Tracked nil:
Expand Down Expand Up @@ -181,13 +225,21 @@ func ReadRepo(dir string) (*Repo, error) {

where := map[string]map[string]bool{"plugins": exp.Plugins, "mu-plugins": exp.MuPlugins}
types := map[string]string{}
sources := map[string]Source{} // by package name, until the patched ones are known
folders := map[string]string{}
for _, p := range lock.Packages { // packages-dev aren't installed on production
types[p.Name] = p.Type
if vendor, slug, _ := strings.Cut(p.Name, "/"); p.Type == "wordpress-plugin" && (vendor == "wp-plugin" || vendor == "wpackagist-plugin") {
exp.WPOrgPlugins = append(exp.WPOrgPlugins, slug)
}
if set := where[installDir(p, cj.Extra.InstallerPaths)]; set != nil {
set[cmp.Or(p.Extra.InstallerName, base(p.Name))] = true
sub := installDir(p, cj.Extra.InstallerPaths)
if set := where[sub]; set != nil {
name := cmp.Or(p.Extra.InstallerName, base(p.Name))
set[name] = true
if m := githubRepo.FindStringSubmatch(p.Source.URL); m != nil && p.Source.Type == "git" && p.Source.Reference != "" {
sources[p.Name] = Source{Repo: m[1], Ref: p.Source.Reference}
folders[p.Name] = ContentDir + "/" + sub + "/" + name
}
}
}

Expand Down Expand Up @@ -234,6 +286,12 @@ func ReadRepo(dir string) (*Repo, error) {
exp.PatchedPlugins[base(name)] = true
}
}
exp.Sources = map[string]Source{}
for name, s := range sources {
if !patched[name] { // the ref doesn't have the patch the install does
exp.Sources[folders[name]] = s
}
}

if rev, err := exec.Command("git", "-C", dir, "rev-parse", "HEAD").Output(); err == nil {
exp.Revision = strings.TrimSpace(string(rev))
Expand Down Expand Up @@ -372,8 +430,13 @@ func hashFile(path string, h hash.Hash, buf []byte) (string, error) {
}

type lockPackage struct {
Name string `json:"name"`
Type string `json:"type"`
Name string `json:"name"`
Type string `json:"type"`
Source struct {
Type string `json:"type"`
URL string `json:"url"`
Reference string `json:"reference"`
} `json:"source"`
Extra struct {
InstallerName string `json:"installer-name"` // composer/installers' folder-name override
} `json:"extra"`
Expand Down
Loading