Skip to content

Change processing of commits to walk all commits only once #116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Kobzol wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Change processing of commits to walk all commits only once #116

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bb4ee23
Change processing of commits to walk all commits only once
Kobzol Aug 16, 2026
71bc677
Cache submodule repositories
Kobzol Aug 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
290 changes: 182 additions & 108 deletions src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::str::FromStr;
use std::sync::Mutex;
use std::time::Instant;
use std::{cmp, fmt, str};

mod config;
Expand Down Expand Up @@ -74,25 +75,6 @@ impl AuthorMap {
self.map.entry(author).or_default().extend(set);
}
}

/// Create a new `AuthorMap` containing just the commits present in the current
/// map but not the other one.
#[must_use]
fn difference(&self, other: &AuthorMap) -> AuthorMap {
let mut new = AuthorMap::new();
new.map.reserve(self.map.len());
for (author, set) in self.map.iter() {
if let Some(other_set) = other.map.get(author) {
let diff: HashSet<_> = set.difference(other_set).cloned().collect();
if !diff.is_empty() {
new.map.insert(author.clone(), diff);
}
} else {
new.map.insert(author.clone(), set.clone());
}
}
new
}
}

pub struct AuthorsWithScores {
Expand Down Expand Up @@ -256,6 +238,22 @@ impl fmt::Debug for VersionTag {
}
}

struct VersionCommits {
main: Vec<Oid>,
submodules: Vec<(Submodule, Vec<Oid>)>,
}

impl VersionCommits {
fn total_commits(&self) -> u64 {
self.main.len() as u64
+ self
.submodules
.iter()
.map(|(_, commits)| commits.len() as u64)
.sum::<u64>()
}
}

fn get_versions(repo: &Repository) -> Result<Vec<VersionTag>, Box<dyn std::error::Error>> {
let tags = repo
.tag_names(None)?
Expand Down Expand Up @@ -324,18 +322,12 @@ fn build_author_map(
repo: &Repository,
reviewers: &Reviewers,
mailmap: &Mailmap,
from: &str,
to: &str,
commits: &[Oid],
) -> Result<AuthorMap, Box<dyn std::error::Error>> {
match build_author_map_(repo, reviewers, mailmap, from, to) {
match build_author_map_(repo, reviewers, mailmap, commits) {
Ok(o) => Ok(o),
Err(err) => Err(ErrorContext(
format!(
"build_author_map(repo={}, from={:?}, to={:?})",
repo.path().display(),
from,
to
),
format!("build_author_map(repo={})", repo.path().display(),),
err,
))?,
}
Expand Down Expand Up @@ -490,33 +482,11 @@ fn build_author_map_(
repo: &Repository,
reviewers: &Reviewers,
mailmap: &Mailmap,
from: &str,
to: &str,
commits: &[Oid],
) -> Result<AuthorMap, Box<dyn std::error::Error>> {
let mut walker = repo.revwalk()?;

if repo.revparse_single(to).is_err() {
// If a commit is not found, try fetching it.
git(&[
"--git-dir",
repo.path().to_str().unwrap(),
"fetch",
"origin",
to,
])?;
}

if from.is_empty() {
let to = repo.revparse_single(to)?.peel_to_commit()?.id();
walker.push(to)?;
} else {
walker.push_range(&format!("{}..{}", from, to))?;
}

let mut author_map = AuthorMap::new();
for oid in walker {
let oid = oid?;
let commit = repo.find_commit(oid)?;
for oid in commits {
let commit = repo.find_commit(*oid)?;

let mut commit_authors = Vec::new();
if !is_rollup_commit(&commit) {
Expand All @@ -543,7 +513,7 @@ fn build_author_map_(
commit_authors.extend(commit_coauthors(&commit));
for author in commit_authors {
let author = mailmap.canonicalize(&author);
author_map.add(author, oid);
author_map.add(author, *oid);
}
}
Ok(author_map)
Expand All @@ -567,38 +537,6 @@ fn mailmap_from_repo(repo: &git2::Repository) -> Result<Mailmap, Box<dyn std::er
Mailmap::from_string(file)
}

fn up_to_release(
repo: &Repository,
reviewers: &Reviewers,
mailmap: &Mailmap,
to: &VersionTag,
) -> Result<AuthorMap, Box<dyn std::error::Error>> {
let to_commit = repo.find_commit(to.commit).map_err(|e| {
ErrorContext(
format!(
"find_commit: repo={}, commit={}",
repo.path().display(),
to.commit
),
Box::new(e),
)
})?;
let modules = get_submodules(repo, &to_commit)?;

let mut author_map = build_author_map(repo, reviewers, mailmap, "", &to.raw_tag)
.map_err(|e| ErrorContext(format!("Up to {}", to), e))?;

for module in &modules {
let path = update_repo(&module.repository)?;
let subrepo = Repository::open(&path)?;
let submap =
build_author_map(&subrepo, reviewers, mailmap, "", &module.commit.to_string())?;
author_map.extend(submap);
}

Ok(author_map)
}

fn generate_thanks() -> Result<BTreeMap<VersionTag, AuthorMap>, Box<dyn std::error::Error>> {
let path = update_repo("https://github.com/rust-lang/rust.git")?;
let repo = git2::Repository::open(&path)?;
Expand Down Expand Up @@ -657,37 +595,173 @@ fn generate_thanks() -> Result<BTreeMap<VersionTag, AuthorMap>, Box<dyn std::err
in_progress: true,
});

let mut version_map = BTreeMap::new();
let start = Instant::now();

let by_version = gather_all_commits(&repo, versions)?;

let version_count = by_version.len();
let commit_count = by_version.values().map(|v| v.total_commits()).sum::<u64>();
eprintln!(
"Gathered {version_count} versions with {commit_count} total commits in {:.2}s",
start.elapsed().as_secs_f64()
);

let start = Instant::now();
let version_map = by_version
.into_iter()
.map(|(version, data)| {
let mut author_map = build_author_map(&repo, &reviewers, &mailmap, &data.main)?;
for (submodule, commits) in data.submodules {
let path = update_repo(&submodule.repository)?;
let subrepo = Repository::open(path)?;
author_map.extend(build_author_map(&subrepo, &reviewers, &mailmap, &commits)?);
}
Ok::<_, Box<dyn std::error::Error>>((version, author_map))
})
.collect::<Result<_, _>>()?;
eprintln!(
"Analyzed contributions in {:.2}s",
start.elapsed().as_secs_f64()
);

let mut cache = HashMap::new();
Ok(version_map)
}

for (idx, version) in versions.iter().enumerate() {
let previous = if let Some(v) = idx.checked_sub(1).map(|idx| &versions[idx]) {
v
/// Gather all commits for the given versions from the given repository, including all its
/// submodules.
/// The commits are grouped by the individual versions.
fn gather_all_commits(
repo: &Repository,
versions: Vec<VersionTag>,
) -> Result<HashMap<VersionTag, VersionCommits>, Box<dyn std::error::Error>> {
// Set of all commits that we visited
let mut seen_commits = HashSet::new();

let mut submodule_last_oid = HashMap::new();
let mut last_version_oid: Option<Oid> = None;

let mut by_version: HashMap<VersionTag, VersionCommits> = HashMap::new();

// Re-opening the same repository multiple times causes us to unpack its object database
// repeatedly. If we cache the repositories, this doesn't have to happen.
let mut subrepo_cache: HashMap<String, Repository> = HashMap::new();

// Iterate all version from the oldest to the newest
for version in &versions {
let mut walk = repo.revwalk()?;

// If we have a previous version, iterate from its commit to this commit
// Note that stable version tag commits are usually "forked" off the commit mainline
// Revwalk should take that into account
if let Some(last) = last_version_oid {
// Note: the left side of this range is exclusive, but that is what we want, because we
// already visited the `last` commit in the previous version
walk.push_range(&format!("{last}..{}", version.commit))?;
} else {
let author_map = build_author_map(&repo, &reviewers, &mailmap, "", &version.raw_tag)?;
version_map.insert(version.clone(), author_map);
continue;
};
// If there is no previous version, iterate from this version to the start of the
// commit history.
walk.push(version.commit)?;
}

eprintln!("Processing {:?} to {:?}", previous, version);
last_version_oid = Some(version.commit);

cache.insert(
version,
up_to_release(&repo, &reviewers, &mailmap, version)?,
// All commits of this version from the main repo. This is kept in a Vec for deterministic
// order.
let mut version_commits = vec![];
for commit in walk {
let commit = commit?;
version_commits.push(commit);
}
let mut submodule_commits = vec![];

// Now find all submodules present in the repo at this commit
let commit = repo.find_commit(version.commit)?;
let modules = get_submodules(repo, &commit)?;
for submodule in modules {
let path = update_repo(&submodule.repository)?;

let subrepo = subrepo_cache.get(&submodule.repository);
let subrepo = match subrepo {
Some(subrepo) => subrepo,
None => {
// We do not use the entry API here because `open` returns a result
subrepo_cache.insert(submodule.repository.clone(), Repository::open(&path)?);
subrepo_cache.get(&submodule.repository).unwrap()
}
};

// Iterate commits of the submodule
let mut subwalk = subrepo.revwalk()?;

// If we know a previous commit of the same submodule from a previous main version,
// then use that to stop the walk
let last_commit = submodule_last_oid.get(&submodule.repository);
if let Some(submodule_last) = last_commit {
// If the submodule didn't change across versions, ignore the submodule for this
// version.
if submodule_last == &submodule.commit {
continue;
}
subwalk.push_range(&format!("{}..{}", submodule_last, submodule.commit))?;
} else {
subwalk.push(submodule.commit)?;
}
submodule_last_oid.insert(submodule.repository.clone(), submodule.commit);

let mut commits = vec![];
for commit in subwalk {
let commit = commit?;
commits.push(commit);
}
submodule_commits.push((submodule, commits));
}

// If we encounter multiple commits across different versions for some reason,
// we always attribute them to the earliest version that encountered the commit.
// Since we iterate versions from oldest to newest, the retain below ensures that.
Comment thread
Kobzol marked this conversation as resolved.
version_commits.retain(|c| seen_commits.insert(*c));
for (_, commits) in &mut submodule_commits {
commits.retain(|c| seen_commits.insert(*c));
}

by_version.insert(
version.clone(),
VersionCommits {
main: version_commits,
submodules: submodule_commits,
},
);
let previous = match cache.remove(&previous) {
Some(v) => v,
None => up_to_release(&repo, &reviewers, &mailmap, previous)?,
};
let current = cache.get(&version).unwrap();
}

// Remove commits reachable from the previous release.
let only_current = current.difference(&previous);
version_map.insert(version.clone(), only_current);
// Sanity check: walk all commits and ensure that we saw them previously
let head = versions.last().unwrap().commit;
let mut walk = repo.revwalk()?;
walk.push(head)?;
for commit in walk {
let commit = commit?;
assert!(
seen_commits.contains(&commit),
"Commit {commit} was not visited"
);
}
let submodules = get_submodules(repo, &repo.find_commit(head)?)?;
for submodule in submodules {
let repo = subrepo_cache
.get(&submodule.repository)
.expect("Submodule repository not found");
let mut walk = repo.revwalk()?;
walk.push(submodule.commit)?;
for commit in walk {
let commit = commit?;
assert!(
seen_commits.contains(&commit),
"Submodule {} commit {commit} was not visited",
submodule.repository
);
}
}

Ok(version_map)
Ok(by_version)
}

enum OutputMode {
Expand Down