Merge pull request libgit2#6557 from libgit2/ethomson/shallow

Shallow (libgit2#6396) with some fixes from review

Author: ethomson

include/git2/errors.h

@@ -109,7 +109,8 @@ typedef enum {
 	GIT_ERROR_WORKTREE,
 	GIT_ERROR_SHA,
 	GIT_ERROR_HTTP,
-	GIT_ERROR_INTERNAL
+	GIT_ERROR_INTERNAL,
+	GIT_ERROR_GRAFTS
 } git_error_t;
 
 /**

include/git2/remote.h

@@ -702,6 +702,15 @@ typedef enum {
 	GIT_REMOTE_DOWNLOAD_TAGS_ALL
 } git_remote_autotag_option_t;
 
+/** Constants for fetch depth (shallowness of fetch). */
+typedef enum {
+	/** The fetch is "full" (not shallow). This is the default. */
+	GIT_FETCH_DEPTH_FULL = 0,
+
+	/** The fetch should "unshallow" and fetch missing data. */
+	GIT_FETCH_DEPTH_UNSHALLOW = 2147483647
+} git_fetch_depth_t;
+
 /**
  * Fetch options structure.
  *
@@ -743,6 +752,15 @@ typedef struct {
 	 */
 	git_proxy_options proxy_opts;
 
+	/**
+	 * Depth of the fetch to perform, or `GIT_FETCH_DEPTH_FULL`
+	 * (or `0`) for full history, or `GIT_FETCH_DEPTH_UNSHALLOW`
+	 * to "unshallow" a shallow repository.
+	 *
+	 * The default is full (`GIT_FETCH_DEPTH_FULL` or `0`).
+	 */
+	int depth;
+
 	/**
 	 * Whether to allow off-site redirects.  If this is not
 	 * specified, the `http.followRedirects` configuration setting

include/git2/sys/transport.h

@@ -25,6 +25,14 @@
 
 GIT_BEGIN_DECL
 
+typedef struct {
+	const git_remote_head * const *refs;
+	size_t refs_len;
+	git_oid *shallow_roots;
+	size_t shallow_roots_len;
+	int depth;
+} git_fetch_negotiation;
+
 struct git_transport {
 	unsigned int version; /**< The struct version */
 
@@ -96,8 +104,17 @@ struct git_transport {
 	int GIT_CALLBACK(negotiate_fetch)(
 		git_transport *transport,
 		git_repository *repo,
-		const git_remote_head * const *refs,
-		size_t count);
+		const git_fetch_negotiation *fetch_data);
+
+	/**
+	 * Return the shallow roots of the remote.
+	 *
+	 * This function may be called after a successful call to
+	 * `negotiate_fetch`.
+	 */
+	int GIT_CALLBACK(shallow_roots)(
+		git_oidarray *out,
+		git_transport *transport);
 
 	/**
 	 * Start downloading the packfile from the remote repository.

src/cli/cmd_clone.c

@@ -18,7 +18,7 @@
 
 #define COMMAND_NAME "clone"
 
-static char *branch, *remote_path, *local_path;
+static char *branch, *remote_path, *local_path, *depth;
 static int show_help, quiet, checkout = 1, bare;
 static bool local_path_exists;
 static cli_progress progress = CLI_PROGRESS_INIT;
@@ -36,6 +36,8 @@ static const cli_opt_spec opts[] = {
 	  CLI_OPT_USAGE_DEFAULT,   NULL,         "don't create a working directory" },
 	{ CLI_OPT_TYPE_VALUE,     "branch",      'b', &branch,      0,
 	  CLI_OPT_USAGE_DEFAULT,  "name",        "branch to check out" },
+	{ CLI_OPT_TYPE_VALUE,     "depth",       0,   &depth,       0,
+	  CLI_OPT_USAGE_DEFAULT,  "depth",       "commit depth to check out " },
 	{ CLI_OPT_TYPE_LITERAL },
 	{ CLI_OPT_TYPE_ARG,       "repository",   0,  &remote_path, 0,
 	  CLI_OPT_USAGE_REQUIRED, "repository",  "repository path" },
@@ -71,6 +73,22 @@ static char *compute_local_path(const char *orig_path)
 	return local_path;
 }
 
+static int compute_depth(const char *depth)
+{
+	int64_t i;
+	const char *endptr;
+
+	if (!depth)
+		return 0;
+
+	if (git__strntol64(&i, depth, strlen(depth), &endptr, 10) < 0 || i < 0 || i > INT_MAX || *endptr) {
+		fprintf(stderr, "fatal: depth '%s' is not valid.\n", depth);
+		exit(128);
+	}
+
+	return (int)i;
+}
+
 static bool validate_local_path(const char *path)
 {
 	if (!git_fs_path_exists(path))
@@ -127,11 +145,9 @@ int cmd_clone(int argc, char **argv)
 		goto done;
 	}
 
-	if (bare)
-		clone_opts.bare = 1;
-
-	if (branch)
-		clone_opts.checkout_branch = branch;
+	clone_opts.bare = !!bare;
+	clone_opts.checkout_branch = branch;
+	clone_opts.fetch_opts.depth = compute_depth(depth);
 
 	if (!checkout)
 		clone_opts.checkout_opts.checkout_strategy = GIT_CHECKOUT_NONE;

src/libgit2/clone.c

@@ -420,7 +420,9 @@ static int clone_into(
 
 	memcpy(&fetch_opts, opts, sizeof(git_fetch_options));
 	fetch_opts.update_fetchhead = 0;
-	fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL;
+
+	if (!opts->depth)
+		fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL;
 
 	if ((error = git_remote_connect_options__from_fetch_opts(&connect_opts, remote, &fetch_opts)) < 0)
 		goto cleanup;

src/libgit2/commit.c

@@ -22,6 +22,7 @@
 #include "object.h"
 #include "array.h"
 #include "oidarray.h"
+#include "grafts.h"
 
 void git_commit__free(void *_commit)
 {
@@ -427,10 +428,6 @@ static int commit_parse(
 		buffer += tree_len;
 	}
 
-	/*
-	 * TODO: commit grafts!
-	 */
-
 	while (git_object__parse_oid_header(&parent_id,
 			&buffer, buffer_end, "parent ",
 			opts->oid_type) == 0) {
@@ -532,16 +529,41 @@ int git_commit__parse_raw(
 	return commit_parse(commit, data, size, &parse_options);
 }
 
+static int assign_commit_parents_from_graft(git_commit *commit, git_commit_graft *graft) {
+	size_t idx;
+	git_oid *oid;
+
+	git_array_clear(commit->parent_ids);
+	git_array_init_to_size(commit->parent_ids, git_array_size(graft->parents));
+	git_array_foreach(graft->parents, idx, oid) {
+		git_oid *id = git_array_alloc(commit->parent_ids);
+		GIT_ERROR_CHECK_ALLOC(id);
+
+		git_oid_cpy(id, oid);
+	}
+
+	return 0;
+}
+
 int git_commit__parse_ext(
 	git_commit *commit,
 	git_odb_object *odb_obj,
 	git_commit__parse_options *parse_opts)
 {
-	return commit_parse(
-		commit,
-		git_odb_object_data(odb_obj),
-		git_odb_object_size(odb_obj),
-		parse_opts);
+	git_repository *repo = git_object_owner((git_object *)commit);
+	git_commit_graft *graft;
+	int error;
+
+	if ((error = commit_parse(commit, git_odb_object_data(odb_obj),
+				  git_odb_object_size(odb_obj), parse_opts)) < 0)
+		return error;
+
+	/* Perform necessary grafts */
+	if (git_grafts_get(&graft, repo->grafts, git_odb_object_id(odb_obj)) != 0 &&
+		git_grafts_get(&graft, repo->shallow_grafts, git_odb_object_id(odb_obj)) != 0)
+		return 0;
+
+	return assign_commit_parents_from_graft(commit, graft);
 }
 
 #define GIT_COMMIT_GETTER(_rvalue, _name, _return, _invalid) \

src/libgit2/fetch.c

@@ -20,6 +20,7 @@
 #include "netops.h"
 #include "repository.h"
 #include "refs.h"
+#include "transports/smart.h"
 
 static int maybe_want(git_remote *remote, git_remote_head *head, git_refspec *tagspec, git_remote_autotag_option_t tagopt)
 {
@@ -59,8 +60,10 @@ static int mark_local(git_remote *remote)
 		return -1;
 
 	git_vector_foreach(&remote->refs, i, head) {
-		/* If we have the object, mark it so we don't ask for it */
-		if (git_odb_exists(odb, &head->oid))
+		/* If we have the object, mark it so we don't ask for it.
+		   However if we are unshallowing, we need to ask for it
+		   even though the head exists locally. */
+		if (remote->nego.depth != INT_MAX && git_odb_exists(odb, &head->oid))
 			head->local = 1;
 		else
 			remote->need_pack = 1;
@@ -166,9 +169,15 @@ static int filter_wants(git_remote *remote, const git_fetch_options *opts)
 int git_fetch_negotiate(git_remote *remote, const git_fetch_options *opts)
 {
 	git_transport *t = remote->transport;
+	int error;
 
 	remote->need_pack = 0;
 
+	if (opts) {
+		GIT_ASSERT_ARG(opts->depth >= 0);
+		remote->nego.depth = opts->depth;
+	}
+
 	if (filter_wants(remote, opts) < 0)
 		return -1;
 
@@ -180,20 +189,40 @@ int git_fetch_negotiate(git_remote *remote, const git_fetch_options *opts)
 	 * Now we have everything set up so we can start tell the
 	 * server what we want and what we have.
 	 */
-	return t->negotiate_fetch(t,
+	remote->nego.refs = (const git_remote_head * const *)remote->refs.contents;
+	remote->nego.refs_len = remote->refs.length;
+
+	if (git_repository__shallow_roots(&remote->nego.shallow_roots,
+	                                  &remote->nego.shallow_roots_len,
+	                                  remote->repo) < 0)
+		return -1;
+
+	error = t->negotiate_fetch(t,
 		remote->repo,
-		(const git_remote_head * const *)remote->refs.contents,
-		remote->refs.length);
+		&remote->nego);
+
+	git__free(remote->nego.shallow_roots);
+
+	return error;
 }
 
 int git_fetch_download_pack(git_remote *remote)
 {
+	git_oidarray shallow_roots = { NULL };
 	git_transport *t = remote->transport;
+	int error;
 
 	if (!remote->need_pack)
 		return 0;
 
-	return t->download_pack(t, remote->repo, &remote->stats);
+	if ((error = t->download_pack(t, remote->repo, &remote->stats)) != 0 ||
+	    (error = t->shallow_roots(&shallow_roots, t)) != 0)
+		return error;
+
+	error = git_repository__shallow_roots_write(remote->repo, &shallow_roots);
+
+	git_oidarray_dispose(&shallow_roots);
+	return error;
 }
 
 int git_fetch_options_init(git_fetch_options *opts, unsigned int version)