Skip to content

http-fetch: add --stdout for direct packfile fetches #2263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
friel-openai wants to merge 1 commit into
base: master
Choose a base branch
from
Open

http-fetch: add --stdout for direct packfile fetches #2263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions Documentation/git-http-fetch.adoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ git-http-fetch - Download from a remote Git repository via HTTP
SYNOPSIS
--------
[verse]
'git http-fetch' [-c] [-t] [-a] [-d] [-v] [-w <filename>] [--recover] [--stdin | --packfile=<hash> | <commit>] <URL>
'git http-fetch' [-c] [-t] [-a] [-d] [-v] [-w <filename>] [--recover]
[--stdout] [--stdin | --packfile=<hash> | <commit>] <URL>

DESCRIPTION
-----------
Expand Down Expand Up @@ -49,8 +50,14 @@ commit-id::
this case), 'git http-fetch' fetches the packfile directly at the given
URL and uses index-pack to generate corresponding .idx and .keep files.
The hash is used to determine the name of the temporary file and is
arbitrary. The output of index-pack is printed to stdout. Requires
--index-pack-args.
arbitrary. The output of index-pack is printed to stdout unless
`--stdout` is given. Requires `--index-pack-args` unless `--stdout`
is given.

--stdout::
For internal use only, and only valid with `--packfile=<hash>`.
Write the packfile contents to standard output instead of
downloading to a temporary file and running `index-pack`.

--index-pack-args=<args>::
For internal use only. The command to run on the contents of the
Expand Down
32 changes: 25 additions & 7 deletions http-fetch.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
#include "trace2.h"

static const char http_fetch_usage[] = "git http-fetch "
"[-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin | --packfile=hash | commit-id] url";
"[-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdout] [--stdin | --packfile=hash | commit-id] url";

static int fetch_using_walker(const char *raw_url, int get_verbosely,
int get_recover, int commits, char **commit_id,
Expand Down Expand Up @@ -54,19 +54,28 @@ static int fetch_using_walker(const char *raw_url, int get_verbosely,

static void fetch_single_packfile(struct object_id *packfile_hash,
const char *url,
const char **index_pack_args) {
const char **index_pack_args,
int packfile_to_stdout)
{
struct http_pack_request *preq;
struct slot_results results;
int ret;

http_init(NULL, url, 0);

preq = new_direct_http_pack_request(packfile_hash->hash, xstrdup(url));
if (packfile_to_stdout)
preq = new_direct_http_pack_request_to_stdout(packfile_hash->hash,
xstrdup(url));
else
preq = new_direct_http_pack_request(packfile_hash->hash,
xstrdup(url));
if (!preq)
die("couldn't create http pack request");
preq->slot->results = &results;
preq->index_pack_args = index_pack_args;
preq->preserve_index_pack_stdout = 1;
if (!packfile_to_stdout) {
preq->index_pack_args = index_pack_args;
preq->preserve_index_pack_stdout = 1;
}

if (start_active_slot(preq->slot)) {
run_active_slot(preq->slot);
Expand Down Expand Up @@ -104,6 +113,7 @@ int cmd_main(int argc, const char **argv)
int get_verbosely = 0;
int get_recover = 0;
int packfile = 0;
int packfile_to_stdout = 0;
int nongit;
struct object_id packfile_hash;
struct strvec index_pack_args = STRVEC_INIT;
Expand All @@ -126,6 +136,8 @@ int cmd_main(int argc, const char **argv)
usage(http_fetch_usage);
} else if (!strcmp(argv[arg], "--recover")) {
get_recover = 1;
} else if (!strcmp(argv[arg], "--stdout")) {
packfile_to_stdout = 1;
} else if (!strcmp(argv[arg], "--stdin")) {
commits_on_stdin = 1;
} else if (skip_prefix(argv[arg], "--packfile=", &p)) {
Expand Down Expand Up @@ -154,15 +166,21 @@ int cmd_main(int argc, const char **argv)
repo_config(the_repository, git_default_config, NULL);

if (packfile) {
if (!index_pack_args.nr)
if (packfile_to_stdout && index_pack_args.nr)
die(_("the option '%s' cannot be used with '%s'"),
"--stdout", "--index-pack-args");
if (!packfile_to_stdout && !index_pack_args.nr)
die(_("the option '%s' requires '%s'"), "--packfile", "--index-pack-args");

fetch_single_packfile(&packfile_hash, argv[arg],
index_pack_args.v);
index_pack_args.v,
packfile_to_stdout);
ret = 0;
goto out;
}

if (packfile_to_stdout)
die(_("the option '%s' requires '%s'"), "--stdout", "--packfile");
if (index_pack_args.nr)
die(_("the option '%s' requires '%s'"), "--index-pack-args", "--packfile");

Expand Down
57 changes: 43 additions & 14 deletions http.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2596,7 +2596,10 @@ int http_get_info_packs(const char *base_url, struct packfile_list *packs)
void release_http_pack_request(struct http_pack_request *preq)
{
if (preq->packfile) {
fclose(preq->packfile);
if (preq->stream_to_stdout)
fflush(preq->packfile);
else
fclose(preq->packfile);
preq->packfile = NULL;
}
preq->slot = NULL;
Expand All @@ -2612,9 +2615,16 @@ static const char *default_index_pack_args[] =
int finish_http_pack_request(struct http_pack_request *preq)
{
struct child_process ip = CHILD_PROCESS_INIT;
int tmpfile_fd;
int tmpfile_fd = -1;
int ret = 0;

if (preq->stream_to_stdout) {
if (fflush(preq->packfile))
return -1;
preq->packfile = NULL;
return 0;
}

fclose(preq->packfile);
preq->packfile = NULL;

Expand All @@ -2637,7 +2647,8 @@ int finish_http_pack_request(struct http_pack_request *preq)
}

cleanup:
close(tmpfile_fd);
if (tmpfile_fd >= 0)
close(tmpfile_fd);
unlink(preq->tmpfile.buf);
return ret;
}
Expand All @@ -2662,8 +2673,8 @@ struct http_pack_request *new_http_pack_request(
strbuf_detach(&buf, NULL));
}

struct http_pack_request *new_direct_http_pack_request(
const unsigned char *packed_git_hash, char *url)
static struct http_pack_request *new_direct_http_pack_request_1(
const unsigned char *packed_git_hash, char *url, int stream_to_stdout)
{
off_t prev_posn = 0;
struct http_pack_request *preq;
Expand All @@ -2672,14 +2683,19 @@ struct http_pack_request *new_direct_http_pack_request(
strbuf_init(&preq->tmpfile, 0);

preq->url = url;
preq->stream_to_stdout = !!stream_to_stdout;

odb_pack_name(the_repository, &preq->tmpfile, packed_git_hash, "pack");
strbuf_addstr(&preq->tmpfile, ".temp");
preq->packfile = fopen(preq->tmpfile.buf, "a");
if (!preq->packfile) {
error("Unable to open local file %s for pack",
preq->tmpfile.buf);
goto abort;
if (preq->stream_to_stdout) {
preq->packfile = stdout;
} else {
odb_pack_name(the_repository, &preq->tmpfile, packed_git_hash, "pack");
strbuf_addstr(&preq->tmpfile, ".temp");
preq->packfile = fopen(preq->tmpfile.buf, "a");
if (!preq->packfile) {
error("Unable to open local file %s for pack",
preq->tmpfile.buf);
goto abort;
}
}

preq->slot = get_active_slot();
Expand All @@ -2693,8 +2709,9 @@ struct http_pack_request *new_direct_http_pack_request(
* If there is data present from a previous transfer attempt,
* resume where it left off
*/
prev_posn = ftello(preq->packfile);
if (prev_posn>0) {
if (!preq->stream_to_stdout)
prev_posn = ftello(preq->packfile);
if (prev_posn > 0) {
if (http_is_verbose)
fprintf(stderr,
"Resuming fetch of pack %s at byte %"PRIuMAX"\n",
Expand All @@ -2712,6 +2729,18 @@ struct http_pack_request *new_direct_http_pack_request(
return NULL;
}

struct http_pack_request *new_direct_http_pack_request(
const unsigned char *packed_git_hash, char *url)
{
return new_direct_http_pack_request_1(packed_git_hash, url, 0);
}

struct http_pack_request *new_direct_http_pack_request_to_stdout(
const unsigned char *packed_git_hash, char *url)
{
return new_direct_http_pack_request_1(packed_git_hash, url, 1);
}

/* Helpers for fetching objects (loose) */
static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb,
void *data)
Expand Down
3 changes: 3 additions & 0 deletions http.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ struct http_pack_request {
*/
const char **index_pack_args;
unsigned preserve_index_pack_stdout : 1;
unsigned stream_to_stdout : 1;

FILE *packfile;
struct strbuf tmpfile;
Expand All @@ -226,6 +227,8 @@ struct http_pack_request *new_http_pack_request(
const unsigned char *packed_git_hash, const char *base_url);
struct http_pack_request *new_direct_http_pack_request(
const unsigned char *packed_git_hash, char *url);
struct http_pack_request *new_direct_http_pack_request_to_stdout(
const unsigned char *packed_git_hash, char *url);
int finish_http_pack_request(struct http_pack_request *preq);
void release_http_pack_request(struct http_pack_request *preq);

Expand Down
20 changes: 20 additions & 0 deletions t/t5550-http-fetch-dumb.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,26 @@ test_expect_success 'http-fetch --packfile' '
git -C packfileclient cat-file -e "$HASH"
'

test_expect_success 'http-fetch --packfile --stdout' '
ARBITRARY=$(git -C "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git rev-parse HEAD) &&

git init packfileclient-stdout &&
p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git && ls objects/pack/pack-*.pack) &&
git -C packfileclient-stdout http-fetch --packfile=$ARBITRARY \
--stdout "$HTTPD_URL"/dumb/repo_pack.git/$p |
git -C packfileclient-stdout index-pack --stdin --keep >out &&

grep -E "^keep.[0-9a-f]{16,}$" out &&
cut -c6- out >packhash &&

test -e "packfileclient-stdout/.git/objects/pack/pack-$(cat packhash).pack" &&
test -e "packfileclient-stdout/.git/objects/pack/pack-$(cat packhash).idx" &&
test -e "packfileclient-stdout/.git/objects/pack/pack-$(cat packhash).keep" &&

HASH=$(git -C "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git rev-parse HEAD) &&
git -C packfileclient-stdout cat-file -e "$HASH"
'

test_expect_success 'fetch notices corrupt pack' '
cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git &&
(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git &&
Expand Down
Loading