From patchwork Fri Mar 8 21:55:13 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jonathan Tan X-Patchwork-Id: 10845463 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 467611669 for ; Fri, 8 Mar 2019 21:55:41 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 325852CFF6 for ; Fri, 8 Mar 2019 21:55:41 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 25FC72FDA5; Fri, 8 Mar 2019 21:55:41 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-15.5 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI, USER_IN_DEF_DKIM_WL autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 804432CFF6 for ; Fri, 8 Mar 2019 21:55:40 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726381AbfCHVzi (ORCPT ); Fri, 8 Mar 2019 16:55:38 -0500 Received: from mail-it1-f201.google.com ([209.85.166.201]:37720 "EHLO mail-it1-f201.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726267AbfCHVzi (ORCPT ); Fri, 8 Mar 2019 16:55:38 -0500 Received: by mail-it1-f201.google.com with SMTP id q141so13172866itc.2 for ; Fri, 08 Mar 2019 13:55:38 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20161025; h=date:in-reply-to:message-id:mime-version:references:subject:from:to :cc; bh=/T1g0yV2QOlVxhv7pQ6ewK8IHn8UwKGZh5cup4X5Lcs=; b=cbm+k83DCYh2Po3dn1TlzcVKXABOi9jcjQ0BkkxF4bJp+RiyhcPQEqg79W7d9625sb vrXQKbXt3jIRFxz+yFNHtH8v4rR5SHItqEzK0n4wHID8xz0NIrFCX26BUyjP1MatAcsF E40bCUMofLIPAAf1vzGrJiy/6/+8YbYKkZef6ao19AFJnXkQk3m6ED00hZU0QF5rpMOf 8SntiU7I+sgW+PMlQ+YdXwJj+cAr1NspJXg1F8nfR8pIUVKVYPBkI45ly6gBHfw0hX8/ eQIk1CY95wMUTaAUcf9KclDf5DxWl4rTywJZoxF1lxcHIBV8Rr6wTlJ55QHjgr6JzWoP YrdA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:date:in-reply-to:message-id:mime-version :references:subject:from:to:cc; bh=/T1g0yV2QOlVxhv7pQ6ewK8IHn8UwKGZh5cup4X5Lcs=; b=hb6ZSbFc4rMMx7sNYMCZ8/51V1qyrj+cIa8YKUCsl9vscvHMxIUELLjCqw+Q9v2l/i u6/p+2FrvCsmShP0RXEQ+4MhwERMKBgHRhBqvyUF8MiEyk6yP7/vIIFa1SS5E1UtNbJb XLxE/NfZFq3F3aF7JXln57qtr1K+214C3Xg/Xtxt/XYHVdZRtiRh0Tr+nZ4tv0RTtymR 96f6ZWTorzCXCQpYCL9nO/4Lzkm9gbGK4hz9vKnUxZNsBxUsZFbdpAXEAydqpnEU1fcu 7Tkhv+TC7CYHZQvjFF4vMoUc+O2GuZNPBHSFH/0aGw3gr3NLhGBQxlZA7ayd+28ewWX1 sv/w== X-Gm-Message-State: APjAAAV+11eszzF85SDH8Lw7t+TkQmR9OvNeiAVy1JHt+bJ/4maZmxtI gsNugAVqZaTmlB37YWaU7DViDpIQnZkDd6rsELAF X-Google-Smtp-Source: APXvYqyQLw3965yFutFqxRWrzgflBTfSO56iSGJGz5uCwxW3WsOElARuHWUtUhS4sh0LgTugXKY9SFcOm6y/ZPAjsFMQ X-Received: by 2002:a24:6283:: with SMTP id d125mr13502677itc.14.1552082137628; Fri, 08 Mar 2019 13:55:37 -0800 (PST) Date: Fri, 8 Mar 2019 13:55:13 -0800 In-Reply-To: Message-Id: <87173d0ad1f1e611b6c2f539ca203be5bb9efb53.1552073690.git.jonathantanmy@google.com> Mime-Version: 1.0 References: X-Mailer: git-send-email 2.19.0.271.gfe8321ec05.dirty Subject: [PATCH v2 1/8] http: use --stdin when getting dumb HTTP pack From: Jonathan Tan To: avarab@gmail.com, git@vger.kernel.org Cc: Jonathan Tan , Junio C Hamano Sender: git-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: git@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP When Git fetches a pack using dumb HTTP, it reuses the server's name for the packfile (which incorporates a hash), which is different from the behavior of fetch-pack and receive-pack. A subsequent patch will allow downloading packs over HTTP(S) as part of a fetch. These downloads will not necessarily be from a Git repository, and thus may not have a hash as part of its name. Thus, teach http to pass --stdin to index-pack, so that we have no reliance on the server's name for the packfile. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- http.c | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/http.c b/http.c index a32ad36ddf..34f82af87c 100644 --- a/http.c +++ b/http.c @@ -2204,9 +2204,9 @@ int finish_http_pack_request(struct http_pack_request *preq) { struct packed_git **lst; struct packed_git *p = preq->target; - char *tmp_idx; - size_t len; struct child_process ip = CHILD_PROCESS_INIT; + int tmpfile_fd; + int ret = 0; close_pack_index(p); @@ -2218,35 +2218,24 @@ int finish_http_pack_request(struct http_pack_request *preq) lst = &((*lst)->next); *lst = (*lst)->next; - if (!strip_suffix(preq->tmpfile.buf, ".pack.temp", &len)) - BUG("pack tmpfile does not end in .pack.temp?"); - tmp_idx = xstrfmt("%.*s.idx.temp", (int)len, preq->tmpfile.buf); + tmpfile_fd = xopen(preq->tmpfile.buf, O_RDONLY); argv_array_push(&ip.args, "index-pack"); - argv_array_pushl(&ip.args, "-o", tmp_idx, NULL); - argv_array_push(&ip.args, preq->tmpfile.buf); + argv_array_push(&ip.args, "--stdin"); ip.git_cmd = 1; - ip.no_stdin = 1; + ip.in = tmpfile_fd; ip.no_stdout = 1; if (run_command(&ip)) { - unlink(preq->tmpfile.buf); - unlink(tmp_idx); - free(tmp_idx); - return -1; - } - - unlink(sha1_pack_index_name(p->sha1)); - - if (finalize_object_file(preq->tmpfile.buf, sha1_pack_name(p->sha1)) - || finalize_object_file(tmp_idx, sha1_pack_index_name(p->sha1))) { - free(tmp_idx); - return -1; + ret = -1; + goto cleanup; } install_packed_git(the_repository, p); - free(tmp_idx); - return 0; +cleanup: + close(tmpfile_fd); + unlink(preq->tmpfile.buf); + return ret; } struct http_pack_request *new_http_pack_request( From patchwork Fri Mar 8 21:55:14 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jonathan Tan X-Patchwork-Id: 10845467 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 1C81C14E1 for ; Fri, 8 Mar 2019 21:55:43 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 09E492CFF6 for ; Fri, 8 Mar 2019 21:55:43 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id F1D072FDA5; Fri, 8 Mar 2019 21:55:42 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-15.5 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI, USER_IN_DEF_DKIM_WL autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id A15EA2CFF6 for ; Fri, 8 Mar 2019 21:55:42 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726409AbfCHVzl (ORCPT ); Fri, 8 Mar 2019 16:55:41 -0500 Received: from mail-qt1-f201.google.com ([209.85.160.201]:53265 "EHLO mail-qt1-f201.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726267AbfCHVzl (ORCPT ); Fri, 8 Mar 2019 16:55:41 -0500 Received: by mail-qt1-f201.google.com with SMTP id g42so4022717qtb.20 for ; Fri, 08 Mar 2019 13:55:40 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20161025; h=date:in-reply-to:message-id:mime-version:references:subject:from:to :cc; bh=wNUkgnbKPaE8zyDrl2JzNKhkZOQyW5hMytqyZiyiuYc=; b=hnla7YMgzQtkNUeDWzp6o/hyY4Bl3pNlozRYXRh8YLUFyPbxGq8teTG+QhOzQpMl5z E6yGIzlRnUUrbDbrqoTx9TNeHHJ1jMWX/x2i4pV4e6OS+vFHYzjHiG7wpVOrRA6qnBRH Zif/Mz2MDz2KVUteqjZeEGJsA8pYk8akPYJtY7juGc1t8pmtaLWN2D3VLy0vRXdIB4dA Th2bOL+V0rglApkZ59YgA+3QkjRnEa25YWyTB1y2Ow1sd9YbBZ5uovLR50rGyRBCkVlT x299FvCastC/BLImlMYEgWYeGjtrE9PUKZcI9a0c79Zn8J/pQS+518fiKdjcbEsj7D33 PpRQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:date:in-reply-to:message-id:mime-version :references:subject:from:to:cc; bh=wNUkgnbKPaE8zyDrl2JzNKhkZOQyW5hMytqyZiyiuYc=; b=jQwn33I+hMKAriK9/gsApqYkRNRuGzFHBJlcyP++pbhZQyEqr58QKbZaZY0IfRFqq/ QDYrB1gWLZBdh9Q69T/YT4FngmxFUV/KPbg4OYfSwP5b5wM5a/kDvwvIRovmDfKh4KRs SQiU8kgSpyWW5U9REwNDH+hr4SWdzU7s2CmJa783+y9BZDlknxNOzcK/MdAS7n2Lrwv2 nU1OMlFow0mdYLTplH80BesOMtV8VfjWXOlvPzdrzdu/wuJQVzVZjy2hAnek3jxczBcn HlBRRpB1EC4e7sG7Q4cVLX8hZ8Lvutd3ctAitcAHcPjj3zFBaNhfQ+1irI2dzdMMIATh R+tw== X-Gm-Message-State: APjAAAXLPScyfkTpy3kH+enxeds1NqBHGLlnKH27KMg/YJcEBEic8PmE 7sJMrKx5UZQNORDyf4jAIYrvezmdVJTitrjdU257 X-Google-Smtp-Source: APXvYqzaJwp3Vv+X2pCYw+/ElIKS7EV175LL1FQ5DC9+jK/chMH2ccoP1qqHKbbeDZuN2hFN0JcAauqkuvj5TO5uRV3K X-Received: by 2002:a0c:b8a3:: with SMTP id y35mr11661953qvf.25.1552082140227; Fri, 08 Mar 2019 13:55:40 -0800 (PST) Date: Fri, 8 Mar 2019 13:55:14 -0800 In-Reply-To: Message-Id: <66d31720a045e8be11561fe9a4fd511b57557f02.1552073690.git.jonathantanmy@google.com> Mime-Version: 1.0 References: X-Mailer: git-send-email 2.19.0.271.gfe8321ec05.dirty Subject: [PATCH v2 2/8] http: improve documentation of http_pack_request From: Jonathan Tan To: avarab@gmail.com, git@vger.kernel.org Cc: Jonathan Tan , Junio C Hamano Sender: git-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: git@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP struct http_pack_request and the functions that use it will be modified in a subsequent patch. Using it is complicated (to use, call the initialization function, then set some but not all fields in the returned struct), so add some documentation to help future users. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- http.h | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/http.h b/http.h index 4eb4e808e5..ded1edcca4 100644 --- a/http.h +++ b/http.h @@ -202,14 +202,31 @@ extern int http_get_info_packs(const char *base_url, struct packed_git **packs_head); struct http_pack_request { + /* + * Initialized by new_http_pack_request(). + */ char *url; struct packed_git *target; + struct active_request_slot *slot; + + /* + * After calling new_http_pack_request(), point lst to the head of the + * pack list that target is in. finish_http_pack_request() will remove + * target from lst and call install_packed_git() on target. + */ struct packed_git **lst; + + /* + * State managed by functions in http.c. + */ FILE *packfile; struct strbuf tmpfile; - struct active_request_slot *slot; }; +/* + * target must be an element in a pack list obtained from + * http_get_info_packs(). + */ extern struct http_pack_request *new_http_pack_request( struct packed_git *target, const char *base_url); extern int finish_http_pack_request(struct http_pack_request *preq); From patchwork Fri Mar 8 21:55:15 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jonathan Tan X-Patchwork-Id: 10845469 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 194FA1669 for ; Fri, 8 Mar 2019 21:55:47 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 03C412CFF6 for ; Fri, 8 Mar 2019 21:55:47 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id EBC552FDA5; Fri, 8 Mar 2019 21:55:46 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-15.5 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI, USER_IN_DEF_DKIM_WL autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 1340E2CFF6 for ; Fri, 8 Mar 2019 21:55:46 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726410AbfCHVzp (ORCPT ); Fri, 8 Mar 2019 16:55:45 -0500 Received: from mail-qt1-f201.google.com ([209.85.160.201]:36257 "EHLO mail-qt1-f201.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726267AbfCHVzo (ORCPT ); Fri, 8 Mar 2019 16:55:44 -0500 Received: by mail-qt1-f201.google.com with SMTP id p5so20062885qtp.3 for ; Fri, 08 Mar 2019 13:55:43 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20161025; h=date:in-reply-to:message-id:mime-version:references:subject:from:to :cc; bh=7hf/jccunXFQPYR8hPqt7wb9FpCZ2kN06SvU1xLvP6E=; b=Sx+gjT9iB/C2nPOa+wxLFcJv0kqUf3g5fNypZMo5+DI1VH3acMH4KWJJRdNAgyVUut VjbElpXte5jxt9EZTRf+JTs2fnbZi/ttDQZc/yRi6lOJDfe790Pc+9XKUXs88FeVAr7f N51tHfDqgcRtqVhPaJjYlFKLN5rbtWAz8iYYXJMATQbDN8nK/hKolcF42qP/DawcBn/A hfbeDGoUa9DX3eBIyCTCbl//+Rkhimjm0rOxhglIgzYfGaNnGs/N9BwITvV02+TFyhE6 HrJH8lxuP8xTWyNNSmz57IMjQt2GO7qAhaURK+AFK0JnDCrcqDR7BXJPgpz24gPYtzMN XFmA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:date:in-reply-to:message-id:mime-version :references:subject:from:to:cc; bh=7hf/jccunXFQPYR8hPqt7wb9FpCZ2kN06SvU1xLvP6E=; b=PCdpwJ0vaMUhkjbPcSsCYl1dUtvMKlbtNK85ZWSfs9Sh0iHKUcQCaxMeji1TMJDEYu Lr6ALnewQSkJthFbnOJQt91/LlwgPmI8b0kgnZTR0+hBYwbsjEv2pSpxQqqBauUG+c4v q3kuH9pdDKF/YPIQB0nWmYAZu7/Vj9jrmhk5McSBXpPL8Cgkwir9JpbnTe+VH89tqOx/ wNviVqUgAUSiHQ2VGEAMyLZVzJxdQWjgILCr7n7MM4Vv9NsTj0mRQsB/mIrgEDDqA3bd RWGMhnvkv5Ijyr+hnkI04ajUjkedXhaFhSJCZNKVRnj8f/qFwOaISCmS76vzY+cGQFCU Acbg== X-Gm-Message-State: APjAAAURroox3pMrqMpXLYTumwqWBeOD4RotqByZqDKOQ9bpcgL1zTiC MK8miMU7c32PCyQHMme3PtKaAtxCbS0ijmiDKwLu X-Google-Smtp-Source: APXvYqwQpFXIBGpBUcNrUFCgCeoyeLCGLEcKlSs/Bcwa7QFAzKZY2sPVhVuVMpO08ZIqR8GR7ouMuHzsQRpHme/XCKg8 X-Received: by 2002:ac8:2539:: with SMTP id 54mr11287052qtm.45.1552082142921; Fri, 08 Mar 2019 13:55:42 -0800 (PST) Date: Fri, 8 Mar 2019 13:55:15 -0800 In-Reply-To: Message-Id: <02373f6afe0c3c93728ff2be21dc26de9b82e0d1.1552073690.git.jonathantanmy@google.com> Mime-Version: 1.0 References: X-Mailer: git-send-email 2.19.0.271.gfe8321ec05.dirty Subject: [PATCH v2 3/8] http-fetch: support fetching packfiles by URL From: Jonathan Tan To: avarab@gmail.com, git@vger.kernel.org Cc: Jonathan Tan , Junio C Hamano Sender: git-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: git@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Teach http-fetch the ability to download packfiles directly, given a URL, and to verify them. The http_pack_request suite of functions have been modified to support a NULL target. When target is NULL, the given URL is downloaded directly instead of being treated as the root of a repository. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- Documentation/git-http-fetch.txt | 8 +++- http-fetch.c | 64 +++++++++++++++++++++++++------- http.c | 49 +++++++++++++++++------- http.h | 19 ++++++++-- t/t5550-http-fetch-dumb.sh | 25 +++++++++++++ 5 files changed, 135 insertions(+), 30 deletions(-) diff --git a/Documentation/git-http-fetch.txt b/Documentation/git-http-fetch.txt index 666b042679..8357359a9b 100644 --- a/Documentation/git-http-fetch.txt +++ b/Documentation/git-http-fetch.txt @@ -9,7 +9,7 @@ git-http-fetch - Download from a remote Git repository via HTTP SYNOPSIS -------- [verse] -'git http-fetch' [-c] [-t] [-a] [-d] [-v] [-w filename] [--recover] [--stdin] +'git http-fetch' [-c] [-t] [-a] [-d] [-v] [-w filename] [--recover] [--stdin | --packfile | ] DESCRIPTION ----------- @@ -40,6 +40,12 @@ commit-id:: ['\t'] +--packfile:: + Instead of a commit id on the command line (which is not expected in + this case), 'git http-fetch' fetches the packfile directly at the given + URL and uses index-pack to generate corresponding .idx and .keep files. + The output of index-pack is printed to stdout. + --recover:: Verify that everything reachable from target is fetched. Used after an earlier fetch is interrupted. diff --git a/http-fetch.c b/http-fetch.c index a32ac118d9..a9764d6f96 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -5,7 +5,7 @@ #include "walker.h" static const char http_fetch_usage[] = "git http-fetch " -"[-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url"; +"[-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin | --packfile | commit-id] url"; int cmd_main(int argc, const char **argv) { @@ -19,6 +19,7 @@ int cmd_main(int argc, const char **argv) int rc = 0; int get_verbosely = 0; int get_recover = 0; + int packfile = 0; while (arg < argc && argv[arg][0] == '-') { if (argv[arg][1] == 't') { @@ -35,43 +36,80 @@ int cmd_main(int argc, const char **argv) get_recover = 1; } else if (!strcmp(argv[arg], "--stdin")) { commits_on_stdin = 1; + } else if (!strcmp(argv[arg], "--packfile")) { + packfile = 1; } arg++; } - if (argc != arg + 2 - commits_on_stdin) + if (argc != arg + 2 - (commits_on_stdin || packfile)) usage(http_fetch_usage); if (commits_on_stdin) { commits = walker_targets_stdin(&commit_id, &write_ref); + } else if (packfile) { + /* URL will be set later */ } else { commit_id = (char **) &argv[arg++]; commits = 1; } - if (argv[arg]) - str_end_url_with_slash(argv[arg], &url); + if (packfile) { + url = xstrdup(argv[arg]); + } else { + if (argv[arg]) + str_end_url_with_slash(argv[arg], &url); + } setup_git_directory(); git_config(git_default_config, NULL); http_init(NULL, url, 0); - walker = get_http_walker(url); - walker->get_verbosely = get_verbosely; - walker->get_recover = get_recover; - rc = walker_fetch(walker, commits, commit_id, write_ref, url); + if (packfile) { + struct http_pack_request *preq; + struct slot_results results; + int ret; + + preq = new_http_pack_request(NULL, url); + if (preq == NULL) + die("couldn't create http pack request"); + preq->slot->results = &results; + preq->generate_keep = 1; + + if (start_active_slot(preq->slot)) { + run_active_slot(preq->slot); + if (results.curl_result != CURLE_OK) { + die("Unable to get pack file %s\n%s", preq->url, + curl_errorstr); + } + } else { + die("Unable to start request"); + } + + if ((ret = finish_http_pack_request(preq))) + die("finish_http_pack_request gave result %d", ret); + release_http_pack_request(preq); + rc = 0; + } else { + walker = get_http_walker(url); + walker->get_verbosely = get_verbosely; + walker->get_recover = get_recover; + + rc = walker_fetch(walker, commits, commit_id, write_ref, url); - if (commits_on_stdin) - walker_targets_free(commits, commit_id, write_ref); + if (commits_on_stdin) + walker_targets_free(commits, commit_id, write_ref); - if (walker->corrupt_object_found) { - fprintf(stderr, + if (walker->corrupt_object_found) { + fprintf(stderr, "Some loose object were found to be corrupt, but they might be just\n" "a false '404 Not Found' error message sent with incorrect HTTP\n" "status code. Suggest running 'git fsck'.\n"); + } + + walker_free(walker); } - walker_free(walker); http_cleanup(); free(url); diff --git a/http.c b/http.c index 34f82af87c..b372e61520 100644 --- a/http.c +++ b/http.c @@ -2208,15 +2208,18 @@ int finish_http_pack_request(struct http_pack_request *preq) int tmpfile_fd; int ret = 0; - close_pack_index(p); + if (p) + close_pack_index(p); fclose(preq->packfile); preq->packfile = NULL; - lst = preq->lst; - while (*lst != p) - lst = &((*lst)->next); - *lst = (*lst)->next; + if (p) { + lst = preq->lst; + while (*lst != p) + lst = &((*lst)->next); + *lst = (*lst)->next; + } tmpfile_fd = xopen(preq->tmpfile.buf, O_RDONLY); @@ -2224,14 +2227,21 @@ int finish_http_pack_request(struct http_pack_request *preq) argv_array_push(&ip.args, "--stdin"); ip.git_cmd = 1; ip.in = tmpfile_fd; - ip.no_stdout = 1; + if (preq->generate_keep) { + argv_array_pushf(&ip.args, "--keep=git %"PRIuMAX, + (uintmax_t)getpid()); + ip.out = 0; + } else { + ip.no_stdout = 1; + } if (run_command(&ip)) { ret = -1; goto cleanup; } - install_packed_git(the_repository, p); + if (p) + install_packed_git(the_repository, p); cleanup: close(tmpfile_fd); unlink(preq->tmpfile.buf); @@ -2249,12 +2259,24 @@ struct http_pack_request *new_http_pack_request( strbuf_init(&preq->tmpfile, 0); preq->target = target; - end_url_with_slash(&buf, base_url); - strbuf_addf(&buf, "objects/pack/pack-%s.pack", - sha1_to_hex(target->sha1)); - preq->url = strbuf_detach(&buf, NULL); + if (target) { + end_url_with_slash(&buf, base_url); + strbuf_addf(&buf, "objects/pack/pack-%s.pack", + sha1_to_hex(target->sha1)); + preq->url = strbuf_detach(&buf, NULL); + } else { + preq->url = xstrdup(base_url); + } + + if (target) { + strbuf_addf(&preq->tmpfile, "%s.temp", + sha1_pack_name(target->sha1)); + } else { + strbuf_addf(&preq->tmpfile, "%s/pack/pack-", get_object_directory()); + strbuf_addstr_urlencode(&preq->tmpfile, base_url, 1); + strbuf_addstr(&preq->tmpfile, ".temp"); + } - strbuf_addf(&preq->tmpfile, "%s.temp", sha1_pack_name(target->sha1)); preq->packfile = fopen(preq->tmpfile.buf, "a"); if (!preq->packfile) { error("Unable to open local file %s for pack", @@ -2278,7 +2300,8 @@ struct http_pack_request *new_http_pack_request( if (http_is_verbose) fprintf(stderr, "Resuming fetch of pack %s at byte %"PRIuMAX"\n", - sha1_to_hex(target->sha1), (uintmax_t)prev_posn); + target ? sha1_to_hex(target->sha1) : base_url, + (uintmax_t)prev_posn); http_opt_request_remainder(preq->slot->curl, prev_posn); } diff --git a/http.h b/http.h index ded1edcca4..8f63c76e09 100644 --- a/http.h +++ b/http.h @@ -210,12 +210,21 @@ struct http_pack_request { struct active_request_slot *slot; /* - * After calling new_http_pack_request(), point lst to the head of the + * After calling new_http_pack_request(), if fetching a pack that + * http_get_info_packs() told us about, point lst to the head of the * pack list that target is in. finish_http_pack_request() will remove * target from lst and call install_packed_git() on target. */ struct packed_git **lst; + /* + * If this is true, finish_http_pack_request() will pass "--keep" to + * index-pack, resulting in the creation of a keep file, and will not + * suppress its stdout (that is, the "keep\t\n" line will be + * printed to stdout). + */ + unsigned generate_keep : 1; + /* * State managed by functions in http.c. */ @@ -224,8 +233,12 @@ struct http_pack_request { }; /* - * target must be an element in a pack list obtained from - * http_get_info_packs(). + * If fetching a pack that http_get_info_packs() told us about, set target to + * an element in a pack list obtained from http_get_info_packs(). The actual + * URL fetched will be base_url followed by a suffix with the hash of the pack. + * + * Otherwise, set target to NULL. The actual URL fetched will be base_url + * itself. */ extern struct http_pack_request *new_http_pack_request( struct packed_git *target, const char *base_url); diff --git a/t/t5550-http-fetch-dumb.sh b/t/t5550-http-fetch-dumb.sh index 6d7d88ccc9..84235fd6d4 100755 --- a/t/t5550-http-fetch-dumb.sh +++ b/t/t5550-http-fetch-dumb.sh @@ -199,6 +199,23 @@ test_expect_success 'fetch packed objects' ' git clone $HTTPD_URL/dumb/repo_pack.git ' +test_expect_success 'http-fetch --packfile' ' + git init packfileclient && + p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git && ls objects/pack/pack-*.pack) && + git -C packfileclient http-fetch --packfile "$HTTPD_URL"/dumb/repo_pack.git/$p >out && + + # Ensure that the expected files are generated + grep "^keep.[0-9a-f]\{16,\}$" out && + cut -c6- out >packhash && + test -e "packfileclient/.git/objects/pack/pack-$(cat packhash).pack" && + test -e "packfileclient/.git/objects/pack/pack-$(cat packhash).idx" && + test -e "packfileclient/.git/objects/pack/pack-$(cat packhash).keep" && + + # Ensure that it has the HEAD of repo_pack, at least + HASH=$(git -C "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git rev-parse HEAD) && + git -C packfileclient cat-file -e "$HASH" +' + test_expect_success 'fetch notices corrupt pack' ' cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git && (cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git && @@ -214,6 +231,14 @@ test_expect_success 'fetch notices corrupt pack' ' ) ' +test_expect_success 'http-fetch --packfile with corrupt pack' ' + rm -rf packfileclient && + git init packfileclient && + p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git && ls objects/pack/pack-*.pack) && + test_must_fail git -C packfileclient http-fetch --packfile \ + "$HTTPD_URL"/dumb/repo_bad1.git/$p +' + test_expect_success 'fetch notices corrupt idx' ' cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad2.git && (cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad2.git && From patchwork Fri Mar 8 21:55:16 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jonathan Tan X-Patchwork-Id: 10845471 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id AC5201669 for ; Fri, 8 Mar 2019 21:55:48 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 99E722CFF6 for ; Fri, 8 Mar 2019 21:55:48 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 8DF9C2FDA5; Fri, 8 Mar 2019 21:55:48 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-15.5 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI, USER_IN_DEF_DKIM_WL autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 263302CFF6 for ; Fri, 8 Mar 2019 21:55:48 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726418AbfCHVzr (ORCPT ); Fri, 8 Mar 2019 16:55:47 -0500 Received: from mail-ot1-f74.google.com ([209.85.210.74]:43278 "EHLO mail-ot1-f74.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726267AbfCHVzq (ORCPT ); Fri, 8 Mar 2019 16:55:46 -0500 Received: by mail-ot1-f74.google.com with SMTP id r13so9567922otn.10 for ; Fri, 08 Mar 2019 13:55:45 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20161025; h=date:in-reply-to:message-id:mime-version:references:subject:from:to :cc; bh=UWi58aZRxQ8bSE3rNvDWEMF7TWiZqIKsWnzqCy604Sc=; b=GckAaajtJIkvCfHx5+rBHDHIiMxM+w1nrq5/JizEXYlOpYea95G13yLgguLm+2kDU1 oBOwFNWR/x9GO92QKF64TeNFZDleONorzAj2CkikyYPALrkVjeQ5jRy/KpSa5UlvAREN 9s9GqTjxVmZCZdUYepwJwdiEdmjc1xe+Sd8XZwAkDDR+WwBAWn8JeWnfRJjvchv4l3dM xrEe8eHSwG7PVkhVy2wdju5ufViZEcH1iKMJDdDDRe+CPjBzvbeQQZyiGRo1D5DrMoi8 gPiwj85MBN5fWsaFrMTtErKJehc9mvwcNunYKULVir6PydeFDfjJ1JYrurYLGp71XSYH NrvQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:date:in-reply-to:message-id:mime-version :references:subject:from:to:cc; bh=UWi58aZRxQ8bSE3rNvDWEMF7TWiZqIKsWnzqCy604Sc=; b=tQ6KieCvW0SWzVigs/QXdT9So/WSzJoBNj7g4lKu3p3nI2uPervV3YEav7U8pasfNM FlYt5JYXg0rdOIwblkYdxx008ipU+oRXI9uxvUUmiNmuQvUxzilhWAbUhh7FriIshxPC 8E9AXQTjZt/NJ/aOJsMNwG+rTlpvFM3FzgEzSJZYK2ifAGBPREcv4Q37js0/dwOtEHx1 ddQaQutNo5+jx/H4PGdpSW2VcnAcTHhGu9jMIwLQp2poTAxmmO3rbmt/05rH+FrXY2A2 4qpuuMmD7CElcG41wRDTGcsYrYlGCYFX9eyQpEbsLEJkYOhVQ7eYKX6MMcQdqGXp6RH3 Nf0g== X-Gm-Message-State: APjAAAWbBrCcFnejq8k456UwuQOoCA6PBqNMaqBc5O1i5Fcgz0bY6t/y Wf3FLCC02I8hOjIxivKv7YfSfPr0rtQhm0oCGwrn X-Google-Smtp-Source: APXvYqzga5saOtyCl2/3HAk7BBsNAB5Qsbl21iw0w1j+ShdWNjzKab1tGSGhkJGVG9LLRyHriW2cg1gXWKysnSJKP2F4 X-Received: by 2002:a9d:635a:: with SMTP id y26mr11509933otk.53.1552082145410; Fri, 08 Mar 2019 13:55:45 -0800 (PST) Date: Fri, 8 Mar 2019 13:55:16 -0800 In-Reply-To: Message-Id: <2e0f2daba00f02500921dc3e3f15a301ee831858.1552073690.git.jonathantanmy@google.com> Mime-Version: 1.0 References: X-Mailer: git-send-email 2.19.0.271.gfe8321ec05.dirty Subject: [PATCH v2 4/8] Documentation: order protocol v2 sections From: Jonathan Tan To: avarab@gmail.com, git@vger.kernel.org Cc: Jonathan Tan , Junio C Hamano Sender: git-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: git@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP The current C Git implementation expects Git servers to follow a specific order of sections when transmitting protocol v2 responses, but this is not explicit in the documentation. Make the order explicit. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- Documentation/technical/protocol-v2.txt | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt index ead85ce35c..36239ec7e9 100644 --- a/Documentation/technical/protocol-v2.txt +++ b/Documentation/technical/protocol-v2.txt @@ -325,11 +325,11 @@ included in the client's request: The response of `fetch` is broken into a number of sections separated by delimiter packets (0001), with each section beginning with its section -header. +header. Most sections are sent only when the packfile is sent. - output = *section - section = (acknowledgments | shallow-info | wanted-refs | packfile) - (flush-pkt | delim-pkt) + output = acknowledgements flush-pkt | + [acknowledgments delim-pkt] [shallow-info delim-pkt] + [wanted-refs delim-pkt] packfile flush-pkt acknowledgments = PKT-LINE("acknowledgments" LF) (nak | *ack) @@ -351,9 +351,10 @@ header. *PKT-LINE(%x01-03 *%x00-ff) acknowledgments section - * If the client determines that it is finished with negotiations - by sending a "done" line, the acknowledgments sections MUST be - omitted from the server's response. + * If the client determines that it is finished with negotiations by + sending a "done" line (thus requiring the server to send a packfile), + the acknowledgments sections MUST be omitted from the server's + response. * Always begins with the section header "acknowledgments" @@ -404,9 +405,6 @@ header. which the client has not indicated was shallow as a part of its request. - * This section is only included if a packfile section is also - included in the response. - wanted-refs section * This section is only included if the client has requested a ref using a 'want-ref' line and if a packfile section is also From patchwork Fri Mar 8 21:55:17 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jonathan Tan X-Patchwork-Id: 10845473 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 988B414E1 for ; Fri, 8 Mar 2019 21:55:51 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 851422CFF6 for ; Fri, 8 Mar 2019 21:55:51 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 794252FDA5; Fri, 8 Mar 2019 21:55:51 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-15.5 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI, USER_IN_DEF_DKIM_WL autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id C59D52CFF6 for ; Fri, 8 Mar 2019 21:55:50 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726427AbfCHVzt (ORCPT ); Fri, 8 Mar 2019 16:55:49 -0500 Received: from mail-pg1-f201.google.com ([209.85.215.201]:53630 "EHLO mail-pg1-f201.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726267AbfCHVzt (ORCPT ); Fri, 8 Mar 2019 16:55:49 -0500 Received: by mail-pg1-f201.google.com with SMTP id k198so21696366pgc.20 for ; Fri, 08 Mar 2019 13:55:48 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20161025; h=date:in-reply-to:message-id:mime-version:references:subject:from:to :cc; bh=l60cNizw3k+R6216CyLcKW576KmjfOXpd+EXhbatOVg=; b=DFfgYn2v2cgz8mfw31cwxrMJ0ZNtlBr7+lp7cPTw7vxbCqRq32LSZ02ZVMqogR2XIO LyKrL7Ivq72nOuUddD5ncyH3qUk46wgevmKlolb4RlNQeT8VY3aSBUubyXQSgIkITaN/ gZYoEbiJ+0ApgjkxpSqdCedIRkckeLYXJ3bGlNPucXVcba30gXG8nNPLN0JdbmVwSg9O fI9faoCcjM4aKx+78E6bWN3TIwcQFhMb5C88QfZ0pKmntHS6aY0ayGyw32ARln5/fmiP nNZ2/pn41R4D7NaUgwFA/8zZo0WE4AjcujtMTa9K/es22yAWV82gT+KT7MfBDC1BgDzw bvzw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:date:in-reply-to:message-id:mime-version :references:subject:from:to:cc; bh=l60cNizw3k+R6216CyLcKW576KmjfOXpd+EXhbatOVg=; b=DUwPtztvdN0BxP+Hxvbt7YYqwQxBbX8mZngzU2pjX2dkldGluhInjv0Kvn6opNJjrE SogXdLMTPfwGCoPVsFWh8VQxt+69eYwWM6uiReuUPCC/0nkYhOOkYKsKImvy/0w69w0C V97BJ5SWqCass96IkA+Gpm2sC/i4tZBI6nwplWpFtmOW5VUpuZ1OJpw9xddFA6hDSJAS mb6mnsfVbPA3cf8yO+b8pPeKF7Fz7NIcq2yJK7WdLdOSmN7dCNtscCF3upwhrD10WXow myTo4iVnQ/Xs9als8z3FuRw+PX3pYarHnBVXtqfQ3mgBCVqxMbDky5wbZz0PTeTOqtE8 oAmA== X-Gm-Message-State: APjAAAWHKqonKp1i+fcyVy0st/FtUFS4TU4MuyVWZ2yIS7ZEajxgOP4i ARNyGd3L2jMeNgRxOH/YIu/wgDRF0dDstgrZRk/F X-Google-Smtp-Source: APXvYqw2w7ERBUNcAbL//4uLGQ2RoQY3Qf7wHHtOfXuYdGPQ23twJnqxujU90vQ5YmlH68gRnMnYr4F4+AlIiipuB38H X-Received: by 2002:a17:902:8643:: with SMTP id y3mr6328474plt.125.1552082148512; Fri, 08 Mar 2019 13:55:48 -0800 (PST) Date: Fri, 8 Mar 2019 13:55:17 -0800 In-Reply-To: Message-Id: <5ce56844d3fb740e29d2f3d4be2ade0b2ad5f7fd.1552073690.git.jonathantanmy@google.com> Mime-Version: 1.0 References: X-Mailer: git-send-email 2.19.0.271.gfe8321ec05.dirty Subject: [PATCH v2 5/8] Documentation: add Packfile URIs design doc From: Jonathan Tan To: avarab@gmail.com, git@vger.kernel.org Cc: Jonathan Tan , Junio C Hamano Sender: git-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: git@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- Documentation/technical/packfile-uri.txt | 78 ++++++++++++++++++++++++ Documentation/technical/protocol-v2.txt | 28 ++++++++- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 Documentation/technical/packfile-uri.txt diff --git a/Documentation/technical/packfile-uri.txt b/Documentation/technical/packfile-uri.txt new file mode 100644 index 0000000000..6a5a6440d5 --- /dev/null +++ b/Documentation/technical/packfile-uri.txt @@ -0,0 +1,78 @@ +Packfile URIs +============= + +This feature allows servers to serve part of their packfile response as URIs. +This allows server designs that improve scalability in bandwidth and CPU usage +(for example, by serving some data through a CDN), and (in the future) provides +some measure of resumability to clients. + +This feature is available only in protocol version 2. + +Protocol +-------- + +The server advertises `packfile-uris`. + +If the client then communicates which protocols (HTTPS, etc.) it supports with +a `packfile-uris` argument, the server MAY send a `packfile-uris` section +directly before the `packfile` section (right after `wanted-refs` if it is +sent) containing URIs of any of the given protocols. The URIs point to +packfiles that use only features that the client has declared that it supports +(e.g. ofs-delta and thin-pack). See protocol-v2.txt for the documentation of +this section. + +Clients then should understand that the returned packfile could be incomplete, +and that it needs to download all the given URIs before the fetch or clone is +complete. + +Server design +------------- + +The server can be trivially made compatible with the proposed protocol by +having it advertise `packfile-uris`, tolerating the client sending +`packfile-uris`, and never sending any `packfile-uris` section. But we should +include some sort of non-trivial implementation in the Minimum Viable Product, +at least so that we can test the client. + +This is the implementation: a feature, marked experimental, that allows the +server to be configured by one or more `uploadpack.blobPackfileUri= +` entries. Whenever the list of objects to be sent is assembled, a blob +with the given sha1 can be replaced by the given URI. This allows, for example, +servers to delegate serving of large blobs to CDNs. + +Client design +------------- + +While fetching, the client needs to remember the list of URIs and cannot +declare that the fetch is complete until all URIs have been downloaded as +packfiles. + +The division of work (initial fetch + additional URIs) introduces convenient +points for resumption of an interrupted clone - such resumption can be done +after the Minimum Viable Product (see "Future work"). + +The client can inhibit this feature (i.e. refrain from sending the +`packfile-uris` parameter) by passing --no-packfile-uris to `git fetch`. + +Future work +----------- + +The protocol design allows some evolution of the server and client without any +need for protocol changes, so only a small-scoped design is included here to +form the MVP. For example, the following can be done: + + * On the server, a long-running process that takes in entire requests and + outputs a list of URIs and the corresponding inclusion and exclusion sets of + objects. This allows, e.g., signed URIs to be used and packfiles for common + requests to be cached. + * On the client, resumption of clone. If a clone is interrupted, information + could be recorded in the repository's config and a "clone-resume" command + can resume the clone in progress. (Resumption of subsequent fetches is more + difficult because that must deal with the user wanting to use the repository + even after the fetch was interrupted.) + +There are some possible features that will require a change in protocol: + + * Additional HTTP headers (e.g. authentication) + * Byte range support + * Different file formats referenced by URIs (e.g. raw object) diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt index 36239ec7e9..7b63c26ecd 100644 --- a/Documentation/technical/protocol-v2.txt +++ b/Documentation/technical/protocol-v2.txt @@ -323,13 +323,26 @@ included in the client's request: indicating its sideband (1, 2, or 3), and the server may send "0005\2" (a PKT-LINE of sideband 2 with no payload) as a keepalive packet. +If the 'packfile-uris' feature is advertised, the following argument +can be included in the client's request as well as the potential +addition of the 'packfile-uris' section in the server's response as +explained below. + + packfile-uris + Indicates to the server that the client is willing to receive + URIs of any of the given protocols in place of objects in the + sent packfile. Before performing the connectivity check, the + client should download from all given URIs. Currently, the + protocols supported are "http" and "https". + The response of `fetch` is broken into a number of sections separated by delimiter packets (0001), with each section beginning with its section header. Most sections are sent only when the packfile is sent. output = acknowledgements flush-pkt | [acknowledgments delim-pkt] [shallow-info delim-pkt] - [wanted-refs delim-pkt] packfile flush-pkt + [wanted-refs delim-pkt] [packfile-uris delim-pkt] + packfile flush-pkt acknowledgments = PKT-LINE("acknowledgments" LF) (nak | *ack) @@ -347,6 +360,9 @@ header. Most sections are sent only when the packfile is sent. *PKT-LINE(wanted-ref LF) wanted-ref = obj-id SP refname + packfile-uris = PKT-LINE("packfile-uris" LF) *packfile-uri + packfile-uri = PKT-LINE(40*(HEXDIGIT) SP *%x20-ff LF) + packfile = PKT-LINE("packfile" LF) *PKT-LINE(%x01-03 *%x00-ff) @@ -418,6 +434,16 @@ header. Most sections are sent only when the packfile is sent. * The server MUST NOT send any refs which were not requested using 'want-ref' lines. + packfile-uris section + * This section is only included if the client sent + 'packfile-uris' and the server has at least one such URI to + send. + + * Always begins with the section header "packfile-uris". + + * For each URI the server sends, it sends a hash of the pack's + contents (as output by git index-pack) followed by the URI. + packfile section * This section is only included if the client has sent 'want' lines in its request and either requested that no more From patchwork Fri Mar 8 21:55:18 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jonathan Tan X-Patchwork-Id: 10845475 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id BD2CF1669 for ; Fri, 8 Mar 2019 21:55:53 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id AB27E2CFF6 for ; Fri, 8 Mar 2019 21:55:53 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 9F85A2FDA5; Fri, 8 Mar 2019 21:55:53 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-15.5 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI, USER_IN_DEF_DKIM_WL autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 2883B2CFF6 for ; Fri, 8 Mar 2019 21:55:53 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726439AbfCHVzw (ORCPT ); Fri, 8 Mar 2019 16:55:52 -0500 Received: from mail-pg1-f202.google.com ([209.85.215.202]:53631 "EHLO mail-pg1-f202.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726267AbfCHVzv (ORCPT ); Fri, 8 Mar 2019 16:55:51 -0500 Received: by mail-pg1-f202.google.com with SMTP id k198so21696449pgc.20 for ; Fri, 08 Mar 2019 13:55:51 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20161025; h=date:in-reply-to:message-id:mime-version:references:subject:from:to :cc; bh=BMaM3xlxYxQ/4Gyyy/9KVL/EDXmthTjCwBkI0ZxCra0=; b=fWJJ24vgP/76z2GPTXT4ZUeTjkX2+2AhEXRJ4ca5bihimExaPlDWQrkgLJ+DcSJntl UIaHK/w0m1680N4C98MXqZi5MUbTZoIpeQanWNYy0DARqUBz8chFJ/vR5WgsCvHqIZMO g7kEINtl/c/lLbcZi+fnWjve4h5WtnjKrW+Axugwif9rhdwNy+A2H2dkCJjrTCqLl3kn CIXoojxNqZBCGcqE2ltXo00NheO/XZ0fqJAdeqE1wNzHdlc1qyKtRDmH6XBpCuqKsh1G G5VExrtkSHj60buCocaBlCdgiXraQ6NFlH+P7K08tb6KhbgBsiYFLh7ceVQ5tsmYzSRx hvBw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:date:in-reply-to:message-id:mime-version :references:subject:from:to:cc; bh=BMaM3xlxYxQ/4Gyyy/9KVL/EDXmthTjCwBkI0ZxCra0=; b=s1eqI3KGzes439xVcbXcH99Ewv7QcSnkWTnDWiXrjnC6Yj4ClEli6AxCBPSbK72ghO qHrOi85MVUMVQKqqGGsjEyPBtynMqPjt3y4QjtMyq7vnMlp2vMsoMJ/Dd3Ya++N288rH ahkbs2vEXZSBuI1jUNdAszAHY4h1KIrsuH3vLDEW7fHkP+DfhiZBJetbDbXqppZWEUmE H7DoeTHFcdMIcvfIsszwf2jksBVFB+avanAeAdZnwA625xugrIMwCTwd6ro38J9Dx609 0AIVDBX3tLrB/W9KE+TBEIeuVgedwcwg2CDNHQwrKEE6cc7LY0agVVC3UswarKzxYeIJ 9TGw== X-Gm-Message-State: APjAAAWiS6TwCTjy47HDih1Z/9AHaQepvSh36Ct4XNJgY4sd1kqOkW65 5zMfREsh08Dd0dIZwukZalibbxIb5oHskjYjJKJF X-Google-Smtp-Source: APXvYqzTIjCcv/zau/tu6bVRWiDm0Hxse4WW/LTD1mUyvnRBcIjpWDWcO3Rbpjywohi6Tktye6vRjKDKXmlBEXANjdZQ X-Received: by 2002:a63:f303:: with SMTP id l3mr7993506pgh.132.1552082150780; Fri, 08 Mar 2019 13:55:50 -0800 (PST) Date: Fri, 8 Mar 2019 13:55:18 -0800 In-Reply-To: Message-Id: Mime-Version: 1.0 References: X-Mailer: git-send-email 2.19.0.271.gfe8321ec05.dirty Subject: [PATCH v2 6/8] upload-pack: refactor reading of pack-objects out From: Jonathan Tan To: avarab@gmail.com, git@vger.kernel.org Cc: Jonathan Tan , Junio C Hamano Sender: git-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: git@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Subsequent patches will change how the output of pack-objects is processed, so extract that processing into its own function. Currently, at most 1 character can be buffered (in the "buffered" local variable). One of those patches will require a larger buffer, so replace that "buffered" local variable with a buffer array. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- upload-pack.c | 81 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/upload-pack.c b/upload-pack.c index d098ef5982..987d2e139b 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -102,14 +102,52 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data) return 0; } +struct output_state { + char buffer[8193]; + int used; +}; + +static int relay_pack_data(int pack_objects_out, struct output_state *os) +{ + /* + * We keep the last byte to ourselves + * in case we detect broken rev-list, so that we + * can leave the stream corrupted. This is + * unfortunate -- unpack-objects would happily + * accept a valid packdata with trailing garbage, + * so appending garbage after we pass all the + * pack data is not good enough to signal + * breakage to downstream. + */ + ssize_t readsz; + + readsz = xread(pack_objects_out, os->buffer + os->used, + sizeof(os->buffer) - os->used); + if (readsz < 0) { + return readsz; + } + os->used += readsz; + + if (os->used > 1) { + send_client_data(1, os->buffer, os->used - 1); + os->buffer[0] = os->buffer[os->used - 1]; + os->used = 1; + } else { + send_client_data(1, os->buffer, os->used); + os->used = 0; + } + + return readsz; +} + static void create_pack_file(const struct object_array *have_obj, const struct object_array *want_obj) { struct child_process pack_objects = CHILD_PROCESS_INIT; - char data[8193], progress[128]; + struct output_state output_state = {0}; + char progress[128]; char abort_msg[] = "aborting due to possible repository " "corruption on the remote side."; - int buffered = -1; ssize_t sz; int i; FILE *pipe_fd; @@ -239,39 +277,15 @@ static void create_pack_file(const struct object_array *have_obj, continue; } if (0 <= pu && (pfd[pu].revents & (POLLIN|POLLHUP))) { - /* Data ready; we keep the last byte to ourselves - * in case we detect broken rev-list, so that we - * can leave the stream corrupted. This is - * unfortunate -- unpack-objects would happily - * accept a valid packdata with trailing garbage, - * so appending garbage after we pass all the - * pack data is not good enough to signal - * breakage to downstream. - */ - char *cp = data; - ssize_t outsz = 0; - if (0 <= buffered) { - *cp++ = buffered; - outsz++; - } - sz = xread(pack_objects.out, cp, - sizeof(data) - outsz); - if (0 < sz) - ; - else if (sz == 0) { + int result = relay_pack_data(pack_objects.out, + &output_state); + + if (result == 0) { close(pack_objects.out); pack_objects.out = -1; - } - else + } else if (result < 0) { goto fail; - sz += outsz; - if (1 < sz) { - buffered = data[sz-1] & 0xFF; - sz--; } - else - buffered = -1; - send_client_data(1, data, sz); } /* @@ -296,9 +310,8 @@ static void create_pack_file(const struct object_array *have_obj, } /* flush the data */ - if (0 <= buffered) { - data[0] = buffered; - send_client_data(1, data, 1); + if (output_state.used > 0) { + send_client_data(1, output_state.buffer, output_state.used); fprintf(stderr, "flushed.\n"); } if (use_sideband) From patchwork Fri Mar 8 21:55:19 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jonathan Tan X-Patchwork-Id: 10845477 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id B5FB51669 for ; Fri, 8 Mar 2019 21:55:57 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id A1DCD2CFF6 for ; Fri, 8 Mar 2019 21:55:57 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 95C4B2FDA5; Fri, 8 Mar 2019 21:55:57 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-15.5 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI, USER_IN_DEF_DKIM_WL autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 6C30E2CFF6 for ; Fri, 8 Mar 2019 21:55:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726450AbfCHVzz (ORCPT ); Fri, 8 Mar 2019 16:55:55 -0500 Received: from mail-io1-f73.google.com ([209.85.166.73]:41680 "EHLO mail-io1-f73.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726431AbfCHVzy (ORCPT ); Fri, 8 Mar 2019 16:55:54 -0500 Received: by mail-io1-f73.google.com with SMTP id p17so16359980ios.8 for ; Fri, 08 Mar 2019 13:55:54 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20161025; h=date:in-reply-to:message-id:mime-version:references:subject:from:to :cc; bh=lDFD+TBuX2saSYWMi+3YBajmy8MmuHY9+Sgg+cYhDHY=; b=WLcmxEXIYlRbGdcwwXOSIBFN2GFD0Lx/gIBRPomdyhtXe3fxDSm43+bEpv9/4yQvWe o+0Gq5G+aoesfsEk5S5vW+cbs+h0Xw0a3A/h9dHQolQS4sMiEhhfPqiAa0tKZfEenr3D PYszG4uP6uOltLtYSWJWdtnRaAVF7gAZhlrPlGrxqkyit00Q5z+iVjcyXaU2+AsX+zr2 K2hUx5g7wI4/3oohKN83fChqpxemm4M0tWA/XZNlC7+46HtWWBh4zl5Cd/I9sspuhB6f VPQ+RfE8X2BO8STxBiNX8ufvuVT5syahfm06xWOaA/PbVBDvCzlqd+IerU3O+UmPn81/ 0TVA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:date:in-reply-to:message-id:mime-version :references:subject:from:to:cc; bh=lDFD+TBuX2saSYWMi+3YBajmy8MmuHY9+Sgg+cYhDHY=; b=DiRy+II40gtLoAewkibW/pmlzj8S34P7noLSpW/nFwWi/hYMT2q2QBGXGqlKdMCUHv 7qm2GwE0Kxv+qjHLmaKdt40IELt31gx2nR2wxVzB+fxis0eWzR4WtKhfMsfpw9csbuWE N92QLrRTginSjLPMFLfL20y8L+dkrsyhmY7kx2t+0XJ/53gQCWJyiP9sfOaBZR+ei+7B qXixQ/V7YraWwOZfFETrmiUJ2KfV6TE5peMP3kp05mfDUmdbJfl8fm3SRYuzRtryfHcg nFwFOqBGYAePxqbSalS7iyrbj/fZjwBcEU+0Xaex5Q7nuq6TJga/6gNNqpU/libbjrD9 o7Ww== X-Gm-Message-State: APjAAAUfH9OOUMFrrpc2MXiGIHyhfI4cto1PS/jO6c2Lf5QhEtZ4Xz2D VKFQd+7hRWOS42eoDq2IVY8vDZKOYCNbaWRKZvZW X-Google-Smtp-Source: APXvYqwdBZd5nu0USjXGvzD2JkWhlcfnhRxhiiNhcnHVKigRfX2x2+2EhSssCrGecwqYCrH2WybHoF/MEITXFxFTy3M2 X-Received: by 2002:a24:78cb:: with SMTP id p194mr13591840itc.7.1552082153566; Fri, 08 Mar 2019 13:55:53 -0800 (PST) Date: Fri, 8 Mar 2019 13:55:19 -0800 In-Reply-To: Message-Id: Mime-Version: 1.0 References: X-Mailer: git-send-email 2.19.0.271.gfe8321ec05.dirty Subject: [PATCH v2 7/8] fetch-pack: support more than one pack lockfile From: Jonathan Tan To: avarab@gmail.com, git@vger.kernel.org Cc: Jonathan Tan Sender: git-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: git@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Whenever a fetch results in a packfile being downloaded, a .keep file is generated, so that the packfile can be preserved (from, say, a running "git repack") until refs are written referring to the contents of the packfile. In a subsequent patch, a successful fetch using protocol v2 may result in more than one .keep file being generated. Therefore, teach fetch_pack() and the transport mechanism to support multiple .keep files. Implementation notes: - builtin/fetch-pack.c normally does not generate .keep files, and thus is unaffected by this or future changes. However, it has an undocumented "--lock-pack" feature, used by remote-curl.c when implementing the "fetch" remote helper command. In keeping with the remote helper protocol, only one "lock" line will ever be written; the rest will result in warnings to stderr. However, in practice, warnings will never be written because the remote-curl.c "fetch" is only used for protocol v0/v1 (which will not generate multiple .keep files). (Protocol v2 uses the "stateless-connect" command, not the "fetch" command.) - connected.c has an optimization in that connectivity checks on a ref need not be done if the target object is in a pack known to be self-contained and connected. If there are multiple packfiles, this optimization can no longer be done. Signed-off-by: Jonathan Tan --- builtin/fetch-pack.c | 17 +++++++++++------ connected.c | 8 +++++--- fetch-pack.c | 23 ++++++++++++----------- fetch-pack.h | 2 +- transport-helper.c | 5 +++-- transport.c | 14 ++++++++------ transport.h | 6 +++--- 7 files changed, 43 insertions(+), 32 deletions(-) diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 153a2bd282..709fc4c44b 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -48,8 +48,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) struct ref **sought = NULL; int nr_sought = 0, alloc_sought = 0; int fd[2]; - char *pack_lockfile = NULL; - char **pack_lockfile_ptr = NULL; + struct string_list pack_lockfiles = STRING_LIST_INIT_DUP; + struct string_list *pack_lockfiles_ptr = NULL; struct child_process *conn; struct fetch_pack_args args; struct oid_array shallow = OID_ARRAY_INIT; @@ -134,7 +134,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) } if (!strcmp("--lock-pack", arg)) { args.lock_pack = 1; - pack_lockfile_ptr = &pack_lockfile; + pack_lockfiles_ptr = &pack_lockfiles; continue; } if (!strcmp("--check-self-contained-and-connected", arg)) { @@ -235,10 +235,15 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) } ref = fetch_pack(&args, fd, conn, ref, dest, sought, nr_sought, - &shallow, pack_lockfile_ptr, version); - if (pack_lockfile) { - printf("lock %s\n", pack_lockfile); + &shallow, pack_lockfiles_ptr, version); + if (pack_lockfiles.nr) { + int i; + + printf("lock %s\n", pack_lockfiles.items[0].string); fflush(stdout); + for (i = 1; i < pack_lockfiles.nr; i++) + warning(_("Lockfile created but not reported: %s"), + pack_lockfiles.items[i].string); } if (args.check_self_contained_and_connected && args.self_contained_and_connected) { diff --git a/connected.c b/connected.c index 1bba888eff..a866fcf9e8 100644 --- a/connected.c +++ b/connected.c @@ -40,10 +40,12 @@ int check_connected(oid_iterate_fn fn, void *cb_data, if (transport && transport->smart_options && transport->smart_options->self_contained_and_connected && - transport->pack_lockfile && - strip_suffix(transport->pack_lockfile, ".keep", &base_len)) { + transport->pack_lockfiles.nr == 1 && + strip_suffix(transport->pack_lockfiles.items[0].string, + ".keep", &base_len)) { struct strbuf idx_file = STRBUF_INIT; - strbuf_add(&idx_file, transport->pack_lockfile, base_len); + strbuf_add(&idx_file, transport->pack_lockfiles.items[0].string, + base_len); strbuf_addstr(&idx_file, ".idx"); new_pack = add_packed_git(idx_file.buf, idx_file.len, 1); strbuf_release(&idx_file); diff --git a/fetch-pack.c b/fetch-pack.c index 812be15d7e..cf89af21d9 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -755,7 +755,7 @@ static int sideband_demux(int in, int out, void *data) } static int get_pack(struct fetch_pack_args *args, - int xd[2], char **pack_lockfile) + int xd[2], struct string_list *pack_lockfiles) { struct async demux; int do_keep = args->keep_pack; @@ -798,7 +798,7 @@ static int get_pack(struct fetch_pack_args *args, } if (do_keep || args->from_promisor) { - if (pack_lockfile) + if (pack_lockfiles) cmd.out = -1; cmd_name = "index-pack"; argv_array_push(&cmd.args, cmd_name); @@ -853,8 +853,9 @@ static int get_pack(struct fetch_pack_args *args, cmd.git_cmd = 1; if (start_command(&cmd)) die(_("fetch-pack: unable to fork off %s"), cmd_name); - if (do_keep && pack_lockfile) { - *pack_lockfile = index_pack_lockfile(cmd.out); + if (do_keep && pack_lockfiles) { + string_list_append_nodup(pack_lockfiles, + index_pack_lockfile(cmd.out)); close(cmd.out); } @@ -886,7 +887,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, const struct ref *orig_ref, struct ref **sought, int nr_sought, struct shallow_info *si, - char **pack_lockfile) + struct string_list *pack_lockfiles) { struct ref *ref = copy_ref_list(orig_ref); struct object_id oid; @@ -992,7 +993,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, alternate_shallow_file = setup_temporary_shallow(si->shallow); else alternate_shallow_file = NULL; - if (get_pack(args, fd, pack_lockfile)) + if (get_pack(args, fd, pack_lockfiles)) die(_("git fetch-pack: fetch failed.")); all_done: @@ -1334,7 +1335,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, int fd[2], const struct ref *orig_ref, struct ref **sought, int nr_sought, - char **pack_lockfile) + struct string_list *pack_lockfiles) { struct ref *ref = copy_ref_list(orig_ref); enum fetch_state state = FETCH_CHECK_LOCAL; @@ -1415,7 +1416,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, /* get the pack */ process_section_header(&reader, "packfile", 0); - if (get_pack(args, fd, pack_lockfile)) + if (get_pack(args, fd, pack_lockfiles)) die(_("git fetch-pack: fetch failed.")); state = FETCH_DONE; @@ -1617,7 +1618,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args, const char *dest, struct ref **sought, int nr_sought, struct oid_array *shallow, - char **pack_lockfile, + struct string_list *pack_lockfiles, enum protocol_version version) { struct ref *ref_cpy; @@ -1648,10 +1649,10 @@ struct ref *fetch_pack(struct fetch_pack_args *args, prepare_shallow_info(&si, shallow); if (version == protocol_v2) ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought, - pack_lockfile); + pack_lockfiles); else ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, - &si, pack_lockfile); + &si, pack_lockfiles); reprepare_packed_git(the_repository); if (!args->cloning && args->deepen) { diff --git a/fetch-pack.h b/fetch-pack.h index 43ec344d95..47af3c1d1e 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -84,7 +84,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args, struct ref **sought, int nr_sought, struct oid_array *shallow, - char **pack_lockfile, + struct string_list *pack_lockfiles, enum protocol_version version); /* diff --git a/transport-helper.c b/transport-helper.c index 1f52c95fd8..94c0e37778 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -397,10 +397,11 @@ static int fetch_with_fetch(struct transport *transport, if (starts_with(buf.buf, "lock ")) { const char *name = buf.buf + 5; - if (transport->pack_lockfile) + if (transport->pack_lockfiles.nr) warning(_("%s also locked %s"), data->name, name); else - transport->pack_lockfile = xstrdup(name); + string_list_append(&transport->pack_lockfiles, + name); } else if (data->check_connectivity && data->transport_options.check_self_contained_and_connected && diff --git a/transport.c b/transport.c index e078812897..80c7903fa7 100644 --- a/transport.c +++ b/transport.c @@ -359,14 +359,14 @@ static int fetch_refs_via_pack(struct transport *transport, refs = fetch_pack(&args, data->fd, data->conn, refs_tmp ? refs_tmp : transport->remote_refs, dest, to_fetch, nr_heads, &data->shallow, - &transport->pack_lockfile, data->version); + &transport->pack_lockfiles, data->version); break; case protocol_v1: case protocol_v0: refs = fetch_pack(&args, data->fd, data->conn, refs_tmp ? refs_tmp : transport->remote_refs, dest, to_fetch, nr_heads, &data->shallow, - &transport->pack_lockfile, data->version); + &transport->pack_lockfiles, data->version); break; case protocol_unknown_version: BUG("unknown protocol version"); @@ -909,6 +909,7 @@ struct transport *transport_get(struct remote *remote, const char *url) struct transport *ret = xcalloc(1, sizeof(*ret)); ret->progress = isatty(2); + string_list_init(&ret->pack_lockfiles, 1); if (!remote) BUG("No remote provided to transport_get()"); @@ -1302,10 +1303,11 @@ int transport_fetch_refs(struct transport *transport, struct ref *refs) void transport_unlock_pack(struct transport *transport) { - if (transport->pack_lockfile) { - unlink_or_warn(transport->pack_lockfile); - FREE_AND_NULL(transport->pack_lockfile); - } + int i; + + for (i = 0; i < transport->pack_lockfiles.nr; i++) + unlink_or_warn(transport->pack_lockfiles.items[i].string); + string_list_clear(&transport->pack_lockfiles, 0); } int transport_connect(struct transport *transport, const char *name, diff --git a/transport.h b/transport.h index f2ee7c4f49..e430677217 100644 --- a/transport.h +++ b/transport.h @@ -5,8 +5,7 @@ #include "run-command.h" #include "remote.h" #include "list-objects-filter-options.h" - -struct string_list; +#include "string-list.h" struct git_transport_options { unsigned thin : 1; @@ -98,7 +97,8 @@ struct transport { */ const struct string_list *server_options; - char *pack_lockfile; + struct string_list pack_lockfiles; + signed verbose : 3; /** * Transports should not set this directly, and should use this From patchwork Fri Mar 8 21:55:20 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jonathan Tan X-Patchwork-Id: 10845479 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 5D84314E1 for ; Fri, 8 Mar 2019 21:56:01 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 45D862CFF6 for ; Fri, 8 Mar 2019 21:56:01 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 3997E2FCCC; Fri, 8 Mar 2019 21:56:01 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-15.5 required=2.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,MAILING_LIST_MULTI,RCVD_IN_DNSWL_HI, USER_IN_DEF_DKIM_WL autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 01D3E2FE07 for ; Fri, 8 Mar 2019 21:56:00 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726456AbfCHVz6 (ORCPT ); Fri, 8 Mar 2019 16:55:58 -0500 Received: from mail-qk1-f201.google.com ([209.85.222.201]:43016 "EHLO mail-qk1-f201.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726267AbfCHVz6 (ORCPT ); Fri, 8 Mar 2019 16:55:58 -0500 Received: by mail-qk1-f201.google.com with SMTP id a11so17207746qkk.10 for ; Fri, 08 Mar 2019 13:55:57 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20161025; h=date:in-reply-to:message-id:mime-version:references:subject:from:to :cc; bh=KHlUdRsvmRLC3jzrMK+9xcgvtvlgxxlZx9J0NjsK+mU=; b=SXvIRpWUIRgAwjXCSVURa7Fz/K2R2YozPAZGYmw3UJTtawPQqmRu4Amql6su4QSc7M wYUMftQMp5raSkyuXXFScIB7OYTVtRvjXPRnGr6E7UiytObrk4cw62lG7DclJg1RZzId w/x2DklXp54ZWMVlNDfAqYqPxWSDmJn6Dhj94nJ+Xp+PH5w6n2jwL26xxEbKU52mx0jy POKk3qoNENtyMfCow+Gyso0OMjeM7IEBSII8lp9peLw6R2iOeW10F5x8egHUF4h81XPX t3RI3VS3kX/sJea4x7B3zZM4f244iaUx65e71i764Ne2FYfhYiU6v37tTjv1dOsCQ46N dPdg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:date:in-reply-to:message-id:mime-version :references:subject:from:to:cc; bh=KHlUdRsvmRLC3jzrMK+9xcgvtvlgxxlZx9J0NjsK+mU=; b=P2+3xP1YrZnUnAh+toaRJZEO3FbiDyhc8Ulx2YcdNV/fBf5pa89wWFlsSrpCR0pkw9 DuPWOqddH1TQR+hw/ttaZ5n5e7fEC7mUXFD990s8NajLTeiAB5frfme2RArj95zUus8G 91bsSQxg3yCH7esAZF6rcWx87ySmOG50oHuy+mEuJT3uepJJYB+5l7PYAhbwpZvMW6rk NL1REMNp11w3bqzfO6bKa7dDGGsxEsyAx8JJ0jkSuBgXvUWx0fAHa4cqMgrlk/yyogfC YkUDkPPChaZf79es/SRgr0m1QtGlvny92qZPrUU+J9DKG8N+H8P+LGqg0DLlWY1aYARZ lPqQ== X-Gm-Message-State: APjAAAUCEvoTvGkhhhv5iemPPc9hTXf9w6v3y7ePlFtC7lBWYDVnYckw 6uRJAjVJJt2Ddg9iDLeC7yC8Fzo3eEKBhONyZ0Bn X-Google-Smtp-Source: APXvYqz4ivWFO0KzzsgSuDoeVUAiHFCdlBLTwKT9N4MBMH3KekUSb6siv02TyPqFpXPnr56JTp+INziSxjZkPGWBJaEe X-Received: by 2002:a0c:88c7:: with SMTP id 7mr12313525qvo.12.1552082156916; Fri, 08 Mar 2019 13:55:56 -0800 (PST) Date: Fri, 8 Mar 2019 13:55:20 -0800 In-Reply-To: Message-Id: Mime-Version: 1.0 References: X-Mailer: git-send-email 2.19.0.271.gfe8321ec05.dirty Subject: [PATCH v2 8/8] upload-pack: send part of packfile response as uri From: Jonathan Tan To: avarab@gmail.com, git@vger.kernel.org Cc: Jonathan Tan , Junio C Hamano Sender: git-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: git@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Teach upload-pack to send part of its packfile response as URIs. An administrator may configure a repository with one or more "uploadpack.blobpackfileuri" lines, each line containing an OID, a pack hash, and a URI. A client may configure fetch.uriprotocols to be a comma-separated list of protocols that it is willing to use to fetch additional packfiles - this list will be sent to the server. Whenever an object with one of those OIDs would appear in the packfile transmitted by upload-pack, the server may exclude that object, and instead send the URI. The client will then download the packs referred to by those URIs before performing the connectivity check. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 76 ++++++++++++++++++++++++++++ fetch-pack.c | 112 +++++++++++++++++++++++++++++++++++++++-- t/t5702-protocol-v2.sh | 57 +++++++++++++++++++++ upload-pack.c | 78 +++++++++++++++++++++++++--- 4 files changed, 312 insertions(+), 11 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index a9fac7c128..2fa962c87d 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -110,6 +110,8 @@ static unsigned long window_memory_limit = 0; static struct list_objects_filter_options filter_options; +static struct string_list uri_protocols = STRING_LIST_INIT_NODUP; + enum missing_action { MA_ERROR = 0, /* fail if any missing objects are encountered */ MA_ALLOW_ANY, /* silently allow ALL missing objects */ @@ -118,6 +120,15 @@ enum missing_action { static enum missing_action arg_missing_action; static show_object_fn fn_show_object; +struct configured_exclusion { + struct oidmap_entry e; + char *pack_hash_hex; + char *uri; +}; +static struct oidmap configured_exclusions; + +static struct oidset excluded_by_config; + /* * stats */ @@ -832,6 +843,25 @@ static off_t write_reused_pack(struct hashfile *f) return reuse_packfile_offset - sizeof(struct pack_header); } +static void write_excluded_by_configs(void) +{ + struct oidset_iter iter; + const struct object_id *oid; + + oidset_iter_init(&excluded_by_config, &iter); + while ((oid = oidset_iter_next(&iter))) { + struct configured_exclusion *ex = + oidmap_get(&configured_exclusions, oid); + + if (!ex) + BUG("configured exclusion wasn't configured"); + write_in_full(1, ex->pack_hash_hex, strlen(ex->pack_hash_hex)); + write_in_full(1, " ", 1); + write_in_full(1, ex->uri, strlen(ex->uri)); + write_in_full(1, "\n", 1); + } +} + static const char no_split_warning[] = N_( "disabling bitmap writing, packs are split due to pack.packSizeLimit" ); @@ -1125,6 +1155,25 @@ static int want_object_in_pack(const struct object_id *oid, } } + if (uri_protocols.nr) { + struct configured_exclusion *ex = + oidmap_get(&configured_exclusions, oid); + int i; + const char *p; + + if (ex) { + for (i = 0; i < uri_protocols.nr; i++) { + if (skip_prefix(ex->uri, + uri_protocols.items[i].string, + &p) && + *p == ':') { + oidset_insert(&excluded_by_config, oid); + return 0; + } + } + } + } + return 1; } @@ -2726,6 +2775,29 @@ static int git_pack_config(const char *k, const char *v, void *cb) pack_idx_opts.version); return 0; } + if (!strcmp(k, "uploadpack.blobpackfileuri")) { + struct configured_exclusion *ex = xmalloc(sizeof(*ex)); + const char *oid_end, *pack_end; + /* + * Stores the pack hash. This is not a true object ID, but is + * of the same form. + */ + struct object_id pack_hash; + + if (parse_oid_hex(v, &ex->e.oid, &oid_end) || + *oid_end != ' ' || + parse_oid_hex(oid_end + 1, &pack_hash, &pack_end) || + *pack_end != ' ') + die(_("value of uploadpack.blobpackfileuri must be " + "of the form ' ' (got '%s')"), v); + if (oidmap_get(&configured_exclusions, &ex->e.oid)) + die(_("object already configured in another " + "uploadpack.blobpackfileuri (got '%s')"), v); + ex->pack_hash_hex = xcalloc(1, pack_end - oid_end); + memcpy(ex->pack_hash_hex, oid_end + 1, pack_end - oid_end - 1); + ex->uri = xstrdup(pack_end + 1); + oidmap_put(&configured_exclusions, ex); + } return git_default_config(k, v, cb); } @@ -3318,6 +3390,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) N_("do not pack objects in promisor packfiles")), OPT_BOOL(0, "delta-islands", &use_delta_islands, N_("respect islands during delta compression")), + OPT_STRING_LIST(0, "uri-protocol", &uri_protocols, + N_("protocol"), + N_("exclude any configured uploadpack.blobpackfileuri with this protocol")), OPT_END(), }; @@ -3492,6 +3567,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) return 0; if (nr_result) prepare_pack(window, depth); + write_excluded_by_configs(); write_pack_file(); if (progress) fprintf_ln(stderr, diff --git a/fetch-pack.c b/fetch-pack.c index cf89af21d9..32695f5243 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -38,6 +38,7 @@ static struct lock_file shallow_lock; static const char *alternate_shallow_file; static char *negotiation_algorithm; static struct strbuf fsck_msg_types = STRBUF_INIT; +static struct string_list uri_protocols = STRING_LIST_INIT_DUP; /* Remember to update object flag allocation in object.h */ #define COMPLETE (1U << 0) @@ -755,7 +756,8 @@ static int sideband_demux(int in, int out, void *data) } static int get_pack(struct fetch_pack_args *args, - int xd[2], struct string_list *pack_lockfiles) + int xd[2], struct string_list *pack_lockfiles, + int only_packfile) { struct async demux; int do_keep = args->keep_pack; @@ -815,8 +817,15 @@ static int get_pack(struct fetch_pack_args *args, "--keep=fetch-pack %"PRIuMAX " on %s", (uintmax_t)getpid(), hostname); } - if (args->check_self_contained_and_connected) + if (only_packfile && args->check_self_contained_and_connected) argv_array_push(&cmd.args, "--check-self-contained-and-connected"); + else + /* + * We cannot perform any connectivity checks because + * not all packs have been downloaded; let the caller + * have this responsibility. + */ + args->check_self_contained_and_connected = 0; if (args->from_promisor) argv_array_push(&cmd.args, "--promisor"); } @@ -993,7 +1002,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, alternate_shallow_file = setup_temporary_shallow(si->shallow); else alternate_shallow_file = NULL; - if (get_pack(args, fd, pack_lockfiles)) + if (get_pack(args, fd, pack_lockfiles, 1)) die(_("git fetch-pack: fetch failed.")); all_done: @@ -1148,6 +1157,26 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, warning("filtering not recognized by server, ignoring"); } + if (server_supports_feature("fetch", "packfile-uris", 0)) { + int i; + struct strbuf to_send = STRBUF_INIT; + + for (i = 0; i < uri_protocols.nr; i++) { + const char *s = uri_protocols.items[i].string; + + if (!strcmp(s, "https") || !strcmp(s, "http")) { + if (to_send.len) + strbuf_addch(&to_send, ','); + strbuf_addstr(&to_send, s); + } + } + if (to_send.len) { + packet_buf_write(&req_buf, "packfile-uris %s", + to_send.buf); + strbuf_release(&to_send); + } + } + /* add wants */ add_wants(args->no_dependents, wants, &req_buf); @@ -1323,6 +1352,21 @@ static void receive_wanted_refs(struct packet_reader *reader, die(_("error processing wanted refs: %d"), reader->status); } +static void receive_packfile_uris(struct packet_reader *reader, + struct string_list *uris) +{ + process_section_header(reader, "packfile-uris", 0); + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + if (reader->pktlen < the_hash_algo->hexsz || + reader->line[the_hash_algo->hexsz] != ' ') + die("expected ' ', got: %s\n", reader->line); + + string_list_append(uris, reader->line); + } + if (reader->status != PACKET_READ_DELIM) + die("expected DELIM"); +} + enum fetch_state { FETCH_CHECK_LOCAL = 0, FETCH_SEND_REQUEST, @@ -1344,6 +1388,9 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, int in_vain = 0; int haves_to_send = INITIAL_FLUSH; struct fetch_negotiator negotiator; + struct string_list packfile_uris = STRING_LIST_INIT_DUP; + int i; + fetch_negotiator_init(&negotiator, negotiation_algorithm); packet_reader_init(&reader, fd[0], NULL, 0, PACKET_READ_CHOMP_NEWLINE | @@ -1414,9 +1461,12 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, if (process_section_header(&reader, "wanted-refs", 1)) receive_wanted_refs(&reader, sought, nr_sought); - /* get the pack */ + /* get the pack(s) */ + if (process_section_header(&reader, "packfile-uris", 1)) + receive_packfile_uris(&reader, &packfile_uris); process_section_header(&reader, "packfile", 0); - if (get_pack(args, fd, pack_lockfiles)) + if (get_pack(args, fd, pack_lockfiles, + !packfile_uris.nr)) die(_("git fetch-pack: fetch failed.")); state = FETCH_DONE; @@ -1426,6 +1476,50 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, } } + for (i = 0; i < packfile_uris.nr; i++) { + struct child_process cmd = CHILD_PROCESS_INIT; + char packname[GIT_MAX_HEXSZ + 1]; + const char *uri = packfile_uris.items[i].string + + the_hash_algo->hexsz + 1; + + argv_array_push(&cmd.args, "http-fetch"); + argv_array_push(&cmd.args, "--packfile"); + argv_array_push(&cmd.args, uri); + cmd.git_cmd = 1; + cmd.no_stdin = 1; + cmd.out = -1; + if (start_command(&cmd)) + die("fetch-pack: unable to spawn http-fetch"); + + if (read_in_full(cmd.out, packname, 5) < 0 || + memcmp(packname, "keep\t", 5)) + die("fetch-pack: expected keep then TAB at start of http-fetch output"); + + if (read_in_full(cmd.out, packname, + the_hash_algo->hexsz + 1) < 0 || + packname[the_hash_algo->hexsz] != '\n') + die("fetch-pack: expected hash then LF at end of http-fetch output"); + + packname[the_hash_algo->hexsz] = '\0'; + + close(cmd.out); + + if (finish_command(&cmd)) + die("fetch-pack: unable to finish http-fetch"); + + if (memcmp(packfile_uris.items[i].string, packname, + the_hash_algo->hexsz)) + die("fetch-pack: pack downloaded from %s does not match expected hash %.*s", + uri, (int) the_hash_algo->hexsz, + packfile_uris.items[i].string); + + string_list_append_nodup(pack_lockfiles, + xstrfmt("%s/pack/pack-%s.keep", + get_object_directory(), + packname)); + } + string_list_clear(&packfile_uris, 0); + negotiator.release(&negotiator); oidset_clear(&common); return ref; @@ -1465,6 +1559,14 @@ static void fetch_pack_config(void) git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects); git_config_get_string("fetch.negotiationalgorithm", &negotiation_algorithm); + if (!uri_protocols.nr) { + char *str; + + if (!git_config_get_string("fetch.uriprotocols", &str) && str) { + string_list_split(&uri_protocols, str, ',', -1); + free(str); + } + } git_config(fetch_pack_config_cb, NULL); } diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index db4ae09f2f..6784dfe279 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -656,6 +656,63 @@ test_expect_success 'when server does not send "ready", expect FLUSH' ' test_i18ngrep "expected no other sections to be sent after no .ready." err ' +configure_exclusion () { + git -C "$1" hash-object "$2" >objh && + git -C "$1" pack-objects "$HTTPD_DOCUMENT_ROOT_PATH/mypack" packh && + git -C "$1" config --add \ + "uploadpack.blobpackfileuri" \ + "$(cat objh) $(cat packh) $HTTPD_URL/dumb/mypack-$(cat packh).pack" && + cat objh +} + +test_expect_success 'part of packfile response provided as URI' ' + P="$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + rm -rf "$P" http_child log && + + git init "$P" && + git -C "$P" config "uploadpack.allowsidebandall" "true" && + + echo my-blob >"$P/my-blob" && + git -C "$P" add my-blob && + echo other-blob >"$P/other-blob" && + git -C "$P" add other-blob && + git -C "$P" commit -m x && + + configure_exclusion "$P" my-blob >h && + configure_exclusion "$P" other-blob >h2 && + + GIT_TRACE=1 GIT_TRACE_PACKET="$(pwd)/log" GIT_TEST_SIDEBAND_ALL=1 \ + git -c protocol.version=2 \ + -c fetch.uriprotocols=http,https \ + clone "$HTTPD_URL/smart/http_parent" http_child && + + # Ensure that my-blob and other-blob are in separate packfiles. + for idx in http_child/.git/objects/pack/*.idx + do + git verify-pack --verbose $idx >out && + { + grep "^[0-9a-f]\{16,\} " out || : + } >out.objectlist && + if test_line_count = 1 out.objectlist + then + if grep $(cat h) out + then + >hfound + fi && + if grep $(cat h2) out + then + >h2found + fi + fi + done && + test -f hfound && + test -f h2found && + + # Ensure that there are exactly 6 files (3 .pack and 3 .idx). + ls http_child/.git/objects/pack/* >filelist && + test_line_count = 6 filelist +' + stop_httpd test_done diff --git a/upload-pack.c b/upload-pack.c index 987d2e139b..d36e1fc06a 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -105,9 +105,12 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data) struct output_state { char buffer[8193]; int used; + unsigned packfile_uris_started : 1; + unsigned packfile_started : 1; }; -static int relay_pack_data(int pack_objects_out, struct output_state *os) +static int relay_pack_data(int pack_objects_out, struct output_state *os, + int write_packfile_line) { /* * We keep the last byte to ourselves @@ -128,6 +131,37 @@ static int relay_pack_data(int pack_objects_out, struct output_state *os) } os->used += readsz; + while (!os->packfile_started) { + char *p; + if (os->used >= 4 && !memcmp(os->buffer, "PACK", 4)) { + os->packfile_started = 1; + if (write_packfile_line) { + if (os->packfile_uris_started) + packet_delim(1); + packet_write_fmt(1, "\1packfile\n"); + } + break; + } + if ((p = memchr(os->buffer, '\n', os->used))) { + if (!os->packfile_uris_started) { + os->packfile_uris_started = 1; + if (!write_packfile_line) + BUG("packfile_uris requires sideband-all"); + packet_write_fmt(1, "\1packfile-uris\n"); + } + *p = '\0'; + packet_write_fmt(1, "\1%s\n", os->buffer); + + os->used -= p - os->buffer + 1; + memmove(os->buffer, p + 1, os->used); + } else { + /* + * Incomplete line. + */ + return readsz; + } + } + if (os->used > 1) { send_client_data(1, os->buffer, os->used - 1); os->buffer[0] = os->buffer[os->used - 1]; @@ -141,7 +175,8 @@ static int relay_pack_data(int pack_objects_out, struct output_state *os) } static void create_pack_file(const struct object_array *have_obj, - const struct object_array *want_obj) + const struct object_array *want_obj, + const struct string_list *uri_protocols) { struct child_process pack_objects = CHILD_PROCESS_INIT; struct output_state output_state = {0}; @@ -192,6 +227,11 @@ static void create_pack_file(const struct object_array *have_obj, expanded_filter_spec.buf); } } + if (uri_protocols) { + for (i = 0; i < uri_protocols->nr; i++) + argv_array_pushf(&pack_objects.args, "--uri-protocol=%s", + uri_protocols->items[0].string); + } pack_objects.in = -1; pack_objects.out = -1; @@ -278,7 +318,8 @@ static void create_pack_file(const struct object_array *have_obj, } if (0 <= pu && (pfd[pu].revents & (POLLIN|POLLHUP))) { int result = relay_pack_data(pack_objects.out, - &output_state); + &output_state, + !!uri_protocols); if (result == 0) { close(pack_objects.out); @@ -1123,7 +1164,7 @@ void upload_pack(struct upload_pack_options *options) if (want_obj.nr) { struct object_array have_obj = OBJECT_ARRAY_INIT; get_common_commits(&reader, &have_obj, &want_obj); - create_pack_file(&have_obj, &want_obj); + create_pack_file(&have_obj, &want_obj, 0); } } @@ -1138,6 +1179,7 @@ struct upload_pack_data { timestamp_t deepen_since; int deepen_rev_list; int deepen_relative; + struct string_list uri_protocols; struct packet_writer writer; @@ -1157,6 +1199,7 @@ static void upload_pack_data_init(struct upload_pack_data *data) struct oid_array haves = OID_ARRAY_INIT; struct object_array shallows = OBJECT_ARRAY_INIT; struct string_list deepen_not = STRING_LIST_INIT_DUP; + struct string_list uri_protocols = STRING_LIST_INIT_DUP; memset(data, 0, sizeof(*data)); data->wants = wants; @@ -1164,6 +1207,7 @@ static void upload_pack_data_init(struct upload_pack_data *data) data->haves = haves; data->shallows = shallows; data->deepen_not = deepen_not; + data->uri_protocols = uri_protocols; packet_writer_init(&data->writer, 1); } @@ -1322,9 +1366,17 @@ static void process_args(struct packet_reader *request, continue; } + if (skip_prefix(arg, "packfile-uris ", &p)) { + string_list_split(&data->uri_protocols, p, ',', -1); + continue; + } + /* ignore unknown lines maybe? */ die("unexpected line: '%s'", arg); } + + if (data->uri_protocols.nr && !data->writer.use_sideband) + string_list_clear(&data->uri_protocols, 0); } static int process_haves(struct oid_array *haves, struct oid_array *common, @@ -1514,8 +1566,13 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys, send_wanted_ref_info(&data); send_shallow_info(&data, &want_obj); - packet_writer_write(&data.writer, "packfile\n"); - create_pack_file(&have_obj, &want_obj); + if (data.uri_protocols.nr) { + create_pack_file(&have_obj, &want_obj, + &data.uri_protocols); + } else { + packet_write_fmt(1, "packfile\n"); + create_pack_file(&have_obj, &want_obj, NULL); + } state = FETCH_DONE; break; case FETCH_DONE: @@ -1536,6 +1593,7 @@ int upload_pack_advertise(struct repository *r, int allow_filter_value; int allow_ref_in_want; int allow_sideband_all_value; + char *str = NULL; strbuf_addstr(value, "shallow"); @@ -1557,6 +1615,14 @@ int upload_pack_advertise(struct repository *r, &allow_sideband_all_value) && allow_sideband_all_value)) strbuf_addstr(value, " sideband-all"); + + if (!repo_config_get_string(the_repository, + "uploadpack.blobpackfileuri", + &str) && + str) { + strbuf_addstr(value, " packfile-uris"); + free(str); + } } return 1;