diff mbox series

[v5,3/6] object-file.c: read stream in a loop in write_loose_object()

Message ID 20211210103435.83656-4-chiyutianyi@gmail.com (mailing list archive)
State New, archived
Headers show
Series unpack large objects in stream | expand

Commit Message

Han Xin Dec. 10, 2021, 10:34 a.m. UTC 
From: Han Xin <hanxin.hx@alibaba-inc.com>

In order to prepare the stream version of "write_loose_object()", read
the input stream in a loop in "write_loose_object()", so that we can
feed the contents of large blob object to "write_loose_object()" using
a small fixed buffer.

Helped-by: Jiang Xin <zhiyou.jx@alibaba-inc.com>
Signed-off-by: Han Xin <hanxin.hx@alibaba-inc.com>
---
 object-file.c | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)
diff mbox series

Patch

diff --git a/object-file.c b/object-file.c
index 41099b137f..455ab3c06e 100644
--- a/object-file.c
+++ b/object-file.c
@@ -1864,7 +1864,7 @@  static int write_loose_object(struct object_id *oid, char *hdr,
 			      int hdrlen, const void *buf, unsigned long len,
 			      time_t mtime, unsigned flags)
 {
-	int fd, ret, err = 0;
+	int fd, ret, err = 0, flush = 0;
 	unsigned char compressed[4096];
 	git_zstream stream;
 	git_hash_ctx c;
@@ -1903,22 +1903,29 @@  static int write_loose_object(struct object_id *oid, char *hdr,
 	the_hash_algo->update_fn(&c, hdr, hdrlen);
 
 	/* Then the data itself.. */
-	if (flags & HASH_STREAM) {
-		struct input_stream *in_stream = (struct input_stream *)buf;
-		stream.next_in = (void *)in_stream->read(in_stream, &len);
-	} else {
+	if (!(flags & HASH_STREAM)) {
 		stream.next_in = (void *)buf;
+		stream.avail_in = len;
+		flush = Z_FINISH;
 	}
-	stream.avail_in = len;
 	do {
 		unsigned char *in0 = stream.next_in;
-		ret = git_deflate(&stream, Z_FINISH);
+		if (flags & HASH_STREAM && !stream.avail_in) {
+			struct input_stream *in_stream = (struct input_stream *)buf;
+			const void *in = in_stream->read(in_stream, &stream.avail_in);
+			stream.next_in = (void *)in;
+			in0 = (unsigned char *)in;
+			/* All data has been read. */
+			if (len + hdrlen == stream.total_in + stream.avail_in)
+				flush = Z_FINISH;
+		}
+		ret = git_deflate(&stream, flush);
 		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
 		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
 			die(_("unable to write loose object file"));
 		stream.next_out = compressed;
 		stream.avail_out = sizeof(compressed);
-	} while (ret == Z_OK);
+	} while (ret == Z_OK || ret == Z_BUF_ERROR);
 
 	if (ret != Z_STREAM_END)
 		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),