@@ -822,11 +822,15 @@ void __init btrfs_init_compress(void)
/*
* Preallocate one workspace for each compression type so
- * we can guarantee forward progress in the worst case
+ * we can guarantee forward progress in the worst case.
+ * Provide the maximum compression level to guarantee large
+ * enough workspace.
*/
- workspace = btrfs_compress_op[i]->alloc_workspace();
+ workspace = btrfs_compress_op[i]->alloc_workspace(
+ btrfs_compress_op[i]->max_level);
if (IS_ERR(workspace)) {
- pr_warn("BTRFS: cannot preallocate compression workspace, will try later\n");
+ pr_warn("BTRFS: cannot preallocate compression "
+ "workspace, will try later\n");
} else {
atomic_set(&btrfs_comp_ws[i].total_ws, 1);
btrfs_comp_ws[i].free_ws = 1;
@@ -835,23 +839,78 @@ void __init btrfs_init_compress(void)
}
}
+/*
+ * put a workspace struct back on the list or free it if we have enough
+ * idle ones sitting around
+ */
+static void __free_workspace(int type, struct list_head *workspace,
+ bool heuristic)
+{
+ int idx = type - 1;
+ struct list_head *idle_ws;
+ spinlock_t *ws_lock;
+ atomic_t *total_ws;
+ wait_queue_head_t *ws_wait;
+ int *free_ws;
+
+ if (heuristic) {
+ idle_ws = &btrfs_heuristic_ws.idle_ws;
+ ws_lock = &btrfs_heuristic_ws.ws_lock;
+ total_ws = &btrfs_heuristic_ws.total_ws;
+ ws_wait = &btrfs_heuristic_ws.ws_wait;
+ free_ws = &btrfs_heuristic_ws.free_ws;
+ } else {
+ idle_ws = &btrfs_comp_ws[idx].idle_ws;
+ ws_lock = &btrfs_comp_ws[idx].ws_lock;
+ total_ws = &btrfs_comp_ws[idx].total_ws;
+ ws_wait = &btrfs_comp_ws[idx].ws_wait;
+ free_ws = &btrfs_comp_ws[idx].free_ws;
+ }
+
+ spin_lock(ws_lock);
+ if (*free_ws <= num_online_cpus()) {
+ list_add(workspace, idle_ws);
+ (*free_ws)++;
+ spin_unlock(ws_lock);
+ goto wake;
+ }
+ spin_unlock(ws_lock);
+
+ if (heuristic)
+ free_heuristic_ws(workspace);
+ else
+ btrfs_compress_op[idx]->free_workspace(workspace);
+ atomic_dec(total_ws);
+wake:
+ cond_wake_up(ws_wait);
+}
+
+static void free_workspace(int type, struct list_head *ws)
+{
+ return __free_workspace(type, ws, false);
+}
+
/*
* This finds an available workspace or allocates a new one.
* If it's not possible to allocate a new one, waits until there's one.
* Preallocation makes a forward progress guarantees and we do not return
* errors.
*/
-static struct list_head *__find_workspace(int type, bool heuristic)
+static struct list_head *__find_workspace(unsigned int type_level,
+ bool heuristic)
{
struct list_head *workspace;
int cpus = num_online_cpus();
+ int type = type_level & 0xF;
int idx = type - 1;
- unsigned nofs_flag;
+ unsigned int level = (type_level & 0xF0) >> 4;
+ unsigned int nofs_flag;
struct list_head *idle_ws;
spinlock_t *ws_lock;
atomic_t *total_ws;
wait_queue_head_t *ws_wait;
int *free_ws;
+ int ret;
if (heuristic) {
idle_ws = &btrfs_heuristic_ws.idle_ws;
@@ -874,8 +933,17 @@ static struct list_head *__find_workspace(int type, bool heuristic)
list_del(workspace);
(*free_ws)--;
spin_unlock(ws_lock);
+ if (!heuristic) {
+ nofs_flag = memalloc_nofs_save();
+ ret = btrfs_compress_op[idx]->set_level(workspace,
+ level);
+ memalloc_nofs_restore(nofs_flag);
+ if (!ret) {
+ free_workspace(type, workspace);
+ goto again;
+ }
+ }
return workspace;
-
}
if (atomic_read(total_ws) > cpus) {
DEFINE_WAIT(wait);
@@ -899,7 +967,8 @@ static struct list_head *__find_workspace(int type, bool heuristic)
if (heuristic)
workspace = alloc_heuristic_ws();
else
- workspace = btrfs_compress_op[idx]->alloc_workspace();
+ workspace = btrfs_compress_op[idx]->alloc_workspace(level);
+
memalloc_nofs_restore(nofs_flag);
if (IS_ERR(workspace)) {
@@ -930,60 +999,22 @@ static struct list_head *__find_workspace(int type, bool heuristic)
return workspace;
}
-static struct list_head *find_workspace(int type)
+static struct list_head *find_workspace(unsigned int type_level)
{
- return __find_workspace(type, false);
+ return __find_workspace(type_level, false);
}
-/*
- * put a workspace struct back on the list or free it if we have enough
- * idle ones sitting around
- */
-static void __free_workspace(int type, struct list_head *workspace,
- bool heuristic)
+static struct list_head *find_decompression_workspace(unsigned int type)
{
- int idx = type - 1;
- struct list_head *idle_ws;
- spinlock_t *ws_lock;
- atomic_t *total_ws;
- wait_queue_head_t *ws_wait;
- int *free_ws;
+ /*
+ * Use the lowest level for decompression, since we don't need to
+ * compress. This can help us save memory when using levels lower than
+ * the default level.
+ */
+ const unsigned int level = 1;
+ const unsigned int type_level = (level << 4) | (type & 0xF);
- if (heuristic) {
- idle_ws = &btrfs_heuristic_ws.idle_ws;
- ws_lock = &btrfs_heuristic_ws.ws_lock;
- total_ws = &btrfs_heuristic_ws.total_ws;
- ws_wait = &btrfs_heuristic_ws.ws_wait;
- free_ws = &btrfs_heuristic_ws.free_ws;
- } else {
- idle_ws = &btrfs_comp_ws[idx].idle_ws;
- ws_lock = &btrfs_comp_ws[idx].ws_lock;
- total_ws = &btrfs_comp_ws[idx].total_ws;
- ws_wait = &btrfs_comp_ws[idx].ws_wait;
- free_ws = &btrfs_comp_ws[idx].free_ws;
- }
-
- spin_lock(ws_lock);
- if (*free_ws <= num_online_cpus()) {
- list_add(workspace, idle_ws);
- (*free_ws)++;
- spin_unlock(ws_lock);
- goto wake;
- }
- spin_unlock(ws_lock);
-
- if (heuristic)
- free_heuristic_ws(workspace);
- else
- btrfs_compress_op[idx]->free_workspace(workspace);
- atomic_dec(total_ws);
-wake:
- cond_wake_up(ws_wait);
-}
-
-static void free_workspace(int type, struct list_head *ws)
-{
- return __free_workspace(type, ws, false);
+ return find_workspace(type_level);
}
/*
@@ -1044,9 +1075,7 @@ int btrfs_compress_pages(unsigned int type_level, struct address_space *mapping,
int ret;
int type = type_level & 0xF;
- workspace = find_workspace(type);
-
- btrfs_compress_op[type - 1]->set_level(workspace, type_level);
+ workspace = find_workspace(type_level);
ret = btrfs_compress_op[type-1]->compress_pages(workspace, mapping,
start, pages,
out_pages,
@@ -1075,7 +1104,7 @@ static int btrfs_decompress_bio(struct compressed_bio *cb)
int ret;
int type = cb->compress_type;
- workspace = find_workspace(type);
+ workspace = find_decompression_workspace(type);
ret = btrfs_compress_op[type - 1]->decompress_bio(workspace, cb);
free_workspace(type, workspace);
@@ -1093,7 +1122,7 @@ int btrfs_decompress(int type, unsigned char *data_in, struct page *dest_page,
struct list_head *workspace;
int ret;
- workspace = find_workspace(type);
+ workspace = find_decompression_workspace(type);
ret = btrfs_compress_op[type-1]->decompress(workspace, data_in,
dest_page, start_byte,
@@ -1591,12 +1620,23 @@ int btrfs_compress_heuristic(struct inode *inode, u64 start, u64 end)
unsigned int btrfs_compress_str2level(const char *str)
{
- if (strncmp(str, "zlib", 4) != 0)
+ int ret;
+ int type;
+ unsigned int level;
+
+ if (strncmp(str, "zlib", 4) == 0)
+ type = BTRFS_COMPRESS_ZLIB;
+ else if (strncmp(str, "zstd", 4) == 0)
+ type = BTRFS_COMPRESS_ZSTD;
+ else
return 0;
- /* Accepted form: zlib:1 up to zlib:9 and nothing left after the number */
- if (str[4] == ':' && '1' <= str[5] && str[5] <= '9' && str[6] == 0)
- return str[5] - '0';
+ if (str[4] == ':') {
+ ret = kstrtouint(str + 5, 10, &level);
+ if (ret == 0 && 0 < level &&
+ level <= btrfs_compress_op[type-1]->max_level)
+ return level;
+ }
- return BTRFS_ZLIB_DEFAULT_LEVEL;
+ return btrfs_compress_op[type-1]->default_level;
}
@@ -23,8 +23,6 @@
/* Maximum size of data before compression */
#define BTRFS_MAX_UNCOMPRESSED (SZ_128K)
-#define BTRFS_ZLIB_DEFAULT_LEVEL 3
-
struct compressed_bio {
/* number of bios pending for this compressed extent */
refcount_t pending_bios;
@@ -87,7 +85,7 @@ blk_status_t btrfs_submit_compressed_write(struct inode *inode, u64 start,
blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
int mirror_num, unsigned long bio_flags);
-unsigned btrfs_compress_str2level(const char *str);
+unsigned int btrfs_compress_str2level(const char *str);
enum btrfs_compression_type {
BTRFS_COMPRESS_NONE = 0,
@@ -98,7 +96,7 @@ enum btrfs_compression_type {
};
struct btrfs_compress_op {
- struct list_head *(*alloc_workspace)(void);
+ struct list_head *(*alloc_workspace)(unsigned int level);
void (*free_workspace)(struct list_head *workspace);
@@ -119,7 +117,17 @@ struct btrfs_compress_op {
unsigned long start_byte,
size_t srclen, size_t destlen);
- void (*set_level)(struct list_head *ws, unsigned int type);
+ /*
+ * Check if memory allocated in workspace is greater than
+ * or equal to memory needed to compress with given level.
+ * If not, try to reallocate memory for the compression level.
+ * Returns true on success.
+ */
+ bool (*set_level)(struct list_head *ws, unsigned int level);
+
+ unsigned int max_level;
+
+ unsigned int default_level;
};
extern const struct btrfs_compress_op btrfs_zlib_compress;
@@ -71,7 +71,7 @@ static void lzo_free_workspace(struct list_head *ws)
kfree(workspace);
}
-static struct list_head *lzo_alloc_workspace(void)
+static struct list_head *lzo_alloc_workspace(unsigned int level)
{
struct workspace *workspace;
@@ -485,8 +485,9 @@ static int lzo_decompress(struct list_head *ws, unsigned char *data_in,
return ret;
}
-static void lzo_set_level(struct list_head *ws, unsigned int type)
+static bool lzo_set_level(struct list_head *ws, unsigned int level)
{
+ return true;
}
const struct btrfs_compress_op btrfs_lzo_compress = {
@@ -520,7 +520,8 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options,
compress_type = "zlib";
info->compress_type = BTRFS_COMPRESS_ZLIB;
- info->compress_level = BTRFS_ZLIB_DEFAULT_LEVEL;
+ info->compress_level =
+ btrfs_zlib_compress.default_level;
/*
* args[0] contains uninitialized data since
* for these tokens we don't expect any
@@ -542,9 +543,11 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options,
btrfs_clear_opt(info->mount_opt, NODATASUM);
btrfs_set_fs_incompat(info, COMPRESS_LZO);
no_compress = 0;
- } else if (strcmp(args[0].from, "zstd") == 0) {
+ } else if (strncmp(args[0].from, "zstd", 4) == 0) {
compress_type = "zstd";
info->compress_type = BTRFS_COMPRESS_ZSTD;
+ info->compress_level =
+ btrfs_compress_str2level(args[0].from);
btrfs_set_opt(info->mount_opt, COMPRESS);
btrfs_clear_opt(info->mount_opt, NODATACOW);
btrfs_clear_opt(info->mount_opt, NODATASUM);
@@ -20,6 +20,9 @@
#include <linux/refcount.h>
#include "compression.h"
+#define BTRFS_ZLIB_DEFAULT_LEVEL 3
+#define BTRFS_ZLIB_MAX_LEVEL 9
+
struct workspace {
z_stream strm;
char *buf;
@@ -36,7 +39,19 @@ static void zlib_free_workspace(struct list_head *ws)
kfree(workspace);
}
-static struct list_head *zlib_alloc_workspace(void)
+static bool zlib_set_level(struct list_head *ws, unsigned int level)
+{
+ struct workspace *workspace = list_entry(ws, struct workspace, list);
+
+ if (level > BTRFS_ZLIB_MAX_LEVEL)
+ level = BTRFS_ZLIB_MAX_LEVEL;
+
+ workspace->level = level > 0 ? level : BTRFS_ZLIB_DEFAULT_LEVEL;
+
+ return true;
+}
+
+static struct list_head *zlib_alloc_workspace(unsigned int level)
{
struct workspace *workspace;
int workspacesize;
@@ -53,6 +68,7 @@ static struct list_head *zlib_alloc_workspace(void)
goto fail;
INIT_LIST_HEAD(&workspace->list);
+ zlib_set_level(&workspace->list, level);
return &workspace->list;
fail:
@@ -390,22 +406,13 @@ static int zlib_decompress(struct list_head *ws, unsigned char *data_in,
return ret;
}
-static void zlib_set_level(struct list_head *ws, unsigned int type)
-{
- struct workspace *workspace = list_entry(ws, struct workspace, list);
- unsigned level = (type & 0xF0) >> 4;
-
- if (level > 9)
- level = 9;
-
- workspace->level = level > 0 ? level : 3;
-}
-
const struct btrfs_compress_op btrfs_zlib_compress = {
.alloc_workspace = zlib_alloc_workspace,
.free_workspace = zlib_free_workspace,
.compress_pages = zlib_compress_pages,
.decompress_bio = zlib_decompress_bio,
.decompress = zlib_decompress,
- .set_level = zlib_set_level,
+ .set_level = zlib_set_level,
+ .max_level = BTRFS_ZLIB_MAX_LEVEL,
+ .default_level = BTRFS_ZLIB_DEFAULT_LEVEL,
};
@@ -19,12 +19,13 @@
#define ZSTD_BTRFS_MAX_WINDOWLOG 17
#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
-#define ZSTD_BTRFS_DEFAULT_LEVEL 3
+#define BTRFS_ZSTD_DEFAULT_LEVEL 3
+#define BTRFS_ZSTD_MAX_LEVEL 15
-static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len)
+static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len,
+ unsigned int level)
{
- ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL,
- src_len, 0);
+ ZSTD_parameters params = ZSTD_getParams(level, src_len, 0);
if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
@@ -37,10 +38,25 @@ struct workspace {
size_t size;
char *buf;
struct list_head list;
+ unsigned int level;
ZSTD_inBuffer in_buf;
ZSTD_outBuffer out_buf;
};
+static bool zstd_reallocate_mem(struct workspace *workspace, int size)
+{
+ void *new_mem;
+
+ new_mem = kvmalloc(size, GFP_KERNEL);
+ if (new_mem) {
+ kvfree(workspace->mem);
+ workspace->mem = new_mem;
+ workspace->size = size;
+ return true;
+ }
+ return false;
+}
+
static void zstd_free_workspace(struct list_head *ws)
{
struct workspace *workspace = list_entry(ws, struct workspace, list);
@@ -50,10 +66,34 @@ static void zstd_free_workspace(struct list_head *ws)
kfree(workspace);
}
-static struct list_head *zstd_alloc_workspace(void)
+static bool zstd_set_level(struct list_head *ws, unsigned int level)
+{
+ struct workspace *workspace = list_entry(ws, struct workspace, list);
+ ZSTD_parameters params;
+ int size;
+
+ if (level > BTRFS_ZSTD_MAX_LEVEL)
+ level = BTRFS_ZSTD_MAX_LEVEL;
+
+ if (level == 0)
+ level = BTRFS_ZSTD_DEFAULT_LEVEL;
+
+ params = ZSTD_getParams(level, ZSTD_BTRFS_MAX_INPUT, 0);
+ size = max_t(size_t,
+ ZSTD_CStreamWorkspaceBound(params.cParams),
+ ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
+ if (size > workspace->size) {
+ if (!zstd_reallocate_mem(workspace, size))
+ return false;
+ }
+ workspace->level = level;
+ return true;
+}
+
+static struct list_head *zstd_alloc_workspace(unsigned int level)
{
ZSTD_parameters params =
- zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT);
+ zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT, level);
struct workspace *workspace;
workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
@@ -69,6 +109,7 @@ static struct list_head *zstd_alloc_workspace(void)
goto fail;
INIT_LIST_HEAD(&workspace->list);
+ zstd_set_level(&workspace->list, level);
return &workspace->list;
fail:
@@ -95,7 +136,8 @@ static int zstd_compress_pages(struct list_head *ws,
unsigned long len = *total_out;
const unsigned long nr_dest_pages = *out_pages;
unsigned long max_out = nr_dest_pages * PAGE_SIZE;
- ZSTD_parameters params = zstd_get_btrfs_parameters(len);
+ ZSTD_parameters params = zstd_get_btrfs_parameters(len,
+ workspace->level);
*out_pages = 0;
*total_out = 0;
@@ -419,15 +461,13 @@ static int zstd_decompress(struct list_head *ws, unsigned char *data_in,
return ret;
}
-static void zstd_set_level(struct list_head *ws, unsigned int type)
-{
-}
-
const struct btrfs_compress_op btrfs_zstd_compress = {
- .alloc_workspace = zstd_alloc_workspace,
- .free_workspace = zstd_free_workspace,
- .compress_pages = zstd_compress_pages,
- .decompress_bio = zstd_decompress_bio,
- .decompress = zstd_decompress,
- .set_level = zstd_set_level,
+ .alloc_workspace = zstd_alloc_workspace,
+ .free_workspace = zstd_free_workspace,
+ .compress_pages = zstd_compress_pages,
+ .decompress_bio = zstd_decompress_bio,
+ .decompress = zstd_decompress,
+ .set_level = zstd_set_level,
+ .max_level = BTRFS_ZSTD_MAX_LEVEL,
+ .default_level = BTRFS_ZSTD_DEFAULT_LEVEL,
};