diff --git a/src/common/options.cc b/src/common/options.cc
index d91827c1a803..769666d2955c 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -4234,6 +4234,10 @@ std::vector<Option> get_global_options() {
      .set_default(false)
      .set_description(""),

+    Option("osd_immediate_completions", Option::TYPE_BOOL, 
Option::LEVEL_ADVANCED)
+    .set_default(false)
+    .set_description(""),
+
      // --------------------------
      // bluestore

diff --git a/src/crimson/os/cyan_store.cc b/src/crimson/os/cyan_store.cc
index f0749cb921f9..c05e0e40b721 100644
--- a/src/crimson/os/cyan_store.cc
+++ b/src/crimson/os/cyan_store.cc
@@ -463,7 +463,11 @@ int CyanStore::_write(const coll_t& cid, const 
ghobject_t& oid,
      return -ENOENT;

    ObjectRef o = c->get_or_create_object(oid);
-  if (len > 0 && !local_conf()->memstore_debug_omit_block_device_write) 
{
+  if (len > 0 &&
+      (!local_conf()->memstore_debug_omit_block_device_write ||
+       // We still want cluster meta-data to be saved, so pass only 
small
+       // writes, expecting user writes will be >= 4k.
+       len < 4096)) {
      const ssize_t old_size = o->get_size();
      o->write(offset, bl);
      used_bytes += (o->get_size() - old_size);
diff --git a/src/crimson/osd/ops_executer.cc 
b/src/crimson/osd/ops_executer.cc
index 13f6f086c4ea..a76fc6e206d8 100644
--- a/src/crimson/osd/ops_executer.cc
+++ b/src/crimson/osd/ops_executer.cc
@@ -431,8 +431,8 @@ OpsExecuter::execute_osd_op(OSDOp& osd_op)

    default:
      logger().warn("unknown op {}", ceph_osd_op_name(op.op));
-    throw std::runtime_error(
-      fmt::format("op '{}' not supported", ceph_osd_op_name(op.op)));
+    // Without that `fio examples/rbd.fio` hangs on exit
+    throw ceph::osd::operation_not_supported{};
    }
  }

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index ddd8742d1a74..737cc266766e 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -17,6 +17,7 @@
  #include "messages/MOSDOp.h"
  #include "messages/MOSDPGLog.h"
  #include "messages/MOSDRepOpReply.h"
+#include "messages/MOSDOpReply.h"
  #include "messages/MPGStats.h"

  #include "os/Transaction.h"
@@ -881,6 +882,25 @@ seastar::future<> OSD::committed_osd_maps(version_t 
first,
  seastar::future<> OSD::handle_osd_op(ceph::net::Connection* conn,
                                       Ref<MOSDOp> m)
  {
+
+  //
+  // Immediately complete requests even without leaving the transport
+  //
+  if (local_conf().get_val<bool>("osd_immediate_completions")) {
+    m->finish_decode();
+
+    for (auto op : m->ops) {
+      if (op.op.op == CEPH_OSD_OP_WRITE &&
+          // Complete big writes only
+          op.op.extent.length >= 4096) {
+
+        auto reply = make_message<MOSDOpReply>(m.get(), 0, 
osdmap->get_epoch(),
+                                CEPH_OSD_FLAG_ACK | 
CEPH_OSD_FLAG_ONDISK, true);
+        return conn->send(reply);
+      }
+    }
+  }
+
    shard_services.start_operation<ClientRequest>(
      *this,
      conn->get_shared(),
diff --git a/src/os/memstore/MemStore.cc b/src/os/memstore/MemStore.cc
index 05d16edb6cc0..265dc64c808d 100644
--- a/src/os/memstore/MemStore.cc
+++ b/src/os/memstore/MemStore.cc
@@ -1047,7 +1047,11 @@ int MemStore::_write(const coll_t& cid, const 
ghobject_t& oid,
      return -ENOENT;

    ObjectRef o = c->get_or_create_object(oid);
-  if (len > 0 && !cct->_conf->memstore_debug_omit_block_device_write) {
+  if (len > 0 &&
+      (!cct->_conf->memstore_debug_omit_block_device_write ||
+       // We still want cluster meta-data to be saved, so pass only 
small
+       // writes, expecting user writes will be bigger than 4k.
+       len < 4096)) {
      const ssize_t old_size = o->get_size();
      o->write(offset, bl);
      used_bytes += (o->get_size() - old_size);
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 96aed0b706e3..796bf927126f 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -7223,6 +7223,30 @@ void OSD::ms_fast_dispatch(Message *m)
      return;
    }

+  //
+  // Immediately complete requests even without leaving the transport
+  //
+  if (g_conf().get_val<bool>("osd_immediate_completions") &&
+      m->get_type() == CEPH_MSG_OSD_OP) {
+    MOSDOp *osdop = static_cast<MOSDOp*>(m);
+
+    osdop->finish_decode();
+
+    for (auto op : osdop->ops) {
+      if (op.op.op == CEPH_OSD_OP_WRITE &&
+          // Complete big writes only
+          op.op.extent.length >= 4096) {
+        MOSDOpReply *reply;
+
+        reply = new MOSDOpReply(osdop, 0, osdmap->get_epoch(),
+                                CEPH_OSD_FLAG_ACK | 
CEPH_OSD_FLAG_ONDISK, true);
+        osdop->get_connection()->send_message(reply);
+        m->put();
+        return;
+      }
+    }
+  }
+
    // peering event?
    switch (m->get_type()) {
