From e8df57b5a49a70e2daa5bb6c00b8e0e06c51306a Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Thu, 6 Apr 2023 14:55:25 -0400
Subject: ioq: Implement an async I/O queue

---
 src/ioq.c | 284 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 284 insertions(+)
 create mode 100644 src/ioq.c

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
new file mode 100644
index 0000000..e09c2a9
--- /dev/null
+++ b/src/ioq.c
@@ -0,0 +1,284 @@
+// Copyright © Tavian Barnes <tavianator@tavianator.com>
+// SPDX-License-Identifier: 0BSD
+
+#include "ioq.h"
+#include "dir.h"
+#include "list.h"
+#include "lock.h"
+#include "sanity.h"
+#include <assert.h>
+#include <errno.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdlib.h>
+
+/**
+ * An I/O queue request.
+ */
+struct ioq_req {
+	/** Base file descriptor for openat(). */
+	int dfd;
+	/** Relative path to dfd. */
+	const char *path;
+
+	/** Arbitrary user data. */
+	void *ptr;
+};
+
+/**
+ * An I/O queue command.
+ */
+struct ioq_cmd {
+	union {
+		struct ioq_req req;
+		struct ioq_res res;
+	};
+
+	struct ioq_cmd *next;
+};
+
+/**
+ * An MPMC queue of I/O commands.
+ */
+struct ioqq {
+	pthread_mutex_t mutex;
+	pthread_cond_t cond;
+
+	bool stop;
+
+	struct ioq_cmd *head;
+	struct ioq_cmd **tail;
+};
+
+static struct ioqq *ioqq_create(void) {
+	struct ioqq *ioqq = malloc(sizeof(*ioqq));
+	if (!ioqq) {
+		goto fail;
+	}
+
+	if (mutex_init(&ioqq->mutex, NULL) != 0) {
+		goto fail_free;
+	}
+
+	if (cond_init(&ioqq->cond, NULL) != 0) {
+		goto fail_mutex;
+	}
+
+	ioqq->stop = false;
+	SLIST_INIT(ioqq);
+	return ioqq;
+
+fail_mutex:
+	mutex_destroy(&ioqq->mutex);
+fail_free:
+	free(ioqq);
+fail:
+	return NULL;
+}
+
+/** Push a command onto the queue. */
+static void ioqq_push(struct ioqq *ioqq, struct ioq_cmd *cmd) {
+	mutex_lock(&ioqq->mutex);
+	SLIST_APPEND(ioqq, cmd);
+	mutex_unlock(&ioqq->mutex);
+	cond_signal(&ioqq->cond);
+}
+
+/** Pop a command from a queue. */
+static struct ioq_cmd *ioqq_pop(struct ioqq *ioqq) {
+	mutex_lock(&ioqq->mutex);
+
+	while (!ioqq->stop && !ioqq->head) {
+		cond_wait(&ioqq->cond, &ioqq->mutex);
+	}
+
+	struct ioq_cmd *cmd = SLIST_POP(ioqq);
+	mutex_unlock(&ioqq->mutex);
+	return cmd;
+}
+
+/** Pop a command from a queue without blocking. */
+static struct ioq_cmd *ioqq_trypop(struct ioqq *ioqq) {
+	if (!mutex_trylock(&ioqq->mutex)) {
+		return NULL;
+	}
+
+	struct ioq_cmd *cmd = SLIST_POP(ioqq);
+	mutex_unlock(&ioqq->mutex);
+	return cmd;
+}
+
+/** Stop a queue, waking up any waiters. */
+static void ioqq_stop(struct ioqq *ioqq) {
+	mutex_lock(&ioqq->mutex);
+	ioqq->stop = true;
+	mutex_unlock(&ioqq->mutex);
+	cond_broadcast(&ioqq->cond);
+}
+
+static void ioqq_destroy(struct ioqq *ioqq) {
+	if (ioqq) {
+		cond_destroy(&ioqq->cond);
+		mutex_destroy(&ioqq->mutex);
+		free(ioqq);
+	}
+}
+
+struct ioq {
+	/** The depth of the queue. */
+	size_t depth;
+	/** The current size of the queue. */
+	size_t size;
+
+	/** Pending I/O requests. */
+	struct ioqq *pending;
+	/** Ready I/O responses. */
+	struct ioqq *ready;
+
+	/** The number of background threads. */
+	size_t nthreads;
+	/** The background threads themselves. */
+	pthread_t *threads;
+};
+
+/** Background thread entry point. */
+static void *ioq_work(void *ptr) {
+	struct ioq *ioq = ptr;
+
+	while (true) {
+		struct ioq_cmd *cmd = ioqq_pop(ioq->pending);
+		if (!cmd) {
+			break;
+		}
+
+		struct ioq_req req = cmd->req;
+		sanitize_uninit(cmd);
+
+		struct ioq_res *res = &cmd->res;
+		res->dir = bfs_opendir(req.dfd, req.path);
+		res->error = errno;
+		ioqq_push(ioq->ready, cmd);
+	}
+
+	return NULL;
+}
+
+struct ioq *ioq_create(size_t depth, size_t threads) {
+	struct ioq *ioq = malloc(sizeof(*ioq));
+	if (!ioq) {
+		goto fail;
+	}
+
+	ioq->depth = depth;
+	ioq->size = 0;
+	ioq->pending = NULL;
+	ioq->ready = NULL;
+	ioq->nthreads = 0;
+
+	ioq->pending = ioqq_create();
+	if (!ioq->pending) {
+		goto fail;
+	}
+
+	ioq->ready = ioqq_create();
+	if (!ioq->ready) {
+		goto fail;
+	}
+
+	ioq->threads = malloc(threads * sizeof(ioq->threads[0]));
+	if (!ioq->threads) {
+		goto fail;
+	}
+
+	for (size_t i = 0; i < threads; ++i) {
+		errno = pthread_create(&ioq->threads[i], NULL, ioq_work, ioq);
+		if (errno != 0) {
+			goto fail;
+		}
+		++ioq->nthreads;
+	}
+
+	return ioq;
+
+	int err;
+fail:
+	err = errno;
+	ioq_destroy(ioq);
+	errno = err;
+	return NULL;
+}
+
+int ioq_opendir(struct ioq *ioq, int dfd, const char *path, void *ptr) {
+	if (ioq->size >= ioq->depth) {
+		return -1;
+	}
+
+	struct ioq_cmd *cmd = malloc(sizeof(*cmd));
+	if (!cmd) {
+		return -1;
+	}
+
+	struct ioq_req *req = &cmd->req;
+	req->dfd = dfd;
+	req->path = path;
+	req->ptr = ptr;
+
+	++ioq->size;
+	ioqq_push(ioq->pending, cmd);
+	return 0;
+}
+
+struct ioq_res *ioq_pop(struct ioq *ioq) {
+	if (ioq->size == 0) {
+		return NULL;
+	}
+
+	struct ioq_cmd *cmd = ioqq_pop(ioq->ready);
+	if (!cmd) {
+		return NULL;
+	}
+
+	--ioq->size;
+	return &cmd->res;
+}
+
+struct ioq_res *ioq_trypop(struct ioq *ioq) {
+	if (ioq->size == 0) {
+		return NULL;
+	}
+
+	struct ioq_cmd *cmd = ioqq_trypop(ioq->ready);
+	if (!cmd) {
+		return NULL;
+	}
+
+	--ioq->size;
+	return &cmd->res;
+}
+
+void ioq_free(struct ioq *ioq, struct ioq_res *res) {
+	struct ioq_cmd *cmd = (struct ioq_cmd *)res;
+	free(cmd);
+}
+
+void ioq_destroy(struct ioq *ioq) {
+	if (!ioq) {
+		return;
+	}
+
+	if (ioq->pending) {
+		ioqq_stop(ioq->pending);
+	}
+
+	for (size_t i = 0; i < ioq->nthreads; ++i) {
+		if (pthread_join(ioq->threads[i], NULL) != 0) {
+			abort();
+		}
+	}
+	free(ioq->threads);
+
+	ioqq_destroy(ioq->ready);
+	ioqq_destroy(ioq->pending);
+
+	free(ioq);
+}
-- 
cgit v1.2.3


From 0c10a1e984bc2bed1fe3a7dadbd94a9c24139a91 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Tue, 11 Apr 2023 22:06:42 -0400
Subject: dir: Add a flag to bfs_freedir() to force the fd to stay the same

---
 src/bftw.c |  4 ++--
 src/dir.c  | 18 +++++++++++++-----
 src/dir.h  | 10 ++++++++--
 src/ioq.c  |  1 +
 4 files changed, 24 insertions(+), 9 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/bftw.c b/src/bftw.c
index 916a56b..a3e8d9b 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -853,13 +853,13 @@ static int bftw_gc(struct bftw_state *state, enum bftw_gc_flags flags) {
 
 		if (file->refcount > 1) {
 			// Keep the fd around if any subdirectories exist
-			file->fd = bfs_freedir(state->dir);
+			file->fd = bfs_freedir(state->dir, false);
 		} else {
-			bfs_closedir(state->dir);
 			file->fd = -1;
 		}
 
 		if (file->fd < 0) {
+			bfs_closedir(state->dir);
 			bftw_cache_remove(&state->cache, file);
 		}
 	}
diff --git a/src/dir.c b/src/dir.c
index 01d26db..126f473 100644
--- a/src/dir.c
+++ b/src/dir.c
@@ -243,13 +243,15 @@ int bfs_closedir(struct bfs_dir *dir) {
 	int ret = xclose(dir->fd);
 #else
 	int ret = closedir(dir->dir);
-	bfs_verify(ret == 0 || errno != EBADF);
+	if (ret != 0) {
+		bfs_verify(errno != EBADF);
+	}
 #endif
 	free(dir);
 	return ret;
 }
 
-int bfs_freedir(struct bfs_dir *dir) {
+int bfs_freedir(struct bfs_dir *dir, bool same_fd) {
 #if BFS_GETDENTS
 	int ret = dir->fd;
 	free(dir);
@@ -257,10 +259,16 @@ int bfs_freedir(struct bfs_dir *dir) {
 	int ret = fdclosedir(dir->dir);
 	free(dir);
 #else
+	if (same_fd) {
+		errno = ENOTSUP;
+		return -1;
+	}
+
 	int ret = dup_cloexec(dirfd(dir->dir));
-	int error = errno;
-	bfs_closedir(dir);
-	errno = error;
+	if (ret >= 0) {
+		bfs_closedir(dir);
+	}
 #endif
+
 	return ret;
 }
diff --git a/src/dir.h b/src/dir.h
index 01eaaba..e0fe913 100644
--- a/src/dir.h
+++ b/src/dir.h
@@ -8,6 +8,7 @@
 #ifndef BFS_DIR_H
 #define BFS_DIR_H
 
+#include "config.h"
 #include <sys/types.h>
 
 /**
@@ -103,9 +104,14 @@ int bfs_closedir(struct bfs_dir *dir);
  *
  * @param dir
  *         The directory to free.
+ * @param same_fd
+ *         If true, require that the returned file descriptor is the same one
+ *         that bfs_dirfd() would have returned.  Otherwise, it may be a new
+ *         file descriptor for the same directory.
  * @return
- *         The file descriptor on success, or -1 on failure.
+ *         On success, a file descriptor for the directory is returned.  On
+ *         failure, -1 is returned, and the directory remains open.
  */
-int bfs_freedir(struct bfs_dir *dir);
+int bfs_freedir(struct bfs_dir *dir, bool same_fd);
 
 #endif // BFS_DIR_H
diff --git a/src/ioq.c b/src/ioq.c
index e09c2a9..1b61fb0 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -155,6 +155,7 @@ static void *ioq_work(void *ptr) {
 		sanitize_uninit(cmd);
 
 		struct ioq_res *res = &cmd->res;
+		res->ptr = req.ptr;
 		res->dir = bfs_opendir(req.dfd, req.path);
 		res->error = errno;
 		ioqq_push(ioq->ready, cmd);
-- 
cgit v1.2.3


From 6719107e95a41b9ef241bdaa272eed505865710c Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Thu, 23 Mar 2023 13:59:48 -0400
Subject: dir: New bfs_polldir() function for directory readahead

---
 src/dir.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++----------------
 src/dir.h | 10 ++++++++
 src/ioq.c |  4 +++
 3 files changed, 80 insertions(+), 22 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/dir.c b/src/dir.c
index 126f473..b9fd74d 100644
--- a/src/dir.c
+++ b/src/dir.c
@@ -107,7 +107,10 @@ struct bfs_dir {
 	// sys_dirent buf[];
 #else
 	DIR *dir;
+	struct dirent *de;
 #endif
+
+	bool eof;
 };
 
 #if BFS_GETDENTS
@@ -152,8 +155,10 @@ struct bfs_dir *bfs_opendir(int at_fd, const char *at_path) {
 		free(dir);
 		return NULL;
 	}
+	dir->de = NULL;
 #endif
 
+	dir->eof = false;
 	return dir;
 }
 
@@ -165,38 +170,52 @@ int bfs_dirfd(const struct bfs_dir *dir) {
 #endif
 }
 
-/** Convert de->d_type to a bfs_type, if it exists. */
-static enum bfs_type bfs_d_type(const sys_dirent *de) {
-#ifdef DTTOIF
-	return bfs_mode_to_type(DTTOIF(de->d_type));
-#else
-	return BFS_UNKNOWN;
-#endif
-}
-
-/** Read a single directory entry. */
-static int bfs_getdent(struct bfs_dir *dir, const sys_dirent **de) {
+int bfs_polldir(struct bfs_dir *dir) {
 #if BFS_GETDENTS
+	if (dir->pos < dir->size) {
+		return 1;
+	} else if (dir->eof) {
+		return 0;
+	}
+
 	char *buf = (char *)(dir + 1);
+	ssize_t size = bfs_getdents(dir->fd, buf, BUF_SIZE);
+	if (size == 0) {
+		dir->eof = true;
+		return 0;
+	} else if (size < 0) {
+		return -1;
+	}
 
-	if (dir->pos >= dir->size) {
-		ssize_t ret = bfs_getdents(dir->fd, buf, BUF_SIZE);
-		if (ret <= 0) {
-			return ret;
+	dir->pos = 0;
+	dir->size = size;
+
+	// Like read(), getdents() doesn't indicate EOF until another call returns zero.
+	// Check that eagerly here to hopefully avoid a syscall in the last bfs_readdir().
+	size_t rest = BUF_SIZE - size;
+	if (rest >= sizeof(sys_dirent)) {
+		size = bfs_getdents(dir->fd, buf + size, rest);
+		if (size > 0) {
+			dir->size += size;
+		} else if (size == 0) {
+			dir->eof = true;
 		}
-		dir->pos = 0;
-		dir->size = ret;
 	}
 
-	*de = (void *)(buf + dir->pos);
-	dir->pos += (*de)->d_reclen;
 	return 1;
-#else
+#else // !BFS_GETDENTS
+	if (dir->de) {
+		return 1;
+	} else if (dir->eof) {
+		return 0;
+	}
+
 	errno = 0;
-	*de = readdir(dir->dir);
-	if (*de) {
+	dir->de = readdir(dir->dir);
+	if (dir->de) {
 		return 1;
 	} else if (errno == 0) {
+		dir->eof = true;
 		return 0;
 	} else {
 		return -1;
@@ -204,6 +223,22 @@ static int bfs_getdent(struct bfs_dir *dir, const sys_dirent **de) {
 #endif
 }
 
+/** Read a single directory entry. */
+static int bfs_getdent(struct bfs_dir *dir, const sys_dirent **de) {
+	int ret = bfs_polldir(dir);
+	if (ret > 0) {
+#if BFS_GETDENTS
+		char *buf = (char *)(dir + 1);
+		*de = (const sys_dirent *)(buf + dir->pos);
+		dir->pos += (*de)->d_reclen;
+#else
+		*de = dir->de;
+		dir->de = NULL;
+#endif
+	}
+	return ret;
+}
+
 /** Skip ".", "..", and deleted/empty dirents. */
 static bool skip_dirent(const sys_dirent *de) {
 #if __FreeBSD__
@@ -217,6 +252,15 @@ static bool skip_dirent(const sys_dirent *de) {
 	return name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'));
 }
 
+/** Convert de->d_type to a bfs_type, if it exists. */
+static enum bfs_type bfs_d_type(const sys_dirent *de) {
+#ifdef DTTOIF
+	return bfs_mode_to_type(DTTOIF(de->d_type));
+#else
+	return BFS_UNKNOWN;
+#endif
+}
+
 int bfs_readdir(struct bfs_dir *dir, struct bfs_dirent *de) {
 	while (true) {
 		const sys_dirent *sysde;
diff --git a/src/dir.h b/src/dir.h
index e0fe913..6fe7ae2 100644
--- a/src/dir.h
+++ b/src/dir.h
@@ -79,6 +79,16 @@ struct bfs_dir *bfs_opendir(int at_fd, const char *at_path);
  */
 int bfs_dirfd(const struct bfs_dir *dir);
 
+/**
+ * Performs any I/O necessary for the next bfs_readdir() call.
+ *
+ * @param dir
+ *         The directory to poll.
+ * @return
+ *         1 on success, 0 on EOF, or -1 on failure.
+ */
+int bfs_polldir(struct bfs_dir *dir);
+
 /**
  * Read a directory entry.
  *
diff --git a/src/ioq.c b/src/ioq.c
index 1b61fb0..74d2e09 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -158,6 +158,10 @@ static void *ioq_work(void *ptr) {
 		res->ptr = req.ptr;
 		res->dir = bfs_opendir(req.dfd, req.path);
 		res->error = errno;
+		if (res->dir) {
+			bfs_polldir(res->dir);
+		}
+
 		ioqq_push(ioq->ready, cmd);
 	}
 
-- 
cgit v1.2.3


From b3aa51d83650bf9f2d264e53110ca248453bb2f0 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Thu, 15 Jun 2023 12:52:47 -0400
Subject: ioq: Use a circular buffer

---
 src/ioq.c | 279 ++++++++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 201 insertions(+), 78 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 74d2e09..d3df7e0 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -2,14 +2,17 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "ioq.h"
+#include "atomic.h"
+#include "bfstd.h"
+#include "bit.h"
+#include "config.h"
+#include "diag.h"
 #include "dir.h"
-#include "list.h"
 #include "lock.h"
 #include "sanity.h"
 #include <assert.h>
 #include <errno.h>
 #include <pthread.h>
-#include <stdbool.h>
 #include <stdlib.h>
 
 /**
@@ -28,102 +31,225 @@ struct ioq_req {
 /**
  * An I/O queue command.
  */
-struct ioq_cmd {
-	union {
-		struct ioq_req req;
-		struct ioq_res res;
-	};
-
-	struct ioq_cmd *next;
+union ioq_cmd {
+	struct ioq_req req;
+	struct ioq_res res;
 };
 
 /**
- * An MPMC queue of I/O commands.
+ * A monitor for an I/O queue slot.
  */
-struct ioqq {
-	pthread_mutex_t mutex;
-	pthread_cond_t cond;
-
-	bool stop;
-
-	struct ioq_cmd *head;
-	struct ioq_cmd **tail;
+struct ioq_monitor {
+	cache_align pthread_mutex_t mutex;
+	pthread_cond_t full;
+	pthread_cond_t empty;
 };
 
-static struct ioqq *ioqq_create(void) {
-	struct ioqq *ioqq = malloc(sizeof(*ioqq));
-	if (!ioqq) {
+/** Initialize an ioq_monitor. */
+static int ioq_monitor_init(struct ioq_monitor *monitor) {
+	if (mutex_init(&monitor->mutex, NULL) != 0) {
 		goto fail;
 	}
 
-	if (mutex_init(&ioqq->mutex, NULL) != 0) {
-		goto fail_free;
+	if (cond_init(&monitor->full, NULL) != 0) {
+		goto fail_mutex;
 	}
 
-	if (cond_init(&ioqq->cond, NULL) != 0) {
-		goto fail_mutex;
+	if (cond_init(&monitor->empty, NULL) != 0) {
+		goto fail_full;
 	}
 
-	ioqq->stop = false;
-	SLIST_INIT(ioqq);
-	return ioqq;
+	return 0;
 
+fail_full:
+	cond_destroy(&monitor->full);
 fail_mutex:
-	mutex_destroy(&ioqq->mutex);
-fail_free:
-	free(ioqq);
+	mutex_destroy(&monitor->mutex);
 fail:
-	return NULL;
+	return -1;
 }
 
-/** Push a command onto the queue. */
-static void ioqq_push(struct ioqq *ioqq, struct ioq_cmd *cmd) {
-	mutex_lock(&ioqq->mutex);
-	SLIST_APPEND(ioqq, cmd);
-	mutex_unlock(&ioqq->mutex);
-	cond_signal(&ioqq->cond);
+/** Destroy an ioq_monitor. */
+static void ioq_monitor_destroy(struct ioq_monitor *monitor) {
+	cond_destroy(&monitor->empty);
+	cond_destroy(&monitor->full);
+	mutex_destroy(&monitor->mutex);
 }
 
-/** Pop a command from a queue. */
-static struct ioq_cmd *ioqq_pop(struct ioqq *ioqq) {
-	mutex_lock(&ioqq->mutex);
+/**
+ * A slot in an I/O queue.
+ */
+struct ioq_slot {
+	struct ioq_monitor *monitor;
+	union ioq_cmd *cmd;
+};
+
+/** Initialize an ioq_slot. */
+static void ioq_slot_init(struct ioq_slot *slot, struct ioq_monitor *monitor) {
+	slot->monitor = monitor;
+	slot->cmd = NULL;
+}
 
-	while (!ioqq->stop && !ioqq->head) {
-		cond_wait(&ioqq->cond, &ioqq->mutex);
+/** Push a command into a slot. */
+static void ioq_slot_push(struct ioq_slot *slot, union ioq_cmd *cmd) {
+	struct ioq_monitor *monitor = slot->monitor;
+
+	mutex_lock(&monitor->mutex);
+	while (slot->cmd) {
+		cond_wait(&monitor->empty, &monitor->mutex);
 	}
+	slot->cmd = cmd;
+	mutex_unlock(&monitor->mutex);
 
-	struct ioq_cmd *cmd = SLIST_POP(ioqq);
-	mutex_unlock(&ioqq->mutex);
-	return cmd;
+	cond_broadcast(&monitor->full);
 }
 
-/** Pop a command from a queue without blocking. */
-static struct ioq_cmd *ioqq_trypop(struct ioqq *ioqq) {
-	if (!mutex_trylock(&ioqq->mutex)) {
-		return NULL;
+/** Pop a command from a slot. */
+static union ioq_cmd *ioq_slot_pop(struct ioq_slot *slot) {
+	struct ioq_monitor *monitor = slot->monitor;
+
+	mutex_lock(&monitor->mutex);
+	while (!slot->cmd) {
+		cond_wait(&monitor->full, &monitor->mutex);
 	}
+	union ioq_cmd *ret = slot->cmd;
+	slot->cmd = NULL;
+	mutex_unlock(&monitor->mutex);
 
-	struct ioq_cmd *cmd = SLIST_POP(ioqq);
-	mutex_unlock(&ioqq->mutex);
-	return cmd;
+	cond_broadcast(&monitor->empty);
+
+	return ret;
 }
 
-/** Stop a queue, waking up any waiters. */
-static void ioqq_stop(struct ioqq *ioqq) {
-	mutex_lock(&ioqq->mutex);
-	ioqq->stop = true;
-	mutex_unlock(&ioqq->mutex);
-	cond_broadcast(&ioqq->cond);
+/** Pop a command from a slot, if one exists. */
+static union ioq_cmd *ioq_slot_trypop(struct ioq_slot *slot) {
+	struct ioq_monitor *monitor = slot->monitor;
+
+	if (!mutex_trylock(&monitor->mutex)) {
+		return NULL;
+	}
+
+	union ioq_cmd *ret = slot->cmd;
+	slot->cmd = NULL;
+
+	mutex_unlock(&monitor->mutex);
+
+	if (ret) {
+		cond_broadcast(&monitor->empty);
+	}
+	return ret;
 }
 
+/**
+ * An MPMC queue of I/O commands.
+ */
+struct ioqq {
+	/** Circular buffer index mask. */
+	size_t mask;
+
+	/** Number of monitors. */
+	size_t nmonitors;
+	/** Array of monitors used by the slots. */
+	struct ioq_monitor *monitors;
+
+	/** Index of next writer. */
+	cache_align atomic size_t head;
+	/** Index of next reader. */
+	cache_align atomic size_t tail;
+
+	/** The circular buffer itself. */
+	cache_align struct ioq_slot slots[];
+};
+
+// If we assign slots sequentially, threads will likely be operating on
+// consecutive slots.  If these slots are in the same cache line, that will
+// result in false sharing.  We can mitigate this by assigning slots with a
+// stride larger than a cache line e.g. 0, 9, 18, ..., 1, 10, 19, ...
+// As long as the stride is relatively prime to circular buffer length, we'll
+// still use every available slot.  Since the length is a power of two, that
+// means the stride must be odd.
+
+#define IOQ_STRIDE ((FALSE_SHARING_SIZE / sizeof(struct ioq_slot)) | 1)
+bfs_static_assert(IOQ_STRIDE % 2 == 1);
+
+/** Destroy an I/O command queue. */
 static void ioqq_destroy(struct ioqq *ioqq) {
-	if (ioqq) {
-		cond_destroy(&ioqq->cond);
-		mutex_destroy(&ioqq->mutex);
-		free(ioqq);
+	for (size_t i = 0; i < ioqq->nmonitors; ++i) {
+		ioq_monitor_destroy(&ioqq->monitors[i]);
+	}
+	free(ioqq->monitors);
+	free(ioqq);
+}
+
+/** Create an I/O command queue. */
+static struct ioqq *ioqq_create(size_t size) {
+	// Circular buffer size must be a power of two
+	size = bit_ceil(size);
+
+	struct ioqq *ioqq = xmemalign(alignof(struct ioqq), flex_sizeof(struct ioqq, slots, size));
+	if (!ioqq) {
+		return NULL;
+	}
+
+	// Use a pool of monitors
+	size_t nmonitors = size < 64 ? size : 64;
+	ioqq->nmonitors = 0;
+	ioqq->monitors = xmemalign(alignof(struct ioq_monitor), nmonitors * sizeof(struct ioq_monitor));
+	if (!ioqq->monitors) {
+		ioqq_destroy(ioqq);
+		return NULL;
+	}
+
+	for (size_t i = 0; i < nmonitors; ++i) {
+		if (ioq_monitor_init(&ioqq->monitors[i]) != 0) {
+			ioqq_destroy(ioqq);
+			return NULL;
+		}
+		++ioqq->nmonitors;
+	}
+
+	ioqq->mask = size - 1;
+
+	atomic_init(&ioqq->head, 0);
+	atomic_init(&ioqq->tail, 0);
+
+	for (size_t i = 0; i < size; ++i) {
+		ioq_slot_init(&ioqq->slots[i], &ioqq->monitors[i % nmonitors]);
+	}
+
+	return ioqq;
+}
+
+/** Push a command onto the queue. */
+static void ioqq_push(struct ioqq *ioqq, union ioq_cmd *cmd) {
+	size_t i = fetch_add(&ioqq->head, IOQ_STRIDE, relaxed);
+	ioq_slot_push(&ioqq->slots[i & ioqq->mask], cmd);
+}
+
+/** Pop a command from a queue. */
+static union ioq_cmd *ioqq_pop(struct ioqq *ioqq) {
+	size_t i = fetch_add(&ioqq->tail, IOQ_STRIDE, relaxed);
+	return ioq_slot_pop(&ioqq->slots[i & ioqq->mask]);
+}
+
+/** Pop a command from a queue if one is available. */
+static union ioq_cmd *ioqq_trypop(struct ioqq *ioqq) {
+	size_t i = load(&ioqq->tail, relaxed);
+	union ioq_cmd *cmd = ioq_slot_trypop(&ioqq->slots[i & ioqq->mask]);
+	if (cmd) {
+#ifdef NDEBUG
+		store(&ioqq->tail, i + IOQ_STRIDE, relaxed);
+#else
+		size_t j = fetch_add(&ioqq->tail, IOQ_STRIDE, relaxed);
+		bfs_assert(j == i, "ioqq_trypop() only supports a single consumer");
+#endif
 	}
+	return cmd;
 }
 
+/** Sentinel stop command. */
+static union ioq_cmd IOQ_STOP;
+
 struct ioq {
 	/** The depth of the queue. */
 	size_t depth;
@@ -146,8 +272,8 @@ static void *ioq_work(void *ptr) {
 	struct ioq *ioq = ptr;
 
 	while (true) {
-		struct ioq_cmd *cmd = ioqq_pop(ioq->pending);
-		if (!cmd) {
+		union ioq_cmd *cmd = ioqq_pop(ioq->pending);
+		if (cmd == &IOQ_STOP) {
 			break;
 		}
 
@@ -176,16 +302,17 @@ struct ioq *ioq_create(size_t depth, size_t threads) {
 
 	ioq->depth = depth;
 	ioq->size = 0;
+
 	ioq->pending = NULL;
 	ioq->ready = NULL;
 	ioq->nthreads = 0;
 
-	ioq->pending = ioqq_create();
+	ioq->pending = ioqq_create(depth);
 	if (!ioq->pending) {
 		goto fail;
 	}
 
-	ioq->ready = ioqq_create();
+	ioq->ready = ioqq_create(depth);
 	if (!ioq->ready) {
 		goto fail;
 	}
@@ -218,7 +345,7 @@ int ioq_opendir(struct ioq *ioq, int dfd, const char *path, void *ptr) {
 		return -1;
 	}
 
-	struct ioq_cmd *cmd = malloc(sizeof(*cmd));
+	union ioq_cmd *cmd = malloc(sizeof(*cmd));
 	if (!cmd) {
 		return -1;
 	}
@@ -228,8 +355,8 @@ int ioq_opendir(struct ioq *ioq, int dfd, const char *path, void *ptr) {
 	req->path = path;
 	req->ptr = ptr;
 
-	++ioq->size;
 	ioqq_push(ioq->pending, cmd);
+	++ioq->size;
 	return 0;
 }
 
@@ -238,11 +365,7 @@ struct ioq_res *ioq_pop(struct ioq *ioq) {
 		return NULL;
 	}
 
-	struct ioq_cmd *cmd = ioqq_pop(ioq->ready);
-	if (!cmd) {
-		return NULL;
-	}
-
+	union ioq_cmd *cmd = ioqq_pop(ioq->ready);
 	--ioq->size;
 	return &cmd->res;
 }
@@ -252,7 +375,7 @@ struct ioq_res *ioq_trypop(struct ioq *ioq) {
 		return NULL;
 	}
 
-	struct ioq_cmd *cmd = ioqq_trypop(ioq->ready);
+	union ioq_cmd *cmd = ioqq_trypop(ioq->ready);
 	if (!cmd) {
 		return NULL;
 	}
@@ -262,7 +385,7 @@ struct ioq_res *ioq_trypop(struct ioq *ioq) {
 }
 
 void ioq_free(struct ioq *ioq, struct ioq_res *res) {
-	struct ioq_cmd *cmd = (struct ioq_cmd *)res;
+	union ioq_cmd *cmd = (union ioq_cmd *)res;
 	free(cmd);
 }
 
@@ -271,8 +394,8 @@ void ioq_destroy(struct ioq *ioq) {
 		return;
 	}
 
-	if (ioq->pending) {
-		ioqq_stop(ioq->pending);
+	for (size_t i = 0; i < ioq->nthreads; ++i) {
+		ioqq_push(ioq->pending, &IOQ_STOP);
 	}
 
 	for (size_t i = 0; i < ioq->nthreads; ++i) {
-- 
cgit v1.2.3


From 425956c9022fda1e98544c4b2d495e91dfde4b4f Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Thu, 15 Jun 2023 14:22:00 -0400
Subject: ioq: Implement a non-blocking fast path

---
 src/ioq.c | 205 ++++++++++++++++++++++++++++++++------------------------------
 1 file changed, 107 insertions(+), 98 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index d3df7e0..5550c91 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -41,114 +41,38 @@ union ioq_cmd {
  */
 struct ioq_monitor {
 	cache_align pthread_mutex_t mutex;
-	pthread_cond_t full;
-	pthread_cond_t empty;
+	pthread_cond_t cond;
 };
 
 /** Initialize an ioq_monitor. */
 static int ioq_monitor_init(struct ioq_monitor *monitor) {
 	if (mutex_init(&monitor->mutex, NULL) != 0) {
-		goto fail;
-	}
-
-	if (cond_init(&monitor->full, NULL) != 0) {
-		goto fail_mutex;
+		return -1;
 	}
 
-	if (cond_init(&monitor->empty, NULL) != 0) {
-		goto fail_full;
+	if (cond_init(&monitor->cond, NULL) != 0) {
+		mutex_destroy(&monitor->mutex);
+		return -1;
 	}
 
 	return 0;
-
-fail_full:
-	cond_destroy(&monitor->full);
-fail_mutex:
-	mutex_destroy(&monitor->mutex);
-fail:
-	return -1;
 }
 
 /** Destroy an ioq_monitor. */
 static void ioq_monitor_destroy(struct ioq_monitor *monitor) {
-	cond_destroy(&monitor->empty);
-	cond_destroy(&monitor->full);
+	cond_destroy(&monitor->cond);
 	mutex_destroy(&monitor->mutex);
 }
 
-/**
- * A slot in an I/O queue.
- */
-struct ioq_slot {
-	struct ioq_monitor *monitor;
-	union ioq_cmd *cmd;
-};
-
-/** Initialize an ioq_slot. */
-static void ioq_slot_init(struct ioq_slot *slot, struct ioq_monitor *monitor) {
-	slot->monitor = monitor;
-	slot->cmd = NULL;
-}
-
-/** Push a command into a slot. */
-static void ioq_slot_push(struct ioq_slot *slot, union ioq_cmd *cmd) {
-	struct ioq_monitor *monitor = slot->monitor;
-
-	mutex_lock(&monitor->mutex);
-	while (slot->cmd) {
-		cond_wait(&monitor->empty, &monitor->mutex);
-	}
-	slot->cmd = cmd;
-	mutex_unlock(&monitor->mutex);
-
-	cond_broadcast(&monitor->full);
-}
-
-/** Pop a command from a slot. */
-static union ioq_cmd *ioq_slot_pop(struct ioq_slot *slot) {
-	struct ioq_monitor *monitor = slot->monitor;
-
-	mutex_lock(&monitor->mutex);
-	while (!slot->cmd) {
-		cond_wait(&monitor->full, &monitor->mutex);
-	}
-	union ioq_cmd *ret = slot->cmd;
-	slot->cmd = NULL;
-	mutex_unlock(&monitor->mutex);
-
-	cond_broadcast(&monitor->empty);
-
-	return ret;
-}
-
-/** Pop a command from a slot, if one exists. */
-static union ioq_cmd *ioq_slot_trypop(struct ioq_slot *slot) {
-	struct ioq_monitor *monitor = slot->monitor;
-
-	if (!mutex_trylock(&monitor->mutex)) {
-		return NULL;
-	}
-
-	union ioq_cmd *ret = slot->cmd;
-	slot->cmd = NULL;
-
-	mutex_unlock(&monitor->mutex);
-
-	if (ret) {
-		cond_broadcast(&monitor->empty);
-	}
-	return ret;
-}
-
 /**
  * An MPMC queue of I/O commands.
  */
 struct ioqq {
 	/** Circular buffer index mask. */
-	size_t mask;
+	size_t slot_mask;
 
-	/** Number of monitors. */
-	size_t nmonitors;
+	/** Monitor index mask. */
+	size_t monitor_mask;
 	/** Array of monitors used by the slots. */
 	struct ioq_monitor *monitors;
 
@@ -158,7 +82,7 @@ struct ioqq {
 	cache_align atomic size_t tail;
 
 	/** The circular buffer itself. */
-	cache_align struct ioq_slot slots[];
+	cache_align atomic uintptr_t slots[];
 };
 
 // If we assign slots sequentially, threads will likely be operating on
@@ -169,12 +93,16 @@ struct ioqq {
 // still use every available slot.  Since the length is a power of two, that
 // means the stride must be odd.
 
-#define IOQ_STRIDE ((FALSE_SHARING_SIZE / sizeof(struct ioq_slot)) | 1)
+#define IOQ_STRIDE ((FALSE_SHARING_SIZE / sizeof(atomic uintptr_t)) | 1)
 bfs_static_assert(IOQ_STRIDE % 2 == 1);
 
+/** Slot flag bit to indicate waiters. */
+#define IOQ_BLOCKED ((uintptr_t)1)
+bfs_static_assert(alignof(union ioq_cmd) > 1);
+
 /** Destroy an I/O command queue. */
 static void ioqq_destroy(struct ioqq *ioqq) {
-	for (size_t i = 0; i < ioqq->nmonitors; ++i) {
+	for (size_t i = 0; i < ioqq->monitor_mask + 1; ++i) {
 		ioq_monitor_destroy(&ioqq->monitors[i]);
 	}
 	free(ioqq->monitors);
@@ -191,9 +119,11 @@ static struct ioqq *ioqq_create(size_t size) {
 		return NULL;
 	}
 
+	ioqq->slot_mask = size - 1;
+	ioqq->monitor_mask = -1;
+
 	// Use a pool of monitors
 	size_t nmonitors = size < 64 ? size : 64;
-	ioqq->nmonitors = 0;
 	ioqq->monitors = xmemalign(alignof(struct ioq_monitor), nmonitors * sizeof(struct ioq_monitor));
 	if (!ioqq->monitors) {
 		ioqq_destroy(ioqq);
@@ -205,38 +135,116 @@ static struct ioqq *ioqq_create(size_t size) {
 			ioqq_destroy(ioqq);
 			return NULL;
 		}
-		++ioqq->nmonitors;
+		++ioqq->monitor_mask;
 	}
 
-	ioqq->mask = size - 1;
-
 	atomic_init(&ioqq->head, 0);
 	atomic_init(&ioqq->tail, 0);
 
 	for (size_t i = 0; i < size; ++i) {
-		ioq_slot_init(&ioqq->slots[i], &ioqq->monitors[i % nmonitors]);
+		atomic_init(&ioqq->slots[i], 0);
 	}
 
 	return ioqq;
 }
 
+/** Atomically wait for a slot to change. */
+static uintptr_t ioqq_wait(struct ioqq *ioqq, size_t i, uintptr_t value) {
+	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
+
+	struct ioq_monitor *monitor = &ioqq->monitors[i & ioqq->monitor_mask];
+	mutex_lock(&monitor->mutex);
+
+	uintptr_t ret;
+	while ((ret = load(slot, relaxed)) == value) {
+		// To avoid missed wakeups, it is important that
+		// cond_broadcast() is not called right here
+		cond_wait(&monitor->cond, &monitor->mutex);
+	}
+
+	mutex_unlock(&monitor->mutex);
+	return ret;
+}
+
+/** Wake up any threads waiting on a slot. */
+static void ioqq_wake(struct ioqq *ioqq, size_t i) {
+	struct ioq_monitor *monitor = &ioqq->monitors[i & ioqq->monitor_mask];
+
+	// The following implementation would clearly avoid the missed wakeup
+	// issue mentioned above in ioqq_wait():
+	//
+	//     mutex_lock(&monitor->mutex);
+	//     cond_broadcast(&monitor->cond);
+	//     mutex_unlock(&monitor->mutex);
+	//
+	// As a minor optimization, we move the broadcast outside of the lock.
+	// This optimization is correct, even though it leads to a seemingly-
+	// useless empty critical section.
+
+	mutex_lock(&monitor->mutex);
+	mutex_unlock(&monitor->mutex);
+	cond_broadcast(&monitor->cond);
+}
+
 /** Push a command onto the queue. */
 static void ioqq_push(struct ioqq *ioqq, union ioq_cmd *cmd) {
 	size_t i = fetch_add(&ioqq->head, IOQ_STRIDE, relaxed);
-	ioq_slot_push(&ioqq->slots[i & ioqq->mask], cmd);
+	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
+
+	uintptr_t addr = (uintptr_t)cmd;
+	bfs_assert(!(addr & IOQ_BLOCKED));
+
+	uintptr_t prev = load(slot, relaxed);
+	do {
+		while (prev & ~IOQ_BLOCKED) {
+			prev = fetch_or(slot, IOQ_BLOCKED, relaxed);
+			if (prev & ~IOQ_BLOCKED) {
+				prev = ioqq_wait(ioqq, i, prev | IOQ_BLOCKED);
+			}
+		}
+	} while (!compare_exchange_weak(slot, &prev, addr, release, relaxed));
+
+	if (prev & IOQ_BLOCKED) {
+		ioqq_wake(ioqq, i);
+	}
 }
 
 /** Pop a command from a queue. */
 static union ioq_cmd *ioqq_pop(struct ioqq *ioqq) {
 	size_t i = fetch_add(&ioqq->tail, IOQ_STRIDE, relaxed);
-	return ioq_slot_pop(&ioqq->slots[i & ioqq->mask]);
+	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
+
+	uintptr_t prev = load(slot, relaxed);
+	do {
+		while (!(prev & ~IOQ_BLOCKED)) {
+			prev = fetch_or(slot, IOQ_BLOCKED, relaxed);
+			if (!(prev & ~IOQ_BLOCKED)) {
+				prev = ioqq_wait(ioqq, i, IOQ_BLOCKED);
+			}
+		}
+	} while (!compare_exchange_weak(slot, &prev, 0, acquire, relaxed));
+
+	if (prev & IOQ_BLOCKED) {
+		ioqq_wake(ioqq, i);
+	}
+	prev &= ~IOQ_BLOCKED;
+
+	return (union ioq_cmd *)prev;
 }
 
 /** Pop a command from a queue if one is available. */
 static union ioq_cmd *ioqq_trypop(struct ioqq *ioqq) {
 	size_t i = load(&ioqq->tail, relaxed);
-	union ioq_cmd *cmd = ioq_slot_trypop(&ioqq->slots[i & ioqq->mask]);
-	if (cmd) {
+	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
+
+	uintptr_t prev = exchange(slot, 0, acquire);
+
+	if (prev & IOQ_BLOCKED) {
+		ioqq_wake(ioqq, i);
+	}
+	prev &= ~IOQ_BLOCKED;
+
+	if (prev) {
 #ifdef NDEBUG
 		store(&ioqq->tail, i + IOQ_STRIDE, relaxed);
 #else
@@ -244,7 +252,8 @@ static union ioq_cmd *ioqq_trypop(struct ioqq *ioqq) {
 		bfs_assert(j == i, "ioqq_trypop() only supports a single consumer");
 #endif
 	}
-	return cmd;
+
+	return (union ioq_cmd *)prev;
 }
 
 /** Sentinel stop command. */
-- 
cgit v1.2.3


From 90ded13e589b0089167ef25ca3d26be599dfec9b Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Mon, 19 Jun 2023 12:11:36 -0400
Subject: alloc: New header for memory allocation utilities

---
 Makefile      |   3 +-
 src/alloc.c   |  50 ++++++++++++++++++++
 src/alloc.h   | 149 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/bfstd.c   |  13 -----
 src/bfstd.h   |  12 -----
 src/bftw.c    |   5 +-
 src/color.c   |   5 +-
 src/config.h  |  50 --------------------
 src/ctx.c     |  31 ++----------
 src/dstring.c |   3 +-
 src/exec.c    |  26 ++++------
 src/ioq.c     |  26 ++++------
 src/ioq.h     |   4 +-
 src/main.c    |   1 +
 src/mtab.c    |   5 +-
 src/parse.c   |  28 ++---------
 src/pwcache.c |   5 +-
 src/trie.c    |   7 ++-
 src/xregex.c  |   3 +-
 src/xspawn.c  |   3 +-
 tests/alloc.c |  24 ++++++++++
 tests/bfstd.c |  13 +----
 22 files changed, 271 insertions(+), 195 deletions(-)
 create mode 100644 src/alloc.c
 create mode 100644 src/alloc.h
 create mode 100644 tests/alloc.c

(limited to 'src/ioq.c')

diff --git a/Makefile b/Makefile
index fb28e29..d38f581 100644
--- a/Makefile
+++ b/Makefile
@@ -217,6 +217,7 @@ $(OBJ)/FLAGS: $(OBJ)/FLAGS.new
 
 # All object files except the entry point
 LIBBFS := \
+    $(OBJ)/src/alloc.o \
     $(OBJ)/src/bar.o \
     $(OBJ)/src/bfstd.o \
     $(OBJ)/src/bftw.o \
@@ -246,7 +247,7 @@ LIBBFS := \
 $(BIN)/bfs: $(OBJ)/src/main.o $(LIBBFS)
 
 # Standalone unit tests
-UNITS := bfstd bit trie xtimegm
+UNITS := alloc bfstd bit trie xtimegm
 UNIT_TESTS := $(UNITS:%=$(BIN)/tests/%)
 UNIT_CHECKS := $(UNITS:%=check-%)
 
diff --git a/src/alloc.c b/src/alloc.c
new file mode 100644
index 0000000..0003108
--- /dev/null
+++ b/src/alloc.c
@@ -0,0 +1,50 @@
+// Copyright © Tavian Barnes <tavianator@tavianator.com>
+// SPDX-License-Identifier: 0BSD
+
+#include "alloc.h"
+#include "bit.h"
+#include "diag.h"
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+/** Portable aligned_alloc()/posix_memalign(). */
+static void *xmemalign(size_t align, size_t size) {
+	bfs_assert(has_single_bit(align));
+	bfs_assert(align >= sizeof(void *));
+	bfs_assert((size & (align - 1)) == 0);
+
+#if __APPLE__
+	void *ptr = NULL;
+	errno = posix_memalign(&ptr, align, size);
+	return ptr;
+#else
+	return aligned_alloc(align, size);
+#endif
+}
+
+void *alloc(size_t align, size_t size) {
+	bfs_assert(has_single_bit(align));
+	bfs_assert((size & (align - 1)) == 0);
+
+	if (align <= alignof(max_align_t)) {
+		return malloc(size);
+	} else {
+		return xmemalign(align, size);
+	}
+}
+
+void *zalloc(size_t align, size_t size) {
+	bfs_assert(has_single_bit(align));
+	bfs_assert((size & (align - 1)) == 0);
+
+	if (align <= alignof(max_align_t)) {
+		return calloc(1, size);
+	}
+
+	void *ret = xmemalign(align, size);
+	if (ret) {
+		memset(ret, 0, size);
+	}
+	return ret;
+}
diff --git a/src/alloc.h b/src/alloc.h
new file mode 100644
index 0000000..899a4ec
--- /dev/null
+++ b/src/alloc.h
@@ -0,0 +1,149 @@
+// Copyright © Tavian Barnes <tavianator@tavianator.com>
+// SPDX-License-Identifier: 0BSD
+
+/**
+ * Memory allocation.
+ */
+
+#ifndef BFS_ALLOC_H
+#define BFS_ALLOC_H
+
+#include "config.h"
+#include <stddef.h>
+
+/** Round down to a multiple of an alignment. */
+static inline size_t align_floor(size_t align, size_t size) {
+	return size & ~(align - 1);
+}
+
+/** Round up to a multiple of an alignment. */
+static inline size_t align_ceil(size_t align, size_t size) {
+	return align_floor(align, size + align - 1);
+}
+
+/**
+ * Saturating array size.
+ *
+ * @param align
+ *         Array element alignment.
+ * @param size
+ *         Array element size.
+ * @param count
+ *         Array element count.
+ * @return
+ *         size * count, saturating to the maximum aligned value on overflow.
+ */
+static inline size_t array_size(size_t align, size_t size, size_t count) {
+	size_t ret = size * count;
+	return ret / size == count ? ret : ~(align - 1);
+}
+
+/** Saturating array sizeof. */
+#define sizeof_array(type, count) \
+	array_size(alignof(type), sizeof(type), count)
+
+/** Size of a struct/union field. */
+#define sizeof_member(type, member) \
+	sizeof(((type *)NULL)->member)
+
+/**
+ * Saturating flexible struct size.
+ *
+ * @param align
+ *         Struct alignment.
+ * @param min
+ *         Minimum struct size.
+ * @param offset
+ *         Flexible array member offset.
+ * @param size
+ *         Flexible array element size.
+ * @param count
+ *         Flexible array element count.
+ * @return
+ *         The size of the struct with count flexible array elements.  Saturates
+ *         to the maximum aligned value on overflow.
+ */
+static inline size_t flex_size(size_t align, size_t min, size_t offset, size_t size, size_t count) {
+	size_t ret = size * count;
+	size_t overflow = ret / size != count;
+
+	size_t extra = offset + align - 1;
+	ret += extra;
+	overflow |= ret < extra;
+	ret |= -overflow;
+	ret = align_floor(align, ret);
+
+	// Make sure flex_sizeof(type, member, 0) >= sizeof(type), even if the
+	// type has more padding than necessary for alignment
+	if (min > align_ceil(align, offset)) {
+		ret = ret < min ? min : ret;
+	}
+
+	return ret;
+}
+
+/**
+ * Computes the size of a flexible struct.
+ *
+ * @param type
+ *         The type of the struct containing the flexible array.
+ * @param member
+ *         The name of the flexible array member.
+ * @param count
+ *         The length of the flexible array.
+ * @return
+ *         The size of the struct with count flexible array elements.  Saturates
+ *         to the maximum aligned value on overflow.
+ */
+#define sizeof_flex(type, member, count) \
+	flex_size(alignof(type), sizeof(type), offsetof(type, member), sizeof_member(type, member[0]), count)
+
+/**
+ * General memory allocator.
+ *
+ * @param align
+ *         The required alignment.
+ * @param size
+ *         The size of the allocation.
+ * @return
+ *         The allocated memory, or NULL on failure.
+ */
+void *alloc(size_t align, size_t size);
+
+/**
+ * Zero-initialized memory allocator.
+ *
+ * @param align
+ *         The required alignment.
+ * @param size
+ *         The size of the allocation.
+ * @return
+ *         The allocated memory, or NULL on failure.
+ */
+void *zalloc(size_t align, size_t size);
+
+/** Allocate memory for the given type. */
+#define ALLOC(type) \
+	(type *)alloc(alignof(type), sizeof(type))
+
+/** Allocate zeroed memory for the given type. */
+#define ZALLOC(type) \
+	(type *)zalloc(alignof(type), sizeof(type))
+
+/** Allocate memory for an array. */
+#define ALLOC_ARRAY(type, count) \
+	(type *)alloc(alignof(type), sizeof_array(type, count));
+
+/** Allocate zeroed memory for an array. */
+#define ZALLOC_ARRAY(type, count) \
+	(type *)zalloc(alignof(type), sizeof_array(type, count));
+
+/** Allocate memory for a flexible struct. */
+#define ALLOC_FLEX(type, member, count) \
+	(type *)alloc(alignof(type), sizeof_flex(type, member, count))
+
+/** Allocate zeroed memory for a flexible struct. */
+#define ZALLOC_FLEX(type, member, count) \
+	(type *)zalloc(alignof(type), sizeof_flex(type, member, count))
+
+#endif // BFS_ALLOC_H
diff --git a/src/bfstd.c b/src/bfstd.c
index 856c76c..0e8ba5f 100644
--- a/src/bfstd.c
+++ b/src/bfstd.c
@@ -143,19 +143,6 @@ char *xgetdelim(FILE *file, char delim) {
 	}
 }
 
-void *xmemalign(size_t align, size_t size) {
-	bfs_assert(has_single_bit(align));
-	bfs_assert((size & (align - 1)) == 0);
-
-#if __APPLE__
-	void *ptr = NULL;
-	errno = posix_memalign(&ptr, align, size);
-	return ptr;
-#else
-	return aligned_alloc(align, size);
-#endif
-}
-
 /** Compile and execute a regular expression for xrpmatch(). */
 static int xrpregex(nl_item item, const char *response) {
 	const char *pattern = nl_langinfo(item);
diff --git a/src/bfstd.h b/src/bfstd.h
index 6f2e21e..cafe28f 100644
--- a/src/bfstd.h
+++ b/src/bfstd.h
@@ -105,18 +105,6 @@ char *xgetdelim(FILE *file, char delim);
 
 // #include <stdlib.h>
 
-/**
- * Portable version of aligned_alloc()/posix_memalign().
- *
- * @param align
- *         The allocation's alignment.
- * @param size
- *         The allocation's size.
- * @return
- *         The allocation, or NULL on failure.
- */
-void *xmemalign(size_t align, size_t size);
-
 /**
  * Process a yes/no prompt.
  *
diff --git a/src/bftw.c b/src/bftw.c
index e711963..7ab14c7 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -17,6 +17,7 @@
  */
 
 #include "bftw.h"
+#include "alloc.h"
 #include "bfstd.h"
 #include "config.h"
 #include "diag.h"
@@ -241,9 +242,7 @@ static void bftw_cache_destroy(struct bftw_cache *cache) {
 /** Create a new bftw_file. */
 static struct bftw_file *bftw_file_new(struct bftw_file *parent, const char *name) {
 	size_t namelen = strlen(name);
-	size_t size = flex_sizeof(struct bftw_file, name, namelen + 1);
-
-	struct bftw_file *file = malloc(size);
+	struct bftw_file *file = ALLOC_FLEX(struct bftw_file, name, namelen + 1);
 	if (!file) {
 		return NULL;
 	}
diff --git a/src/color.c b/src/color.c
index 1edd8b5..b54ad53 100644
--- a/src/color.c
+++ b/src/color.c
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "color.h"
+#include "alloc.h"
 #include "bfstd.h"
 #include "bftw.h"
 #include "config.h"
@@ -404,7 +405,7 @@ static void parse_gnu_ls_colors(struct colors *colors, const char *ls_colors) {
 }
 
 struct colors *parse_colors(void) {
-	struct colors *colors = malloc(sizeof(struct colors));
+	struct colors *colors = ALLOC(struct colors);
 	if (!colors) {
 		return NULL;
 	}
@@ -497,7 +498,7 @@ void free_colors(struct colors *colors) {
 }
 
 CFILE *cfwrap(FILE *file, const struct colors *colors, bool close) {
-	CFILE *cfile = malloc(sizeof(*cfile));
+	CFILE *cfile = ALLOC(CFILE);
 	if (!cfile) {
 		return NULL;
 	}
diff --git a/src/config.h b/src/config.h
index 73348ac..1671a0d 100644
--- a/src/config.h
+++ b/src/config.h
@@ -141,56 +141,6 @@
  */
 #define countof(array) (sizeof(array) / sizeof(0[array]))
 
-/**
- * Round down to a multiple of an alignment.
- */
-static inline size_t align_floor(size_t align, size_t size) {
-	return size & ~(align - 1);
-}
-
-/**
- * Round up to a multiple of an alignment.
- */
-static inline size_t align_ceil(size_t align, size_t size) {
-	return align_floor(align, size + align - 1);
-}
-
-/**
- * Computes the size of a struct containing a flexible array member of the given
- * length.
- *
- * @param type
- *         The type of the struct containing the flexible array.
- * @param member
- *         The name of the flexible array member.
- * @param count
- *         The length of the flexible array.
- */
-#define flex_sizeof(type, member, count) \
-	flex_sizeof_impl(alignof(type), sizeof(type), offsetof(type, member), sizeof(((type *)NULL)->member[0]), count)
-
-static inline size_t flex_sizeof_impl(size_t align, size_t min, size_t offset, size_t size, size_t count) {
-	size_t ret = size * count;
-	size_t overflow = ret / size != count;
-
-	ret += offset;
-	overflow |= ret < offset;
-
-	size_t mask = align - 1;
-	ret += mask;
-	overflow |= ret < mask;
-	ret |= -overflow;
-	ret &= ~mask;
-
-	// Make sure flex_sizeof(type, member, 0) >= sizeof(type), even if the
-	// type has more padding than necessary for alignment
-	if (min > align_ceil(align, offset) && ret < min) {
-		ret = min;
-	}
-
-	return ret;
-}
-
 /**
  * False sharing/destructive interference/largest cache line size.
  */
diff --git a/src/ctx.c b/src/ctx.c
index e8ce0e8..a940bed 100644
--- a/src/ctx.c
+++ b/src/ctx.c
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "ctx.h"
+#include "alloc.h"
 #include "color.h"
 #include "darray.h"
 #include "diag.h"
@@ -42,43 +43,17 @@ const char *debug_flag_name(enum debug_flags flag) {
 }
 
 struct bfs_ctx *bfs_ctx_new(void) {
-	struct bfs_ctx *ctx = malloc(sizeof(*ctx));
+	struct bfs_ctx *ctx = ZALLOC(struct bfs_ctx);
 	if (!ctx) {
 		return NULL;
 	}
 
-	ctx->argv = NULL;
-	ctx->paths = NULL;
-	ctx->expr = NULL;
-	ctx->exclude = NULL;
-
-	ctx->mindepth = 0;
 	ctx->maxdepth = INT_MAX;
 	ctx->flags = BFTW_RECOVER;
 	ctx->strategy = BFTW_BFS;
-	ctx->threads = 0;
 	ctx->optlevel = 3;
-	ctx->debug = 0;
-	ctx->ignore_races = false;
-	ctx->posixly_correct = false;
-	ctx->status = false;
-	ctx->unique = false;
-	ctx->warn = false;
-	ctx->xargs_safe = false;
-
-	ctx->colors = NULL;
-	ctx->colors_error = 0;
-	ctx->cout = NULL;
-	ctx->cerr = NULL;
-
-	ctx->users = NULL;
-	ctx->groups = NULL;
-
-	ctx->mtab = NULL;
-	ctx->mtab_error = 0;
 
 	trie_init(&ctx->files);
-	ctx->nfiles = 0;
 
 	struct rlimit rl;
 	if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
@@ -155,7 +130,7 @@ CFILE *bfs_ctx_dedup(struct bfs_ctx *ctx, CFILE *cfile, const char *path) {
 		return ctx_file->cfile;
 	}
 
-	leaf->value = ctx_file = malloc(sizeof(*ctx_file));
+	leaf->value = ctx_file = ALLOC(struct bfs_ctx_file);
 	if (!ctx_file) {
 		trie_remove(&ctx->files, leaf);
 		return NULL;
diff --git a/src/dstring.c b/src/dstring.c
index 2c9869d..7ca74d0 100644
--- a/src/dstring.c
+++ b/src/dstring.c
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "dstring.h"
+#include "alloc.h"
 #include "diag.h"
 #include <stdarg.h>
 #include <stdio.h>
@@ -24,7 +25,7 @@ static struct dstring *dstrheader(const char *dstr) {
 
 /** Get the correct size for a dstring with the given capacity. */
 static size_t dstrsize(size_t capacity) {
-	return flex_sizeof(struct dstring, data, capacity + 1);
+	return sizeof_flex(struct dstring, data, capacity + 1);
 }
 
 /** Allocate a dstring with the given contents. */
diff --git a/src/exec.c b/src/exec.c
index 5912ad6..ea7f897 100644
--- a/src/exec.c
+++ b/src/exec.c
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "exec.h"
+#include "alloc.h"
 #include "bfstd.h"
 #include "bftw.h"
 #include "ctx.h"
@@ -124,26 +125,16 @@ static void bfs_exec_parse_error(const struct bfs_ctx *ctx, const struct bfs_exe
 }
 
 struct bfs_exec *bfs_exec_parse(const struct bfs_ctx *ctx, char **argv, enum bfs_exec_flags flags) {
-	struct bfs_exec *execbuf = malloc(sizeof(*execbuf));
+	struct bfs_exec *execbuf = ZALLOC(struct bfs_exec);
 	if (!execbuf) {
-		bfs_perror(ctx, "malloc()");
+		bfs_perror(ctx, "zalloc()");
 		goto fail;
 	}
 
 	execbuf->flags = flags;
 	execbuf->ctx = ctx;
 	execbuf->tmpl_argv = argv + 1;
-	execbuf->tmpl_argc = 0;
-	execbuf->argv = NULL;
-	execbuf->argc = 0;
-	execbuf->argv_cap = 0;
-	execbuf->arg_size = 0;
-	execbuf->arg_max = 0;
-	execbuf->arg_min = 0;
 	execbuf->wd_fd = -1;
-	execbuf->wd_path = NULL;
-	execbuf->wd_len = 0;
-	execbuf->ret = 0;
 
 	while (true) {
 		const char *arg = execbuf->tmpl_argv[execbuf->tmpl_argc];
@@ -176,9 +167,9 @@ struct bfs_exec *bfs_exec_parse(const struct bfs_ctx *ctx, char **argv, enum bfs
 	}
 
 	execbuf->argv_cap = execbuf->tmpl_argc + 1;
-	execbuf->argv = malloc(execbuf->argv_cap*sizeof(*execbuf->argv));
+	execbuf->argv = ALLOC_ARRAY(char *, execbuf->argv_cap);
 	if (!execbuf->argv) {
-		bfs_perror(ctx, "malloc()");
+		bfs_perror(ctx, "alloc()");
 		goto fail;
 	}
 
@@ -224,9 +215,8 @@ static char *bfs_exec_format_path(const struct bfs_exec *execbuf, const struct B
 		return NULL;
 	}
 
-	strcpy(path, "./");
-	strcpy(path + 2, name);
-
+	char *cur = stpcpy(path, "./");
+	cur = stpcpy(cur, name);
 	return path;
 }
 
@@ -612,7 +602,7 @@ static int bfs_exec_push(struct bfs_exec *execbuf, char *arg) {
 
 	if (execbuf->argc + 1 >= execbuf->argv_cap) {
 		size_t cap = 2*execbuf->argv_cap;
-		char **argv = realloc(execbuf->argv, cap*sizeof(*argv));
+		char **argv = realloc(execbuf->argv, sizeof_array(char *, cap));
 		if (!argv) {
 			return -1;
 		}
diff --git a/src/ioq.c b/src/ioq.c
index 5550c91..3e304ce 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "ioq.h"
+#include "alloc.h"
 #include "atomic.h"
 #include "bfstd.h"
 #include "bit.h"
@@ -114,7 +115,7 @@ static struct ioqq *ioqq_create(size_t size) {
 	// Circular buffer size must be a power of two
 	size = bit_ceil(size);
 
-	struct ioqq *ioqq = xmemalign(alignof(struct ioqq), flex_sizeof(struct ioqq, slots, size));
+	struct ioqq *ioqq = ALLOC_FLEX(struct ioqq, slots, size);
 	if (!ioqq) {
 		return NULL;
 	}
@@ -124,7 +125,7 @@ static struct ioqq *ioqq_create(size_t size) {
 
 	// Use a pool of monitors
 	size_t nmonitors = size < 64 ? size : 64;
-	ioqq->monitors = xmemalign(alignof(struct ioq_monitor), nmonitors * sizeof(struct ioq_monitor));
+	ioqq->monitors = ALLOC_ARRAY(struct ioq_monitor, nmonitors);
 	if (!ioqq->monitors) {
 		ioqq_destroy(ioqq);
 		return NULL;
@@ -273,7 +274,7 @@ struct ioq {
 	/** The number of background threads. */
 	size_t nthreads;
 	/** The background threads themselves. */
-	pthread_t *threads;
+	pthread_t threads[];
 };
 
 /** Background thread entry point. */
@@ -303,18 +304,13 @@ static void *ioq_work(void *ptr) {
 	return NULL;
 }
 
-struct ioq *ioq_create(size_t depth, size_t threads) {
-	struct ioq *ioq = malloc(sizeof(*ioq));
+struct ioq *ioq_create(size_t depth, size_t nthreads) {
+	struct ioq *ioq = ZALLOC_FLEX(struct ioq, threads, nthreads);
 	if (!ioq) {
 		goto fail;
 	}
 
 	ioq->depth = depth;
-	ioq->size = 0;
-
-	ioq->pending = NULL;
-	ioq->ready = NULL;
-	ioq->nthreads = 0;
 
 	ioq->pending = ioqq_create(depth);
 	if (!ioq->pending) {
@@ -326,12 +322,7 @@ struct ioq *ioq_create(size_t depth, size_t threads) {
 		goto fail;
 	}
 
-	ioq->threads = malloc(threads * sizeof(ioq->threads[0]));
-	if (!ioq->threads) {
-		goto fail;
-	}
-
-	for (size_t i = 0; i < threads; ++i) {
+	for (size_t i = 0; i < nthreads; ++i) {
 		errno = pthread_create(&ioq->threads[i], NULL, ioq_work, ioq);
 		if (errno != 0) {
 			goto fail;
@@ -354,7 +345,7 @@ int ioq_opendir(struct ioq *ioq, int dfd, const char *path, void *ptr) {
 		return -1;
 	}
 
-	union ioq_cmd *cmd = malloc(sizeof(*cmd));
+	union ioq_cmd *cmd = ALLOC(union ioq_cmd);
 	if (!cmd) {
 		return -1;
 	}
@@ -412,7 +403,6 @@ void ioq_destroy(struct ioq *ioq) {
 			abort();
 		}
 	}
-	free(ioq->threads);
 
 	ioqq_destroy(ioq->ready);
 	ioqq_destroy(ioq->pending);
diff --git a/src/ioq.h b/src/ioq.h
index 9492034..0af5779 100644
--- a/src/ioq.h
+++ b/src/ioq.h
@@ -33,12 +33,12 @@ struct ioq_res {
  *
  * @param depth
  *         The maximum depth of the queue.
- * @param threads
+ * @param nthreads
  *         The maximum number of background threads.
  * @return
  *         The new I/O queue, or NULL on failure.
  */
-struct ioq *ioq_create(size_t depth, size_t threads);
+struct ioq *ioq_create(size_t depth, size_t nthreads);
 
 /**
  * Asynchronous bfs_opendir().
diff --git a/src/main.c b/src/main.c
index 76dde86..b7a08c1 100644
--- a/src/main.c
+++ b/src/main.c
@@ -20,6 +20,7 @@
  *     - bftw.[ch]     (an extended version of nftw(3))
  *
  * - Utilities:
+ *     - alloc.[ch]    (memory allocation)
  *     - atomic.h      (atomic operations)
  *     - bar.[ch]      (a terminal status bar)
  *     - bit.h         (bit manipulation)
diff --git a/src/mtab.c b/src/mtab.c
index 1d1ad94..e5c25ba 100644
--- a/src/mtab.c
+++ b/src/mtab.c
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "mtab.h"
+#include "alloc.h"
 #include "bfstd.h"
 #include "config.h"
 #include "darray.h"
@@ -87,15 +88,13 @@ fail:
 }
 
 struct bfs_mtab *bfs_mtab_parse(void) {
-	struct bfs_mtab *mtab = malloc(sizeof(*mtab));
+	struct bfs_mtab *mtab = ZALLOC(struct bfs_mtab);
 	if (!mtab) {
 		return NULL;
 	}
 
-	mtab->entries = NULL;
 	trie_init(&mtab->names);
 	trie_init(&mtab->types);
-	mtab->types_filled = false;
 
 	int error = 0;
 
diff --git a/src/parse.c b/src/parse.c
index 64e08cd..cf4f696 100644
--- a/src/parse.c
+++ b/src/parse.c
@@ -9,6 +9,7 @@
  */
 
 #include "parse.h"
+#include "alloc.h"
 #include "bfstd.h"
 #include "bftw.h"
 #include "color.h"
@@ -55,39 +56,16 @@ static char *fake_print_arg = "-print";
 static char *fake_true_arg = "-true";
 
 struct bfs_expr *bfs_expr_new(bfs_eval_fn *eval_fn, size_t argc, char **argv) {
-	struct bfs_expr *expr = malloc(sizeof(*expr));
+	struct bfs_expr *expr = ZALLOC(struct bfs_expr);
 	if (!expr) {
-		perror("malloc()");
+		perror("zalloc()");
 		return NULL;
 	}
 
 	expr->eval_fn = eval_fn;
 	expr->argc = argc;
 	expr->argv = argv;
-	expr->persistent_fds = 0;
-	expr->ephemeral_fds = 0;
-	expr->pure = false;
-	expr->always_true = false;
-	expr->always_false = false;
-	expr->cost = 0.0;
 	expr->probability = 0.5;
-	expr->evaluations = 0;
-	expr->successes = 0;
-	expr->elapsed.tv_sec = 0;
-	expr->elapsed.tv_nsec = 0;
-
-	// Prevent bfs_expr_free() from freeing uninitialized pointers on error paths
-	if (bfs_expr_is_parent(expr)) {
-		expr->lhs = NULL;
-		expr->rhs = NULL;
-	} else if (eval_fn == eval_exec) {
-		expr->exec = NULL;
-	} else if (eval_fn == eval_fprintf) {
-		expr->printf = NULL;
-	} else if (eval_fn == eval_regex) {
-		expr->regex = NULL;
-	}
-
 	return expr;
 }
 
diff --git a/src/pwcache.c b/src/pwcache.c
index f52e4e1..9f32eb0 100644
--- a/src/pwcache.c
+++ b/src/pwcache.c
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "pwcache.h"
+#include "alloc.h"
 #include "config.h"
 #include "darray.h"
 #include "trie.h"
@@ -71,7 +72,7 @@ struct bfs_users {
 };
 
 struct bfs_users *bfs_users_new(void) {
-	struct bfs_users *users = malloc(sizeof(*users));
+	struct bfs_users *users = ALLOC(struct bfs_users);
 	if (!users) {
 		return NULL;
 	}
@@ -144,7 +145,7 @@ struct bfs_groups {
 };
 
 struct bfs_groups *bfs_groups_new(void) {
-	struct bfs_groups *groups = malloc(sizeof(*groups));
+	struct bfs_groups *groups = ALLOC(struct bfs_groups);
 	if (!groups) {
 		return NULL;
 	}
diff --git a/src/trie.c b/src/trie.c
index 8543eb1..19423cf 100644
--- a/src/trie.c
+++ b/src/trie.c
@@ -82,6 +82,7 @@
  */
 
 #include "trie.h"
+#include "alloc.h"
 #include "bit.h"
 #include "config.h"
 #include "diag.h"
@@ -317,7 +318,7 @@ struct trie_leaf *trie_find_prefix(const struct trie *trie, const char *key) {
 
 /** Create a new leaf, holding a copy of the given key. */
 static struct trie_leaf *trie_leaf_alloc(struct trie *trie, const void *key, size_t length) {
-	struct trie_leaf *leaf = malloc(flex_sizeof(struct trie_leaf, key, length));
+	struct trie_leaf *leaf = ALLOC_FLEX(struct trie_leaf, key, length);
 	if (!leaf) {
 		return NULL;
 	}
@@ -339,12 +340,10 @@ static void trie_leaf_free(struct trie *trie, struct trie_leaf *leaf) {
 
 /** Compute the size of a trie node with a certain number of children. */
 static size_t trie_node_size(unsigned int size) {
-	// Empty nodes aren't supported
-	bfs_assert(size > 0);
 	// Node size must be a power of two
 	bfs_assert(has_single_bit(size));
 
-	return flex_sizeof(struct trie_node, children, size);
+	return sizeof_flex(struct trie_node, children, size);
 }
 
 #if ENDIAN_NATIVE == ENDIAN_LITTLE
diff --git a/src/xregex.c b/src/xregex.c
index 89c2e90..ab5f793 100644
--- a/src/xregex.c
+++ b/src/xregex.c
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "xregex.h"
+#include "alloc.h"
 #include "config.h"
 #include "diag.h"
 #include "sanity.h"
@@ -115,7 +116,7 @@ static int bfs_onig_initialize(OnigEncoding *enc) {
 #endif
 
 int bfs_regcomp(struct bfs_regex **preg, const char *pattern, enum bfs_regex_type type, enum bfs_regcomp_flags flags) {
-	struct bfs_regex *regex = *preg = malloc(sizeof(*regex));
+	struct bfs_regex *regex = *preg = ALLOC(struct bfs_regex);
 	if (!regex) {
 		return -1;
 	}
diff --git a/src/xspawn.c b/src/xspawn.c
index 740e38e..2cabdcc 100644
--- a/src/xspawn.c
+++ b/src/xspawn.c
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "xspawn.h"
+#include "alloc.h"
 #include "bfstd.h"
 #include "config.h"
 #include "list.h"
@@ -62,7 +63,7 @@ int bfs_spawn_setflags(struct bfs_spawn *ctx, enum bfs_spawn_flags flags) {
 
 /** Add a spawn action to the chain. */
 static struct bfs_spawn_action *bfs_spawn_add(struct bfs_spawn *ctx, enum bfs_spawn_op op) {
-	struct bfs_spawn_action *action = malloc(sizeof(*action));
+	struct bfs_spawn_action *action = ALLOC(struct bfs_spawn_action);
 	if (!action) {
 		return NULL;
 	}
diff --git a/tests/alloc.c b/tests/alloc.c
new file mode 100644
index 0000000..91b1b43
--- /dev/null
+++ b/tests/alloc.c
@@ -0,0 +1,24 @@
+// Copyright © Tavian Barnes <tavianator@tavianator.com>
+// SPDX-License-Identifier: 0BSD
+
+#include "../src/alloc.h"
+#include "../src/diag.h"
+#include <stdlib.h>
+
+int main(void) {
+	// Check sizeof_flex()
+	struct flexible {
+		alignas(64) int foo;
+		int bar[];
+	};
+	bfs_verify(sizeof_flex(struct flexible, bar, 0) >= sizeof(struct flexible));
+	bfs_verify(sizeof_flex(struct flexible, bar, 16) % alignof(struct flexible) == 0);
+	bfs_verify(sizeof_flex(struct flexible, bar, SIZE_MAX / sizeof(int) + 1)
+	           == align_floor(alignof(struct flexible), SIZE_MAX));
+
+	// Corner case: sizeof(type) > align_ceil(alignof(type), offsetof(type, member))
+	// Doesn't happen in typical ABIs
+	bfs_verify(flex_size(8, 16, 4, 4, 1) == 16);
+
+	return EXIT_SUCCESS;
+}
diff --git a/tests/bfstd.c b/tests/bfstd.c
index 7fea9b5..fa854a8 100644
--- a/tests/bfstd.c
+++ b/tests/bfstd.c
@@ -24,17 +24,6 @@ static void check_base_dir(const char *path, const char *dir, const char *base)
 }
 
 int main(void) {
-	// Check flex_sizeof()
-	struct flexible {
-		alignas(64) int foo;
-		int bar[];
-	};
-	bfs_verify(flex_sizeof(struct flexible, bar, 0) >= sizeof(struct flexible));
-	bfs_verify(flex_sizeof(struct flexible, bar, 16) % alignof(struct flexible) == 0);
-	bfs_verify(flex_sizeof(struct flexible, bar, SIZE_MAX / sizeof(int) + 1)
-	           == align_floor(alignof(struct flexible), SIZE_MAX));
-	bfs_verify(flex_sizeof_impl(8, 16, 4, 4, 1) == 16);
-
 	// From man 3p basename
 	check_base_dir("usr", ".", "usr");
 	check_base_dir("usr/", ".", "usr");
@@ -46,4 +35,6 @@ int main(void) {
 	check_base_dir("/usr/lib", "/usr", "lib");
 	check_base_dir("//usr//lib//", "//usr", "lib");
 	check_base_dir("/home//dwc//test", "/home//dwc", "test");
+
+	return EXIT_SUCCESS;
 }
-- 
cgit v1.2.3


From 4889f3ebb59c926b8e53a2e12edd5009d7cd4cbe Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Mon, 19 Jun 2023 13:46:54 -0400
Subject: ioq: Arena-allocate ioq_cmd

---
 src/ioq.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 3e304ce..33316fa 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -266,6 +266,9 @@ struct ioq {
 	/** The current size of the queue. */
 	size_t size;
 
+	/** ioq_cmd command arena. */
+	struct arena cmds;
+
 	/** Pending I/O requests. */
 	struct ioqq *pending;
 	/** Ready I/O responses. */
@@ -311,6 +314,7 @@ struct ioq *ioq_create(size_t depth, size_t nthreads) {
 	}
 
 	ioq->depth = depth;
+	ARENA_INIT(&ioq->cmds, union ioq_cmd);
 
 	ioq->pending = ioqq_create(depth);
 	if (!ioq->pending) {
@@ -345,7 +349,7 @@ int ioq_opendir(struct ioq *ioq, int dfd, const char *path, void *ptr) {
 		return -1;
 	}
 
-	union ioq_cmd *cmd = ALLOC(union ioq_cmd);
+	union ioq_cmd *cmd = arena_alloc(&ioq->cmds);
 	if (!cmd) {
 		return -1;
 	}
@@ -385,8 +389,7 @@ struct ioq_res *ioq_trypop(struct ioq *ioq) {
 }
 
 void ioq_free(struct ioq *ioq, struct ioq_res *res) {
-	union ioq_cmd *cmd = (union ioq_cmd *)res;
-	free(cmd);
+	arena_free(&ioq->cmds, (union ioq_cmd *)res);
 }
 
 void ioq_destroy(struct ioq *ioq) {
@@ -407,5 +410,7 @@ void ioq_destroy(struct ioq *ioq) {
 	ioqq_destroy(ioq->ready);
 	ioqq_destroy(ioq->pending);
 
+	arena_destroy(&ioq->cmds);
+
 	free(ioq);
 }
-- 
cgit v1.2.3


From a1490d98a1aebb3bfbd3873613977d0341ec7f98 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Tue, 20 Jun 2023 12:02:08 -0400
Subject: dir: Arena-allocate directories

---
 src/bftw.c | 37 +++++++++++++++++++++++++++++--------
 src/dir.c  | 44 +++++++++++++++++++++++---------------------
 src/dir.h  | 27 +++++++++++++++++++++++----
 src/eval.c | 34 ++++++++++++++++++++++------------
 src/ioq.c  | 15 ++++++++++-----
 src/ioq.h  |  4 +++-
 6 files changed, 110 insertions(+), 51 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/bftw.c b/src/bftw.c
index d0491e0..69e41a2 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -103,6 +103,8 @@ struct bftw_cache {
 
 	/** bftw_file arena. */
 	struct varena files;
+	/** bfs_dir arena. */
+	struct arena dirs;
 };
 
 /** Initialize a cache. */
@@ -111,6 +113,7 @@ static void bftw_cache_init(struct bftw_cache *cache, size_t capacity) {
 	cache->target = NULL;
 	cache->capacity = capacity;
 	VARENA_INIT(&cache->files, struct bftw_file, name);
+	bfs_dir_arena(&cache->dirs);
 }
 
 /** Remove a bftw_file from the LRU list. */
@@ -136,6 +139,7 @@ static void bftw_file_close(struct bftw_cache *cache, struct bftw_file *file) {
 	if (file->dir) {
 		bfs_assert(file->fd == bfs_dirfd(file->dir));
 		bfs_closedir(file->dir);
+		arena_free(&cache->dirs, file->dir);
 		file->dir = NULL;
 	} else {
 		xclose(file->fd);
@@ -157,9 +161,10 @@ static void bftw_file_freedir(struct bftw_cache *cache, struct bftw_file *file)
 	bool pinned = file->pincount > 0;
 
 	if (reffed || pinned) {
-		int fd = bfs_freedir(file->dir, pinned);
+		int fd = bfs_fdclosedir(file->dir, pinned);
 		if (fd >= 0) {
 			file->fd = fd;
+			arena_free(&cache->dirs, file->dir);
 			file->dir = NULL;
 		}
 	} else {
@@ -243,6 +248,7 @@ static void bftw_cache_destroy(struct bftw_cache *cache) {
 	bfs_assert(!cache->target);
 
 	varena_destroy(&cache->files);
+	arena_destroy(&cache->dirs);
 }
 
 /** Create a new bftw_file. */
@@ -410,10 +416,17 @@ static struct bfs_dir *bftw_file_opendir(struct bftw_cache *cache, struct bftw_f
 		return NULL;
 	}
 
-	struct bfs_dir *dir = bfs_opendir(fd, NULL);
-	if (dir) {
-		bftw_file_set_dir(cache, file, dir);
+	struct bfs_dir *dir = arena_alloc(&cache->dirs);
+	if (!dir) {
+		return NULL;
+	}
+
+	if (bfs_opendir(dir, fd, NULL) != 0) {
+		arena_free(&cache->dirs, dir);
+		return NULL;
 	}
+
+	bftw_file_set_dir(cache, file, dir);
 	return dir;
 }
 
@@ -900,14 +913,18 @@ static void bftw_push_dir(struct bftw_state *state, struct bftw_file *file) {
 			goto unpin;
 		}
 	}
-	--cache->capacity;
 
-	if (ioq_opendir(state->ioq, dfd, file->name, file) != 0) {
-		++cache->capacity;
+	struct bfs_dir *dir = arena_alloc(&state->cache.dirs);
+	if (!dir) {
 		goto unpin;
 	}
 
+	if (ioq_opendir(state->ioq, dir, dfd, file->name, file) != 0) {
+		goto free;
+	}
+
 	file->ioqueued = true;
+	--cache->capacity;
 
 	if (state->flags & BFTW_SORT) {
 		goto append;
@@ -915,6 +932,8 @@ static void bftw_push_dir(struct bftw_state *state, struct bftw_file *file) {
 		return;
 	}
 
+free:
+	arena_free(&state->cache.dirs, dir);
 unpin:
 	if (file->parent) {
 		bftw_cache_unpin(cache, file->parent);
@@ -950,7 +969,9 @@ static int bftw_ioq_pop(struct bftw_state *state, bool block) {
 		bftw_cache_unpin(cache, file->parent);
 	}
 
-	if (res->dir) {
+	if (res->error) {
+		arena_free(&state->cache.dirs, res->dir);
+	} else {
 		bftw_file_set_dir(cache, file, res->dir);
 	}
 
diff --git a/src/dir.c b/src/dir.c
index b9fd74d..a24b572 100644
--- a/src/dir.c
+++ b/src/dir.c
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "dir.h"
+#include "alloc.h"
 #include "bfstd.h"
 #include "config.h"
 #include "diag.h"
@@ -120,26 +121,26 @@ struct bfs_dir {
 #  define DIR_SIZE sizeof(struct bfs_dir)
 #endif
 
-struct bfs_dir *bfs_opendir(int at_fd, const char *at_path) {
-	struct bfs_dir *dir = malloc(DIR_SIZE);
-	if (!dir) {
-		return NULL;
-	}
+struct bfs_dir *bfs_allocdir(void) {
+	return malloc(DIR_SIZE);
+}
+
+void bfs_dir_arena(struct arena *arena) {
+	arena_init(arena, alignof(struct bfs_dir), DIR_SIZE);
+}
 
+int bfs_opendir(struct bfs_dir *dir, int at_fd, const char *at_path) {
 	int fd;
 	if (at_path) {
 		fd = openat(at_fd, at_path, O_RDONLY | O_CLOEXEC | O_DIRECTORY);
+		if (fd < 0) {
+			return -1;
+		}
 	} else if (at_fd >= 0) {
 		fd = at_fd;
 	} else {
-		free(dir);
 		errno = EBADF;
-		return NULL;
-	}
-
-	if (fd < 0) {
-		free(dir);
-		return NULL;
+		return -1;
 	}
 
 #if BFS_GETDENTS
@@ -152,14 +153,13 @@ struct bfs_dir *bfs_opendir(int at_fd, const char *at_path) {
 		if (at_path) {
 			close_quietly(fd);
 		}
-		free(dir);
-		return NULL;
+		return -1;
 	}
 	dir->de = NULL;
 #endif
 
 	dir->eof = false;
-	return dir;
+	return 0;
 }
 
 int bfs_dirfd(const struct bfs_dir *dir) {
@@ -291,17 +291,16 @@ int bfs_closedir(struct bfs_dir *dir) {
 		bfs_verify(errno != EBADF);
 	}
 #endif
-	free(dir);
+
+	sanitize_uninit(dir, DIR_SIZE);
 	return ret;
 }
 
-int bfs_freedir(struct bfs_dir *dir, bool same_fd) {
+int bfs_fdclosedir(struct bfs_dir *dir, bool same_fd) {
 #if BFS_GETDENTS
 	int ret = dir->fd;
-	free(dir);
 #elif __FreeBSD__
 	int ret = fdclosedir(dir->dir);
-	free(dir);
 #else
 	if (same_fd) {
 		errno = ENOTSUP;
@@ -309,10 +308,13 @@ int bfs_freedir(struct bfs_dir *dir, bool same_fd) {
 	}
 
 	int ret = dup_cloexec(dirfd(dir->dir));
-	if (ret >= 0) {
-		bfs_closedir(dir);
+	if (ret < 0) {
+		return -1;
 	}
+
+	bfs_closedir(dir);
 #endif
 
+	sanitize_uninit(dir, DIR_SIZE);
 	return ret;
 }
diff --git a/src/dir.h b/src/dir.h
index 6fe7ae2..16f592e 100644
--- a/src/dir.h
+++ b/src/dir.h
@@ -8,6 +8,7 @@
 #ifndef BFS_DIR_H
 #define BFS_DIR_H
 
+#include "alloc.h"
 #include "config.h"
 #include <sys/types.h>
 
@@ -61,18 +62,36 @@ struct bfs_dirent {
 	const char *name;
 };
 
+/**
+ * Allocate space for a directory.
+ *
+ * @return
+ *         An allocated, unopen directory, or NULL on failure.
+ */
+struct bfs_dir *bfs_allocdir(void);
+
+/**
+ * Initialize an arena for directories.
+ *
+ * @param arena
+ *         The arena to initialize.
+ */
+void bfs_dir_arena(struct arena *arena);
+
 /**
  * Open a directory.
  *
+ * @param dir
+ *         The allocated directory.
  * @param at_fd
  *         The base directory for path resolution.
  * @param at_path
  *         The path of the directory to open, relative to at_fd.  Pass NULL to
  *         open at_fd itself.
  * @return
- *         The opened directory, or NULL on failure.
+ *         0 on success, or -1 on failure.
  */
-struct bfs_dir *bfs_opendir(int at_fd, const char *at_path);
+int bfs_opendir(struct bfs_dir *dir, int at_fd, const char *at_path);
 
 /**
  * Get the file descriptor for a directory.
@@ -110,7 +129,7 @@ int bfs_readdir(struct bfs_dir *dir, struct bfs_dirent *de);
 int bfs_closedir(struct bfs_dir *dir);
 
 /**
- * Free a directory, keeping an open file descriptor to it.
+ * Extract the file descriptor from an open directory.
  *
  * @param dir
  *         The directory to free.
@@ -122,6 +141,6 @@ int bfs_closedir(struct bfs_dir *dir);
  *         On success, a file descriptor for the directory is returned.  On
  *         failure, -1 is returned, and the directory remains open.
  */
-int bfs_freedir(struct bfs_dir *dir, bool same_fd);
+int bfs_fdclosedir(struct bfs_dir *dir, bool same_fd);
 
 #endif // BFS_DIR_H
diff --git a/src/eval.c b/src/eval.c
index f16821e..5f27681 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -421,10 +421,15 @@ bool eval_empty(const struct bfs_expr *expr, struct bfs_eval *state) {
 	const struct BFTW *ftwbuf = state->ftwbuf;
 
 	if (ftwbuf->type == BFS_DIR) {
-		struct bfs_dir *dir = bfs_opendir(ftwbuf->at_fd, ftwbuf->at_path);
+		struct bfs_dir *dir = bfs_allocdir();
 		if (!dir) {
 			eval_report_error(state);
-			goto done;
+			return ret;
+		}
+
+		if (bfs_opendir(dir, ftwbuf->at_fd, ftwbuf->at_path) != 0) {
+			eval_report_error(state);
+			return ret;
 		}
 
 		int did_read = bfs_readdir(dir, NULL);
@@ -435,6 +440,7 @@ bool eval_empty(const struct bfs_expr *expr, struct bfs_eval *state) {
 		}
 
 		bfs_closedir(dir);
+		free(dir);
 	} else if (ftwbuf->type == BFS_REG) {
 		const struct bfs_stat *statbuf = eval_stat(state);
 		if (statbuf) {
@@ -442,7 +448,6 @@ bool eval_empty(const struct bfs_expr *expr, struct bfs_eval *state) {
 		}
 	}
 
-done:
 	return ret;
 }
 
@@ -1495,20 +1500,25 @@ static int infer_fdlimit(const struct bfs_ctx *ctx, int limit) {
 
 	// Check /proc/self/fd for the current number of open fds, if possible
 	// (we may have inherited more than just the standard ones)
-	struct bfs_dir *dir = bfs_opendir(AT_FDCWD, "/proc/self/fd");
+	struct bfs_dir *dir = bfs_allocdir();
 	if (!dir) {
-		dir = bfs_opendir(AT_FDCWD, "/dev/fd");
+		goto done;
 	}
-	if (dir) {
-		// Account for 'dir' itself
-		nopen = -1;
 
-		while (bfs_readdir(dir, NULL) > 0) {
-			++nopen;
-		}
+	if (bfs_opendir(dir, AT_FDCWD, "/proc/self/fd") != 0
+	    && bfs_opendir(dir, AT_FDCWD, "/dev/fd") != 0) {
+		goto done;
+	}
 
-		bfs_closedir(dir);
+	// Account for 'dir' itself
+	nopen = -1;
+
+	while (bfs_readdir(dir, NULL) > 0) {
+		++nopen;
 	}
+	bfs_closedir(dir);
+done:
+	free(dir);
 
 	int ret = limit - nopen;
 	ret -= ctx->expr->persistent_fds;
diff --git a/src/ioq.c b/src/ioq.c
index 33316fa..47b082a 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -20,9 +20,11 @@
  * An I/O queue request.
  */
 struct ioq_req {
+	/** Directory allocation. */
+	struct bfs_dir *dir;
 	/** Base file descriptor for openat(). */
 	int dfd;
-	/** Relative path to dfd. */
+	/** Path to open, relative to dfd. */
 	const char *path;
 
 	/** Arbitrary user data. */
@@ -295,10 +297,12 @@ static void *ioq_work(void *ptr) {
 
 		struct ioq_res *res = &cmd->res;
 		res->ptr = req.ptr;
-		res->dir = bfs_opendir(req.dfd, req.path);
-		res->error = errno;
-		if (res->dir) {
+		res->dir = req.dir;
+		res->error = 0;
+		if (bfs_opendir(req.dir, req.dfd, req.path) == 0) {
 			bfs_polldir(res->dir);
+		} else {
+			res->error = errno;
 		}
 
 		ioqq_push(ioq->ready, cmd);
@@ -344,7 +348,7 @@ fail:
 	return NULL;
 }
 
-int ioq_opendir(struct ioq *ioq, int dfd, const char *path, void *ptr) {
+int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path, void *ptr) {
 	if (ioq->size >= ioq->depth) {
 		return -1;
 	}
@@ -355,6 +359,7 @@ int ioq_opendir(struct ioq *ioq, int dfd, const char *path, void *ptr) {
 	}
 
 	struct ioq_req *req = &cmd->req;
+	req->dir = dir;
 	req->dfd = dfd;
 	req->path = path;
 	req->ptr = ptr;
diff --git a/src/ioq.h b/src/ioq.h
index 0af5779..50e02b1 100644
--- a/src/ioq.h
+++ b/src/ioq.h
@@ -45,6 +45,8 @@ struct ioq *ioq_create(size_t depth, size_t nthreads);
  *
  * @param ioq
  *         The I/O queue.
+ * @param dir
+ *         The allocated directory.
  * @param dfd
  *         The base file descriptor.
  * @param path
@@ -54,7 +56,7 @@ struct ioq *ioq_create(size_t depth, size_t nthreads);
  * @return
  *         0 on success, or -1 on failure.
  */
-int ioq_opendir(struct ioq *ioq, int dfd, const char *path, void *ptr);
+int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path, void *ptr);
 
 /**
  * Pop a response from the queue.
-- 
cgit v1.2.3


From 6b96d7b0ad73e6ed63cf5e32fd2544121e2b0284 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Mon, 26 Jun 2023 11:35:23 -0400
Subject: ioq: Don't check NDEBUG manually in ioqq_trypop()

---
 src/ioq.c | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 47b082a..617bd5f 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -248,12 +248,9 @@ static union ioq_cmd *ioqq_trypop(struct ioqq *ioqq) {
 	prev &= ~IOQ_BLOCKED;
 
 	if (prev) {
-#ifdef NDEBUG
-		store(&ioqq->tail, i + IOQ_STRIDE, relaxed);
-#else
-		size_t j = fetch_add(&ioqq->tail, IOQ_STRIDE, relaxed);
+		size_t j = exchange(&ioqq->tail, i + IOQ_STRIDE, relaxed);
 		bfs_assert(j == i, "ioqq_trypop() only supports a single consumer");
-#endif
+		(void)j;
 	}
 
 	return (union ioq_cmd *)prev;
-- 
cgit v1.2.3


From abd29143d805fa16c65489d5b1d79428943d0187 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Mon, 26 Jun 2023 11:47:41 -0400
Subject: ioq: New ioq_cancel() function

---
 src/bftw.c |  4 ++++
 src/ioq.c  | 27 ++++++++++++++++++++-------
 src/ioq.h  |  5 +++++
 3 files changed, 29 insertions(+), 7 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/bftw.c b/src/bftw.c
index 69e41a2..2bdf12d 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -1147,6 +1147,10 @@ static int bftw_gc(struct bftw_state *state, enum bftw_gc_flags flags) {
 static int bftw_state_destroy(struct bftw_state *state) {
 	dstrfree(state->path);
 
+	if (state->ioq) {
+		ioq_cancel(state->ioq);
+	}
+
 	SLIST_EXTEND(&state->files, &state->batch);
 	do {
 		bftw_gc(state, BFTW_VISIT_NONE);
diff --git a/src/ioq.c b/src/ioq.c
index 617bd5f..457ead7 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -264,6 +264,8 @@ struct ioq {
 	size_t depth;
 	/** The current size of the queue. */
 	size_t size;
+	/** Cancellation flag. */
+	atomic bool cancel;
 
 	/** ioq_cmd command arena. */
 	struct arena cmds;
@@ -289,17 +291,22 @@ static void *ioq_work(void *ptr) {
 			break;
 		}
 
+		bool cancel = load(&ioq->cancel, relaxed);
+
 		struct ioq_req req = cmd->req;
 		sanitize_uninit(cmd);
 
 		struct ioq_res *res = &cmd->res;
 		res->ptr = req.ptr;
 		res->dir = req.dir;
-		res->error = 0;
-		if (bfs_opendir(req.dir, req.dfd, req.path) == 0) {
-			bfs_polldir(res->dir);
-		} else {
+
+		if (cancel) {
+			res->error = EINTR;
+		} else if (bfs_opendir(req.dir, req.dfd, req.path) != 0) {
 			res->error = errno;
+		} else {
+			res->error = 0;
+			bfs_polldir(res->dir);
 		}
 
 		ioqq_push(ioq->ready, cmd);
@@ -394,14 +401,20 @@ void ioq_free(struct ioq *ioq, struct ioq_res *res) {
 	arena_free(&ioq->cmds, (union ioq_cmd *)res);
 }
 
+void ioq_cancel(struct ioq *ioq) {
+	if (!exchange(&ioq->cancel, true, relaxed)) {
+		for (size_t i = 0; i < ioq->nthreads; ++i) {
+			ioqq_push(ioq->pending, &IOQ_STOP);
+		}
+	}
+}
+
 void ioq_destroy(struct ioq *ioq) {
 	if (!ioq) {
 		return;
 	}
 
-	for (size_t i = 0; i < ioq->nthreads; ++i) {
-		ioqq_push(ioq->pending, &IOQ_STOP);
-	}
+	ioq_cancel(ioq);
 
 	for (size_t i = 0; i < ioq->nthreads; ++i) {
 		if (pthread_join(ioq->threads[i], NULL) != 0) {
diff --git a/src/ioq.h b/src/ioq.h
index 50e02b1..064e2e2 100644
--- a/src/ioq.h
+++ b/src/ioq.h
@@ -88,6 +88,11 @@ struct ioq_res *ioq_trypop(struct ioq *ioq);
  */
 void ioq_free(struct ioq *ioq, struct ioq_res *res);
 
+/**
+ * Cancel any pending I/O operations.
+ */
+void ioq_cancel(struct ioq *ioq);
+
 /**
  * Stop and destroy an I/O queue.
  */
-- 
cgit v1.2.3


From 1313875b02c690ca5a40e585d24fdec240bb419d Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Mon, 26 Jun 2023 15:04:13 -0400
Subject: thread: Wrap more pthread APIs

---
 src/ioq.c    |   9 ++----
 src/lock.h   |  85 ------------------------------------------------
 src/main.c   |   2 +-
 src/thread.h | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/xregex.c |   5 ++-
 5 files changed, 109 insertions(+), 95 deletions(-)
 delete mode 100644 src/lock.h
 create mode 100644 src/thread.h

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 457ead7..1160b34 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -9,7 +9,7 @@
 #include "config.h"
 #include "diag.h"
 #include "dir.h"
-#include "lock.h"
+#include "thread.h"
 #include "sanity.h"
 #include <assert.h>
 #include <errno.h>
@@ -335,8 +335,7 @@ struct ioq *ioq_create(size_t depth, size_t nthreads) {
 	}
 
 	for (size_t i = 0; i < nthreads; ++i) {
-		errno = pthread_create(&ioq->threads[i], NULL, ioq_work, ioq);
-		if (errno != 0) {
+		if (thread_create(&ioq->threads[i], NULL, ioq_work, ioq) != 0) {
 			goto fail;
 		}
 		++ioq->nthreads;
@@ -417,9 +416,7 @@ void ioq_destroy(struct ioq *ioq) {
 	ioq_cancel(ioq);
 
 	for (size_t i = 0; i < ioq->nthreads; ++i) {
-		if (pthread_join(ioq->threads[i], NULL) != 0) {
-			abort();
-		}
+		thread_join(ioq->threads[i], NULL);
 	}
 
 	ioqq_destroy(ioq->ready);
diff --git a/src/lock.h b/src/lock.h
deleted file mode 100644
index 2b5f951..0000000
--- a/src/lock.h
+++ /dev/null
@@ -1,85 +0,0 @@
-// Copyright © Tavian Barnes <tavianator@tavianator.com>
-// SPDX-License-Identifier: 0BSD
-
-/**
- * Wrappers for POSIX synchronization primitives.
- */
-
-#ifndef BFS_LOCK_H
-#define BFS_LOCK_H
-
-#include "diag.h"
-#include <errno.h>
-#include <pthread.h>
-#include <string.h>
-
-#define lock_verify(expr, cond) \
-	bfs_verify((errno = (expr), (cond)), "%s: %s", #expr, strerror(errno))
-
-/**
- * Wrapper for pthread_mutex_init().
- *
- * @return
- *         0 on success, -1 on error.
- */
-#define mutex_init(mutex, attr) \
-	((errno = pthread_mutex_init(mutex, attr)) ? -1 : 0)
-
-/**
- * Wrapper for pthread_mutex_lock().
- */
-#define mutex_lock(mutex) \
-	lock_verify(pthread_mutex_lock(mutex), errno == 0)
-
-/**
- * Wrapper for pthread_mutex_trylock().
- *
- * @return
- *         Whether the mutex was locked.
- */
-#define mutex_trylock(mutex) \
-	(lock_verify(pthread_mutex_trylock(mutex), errno == 0 || errno == EBUSY), errno == 0)
-
-/**
- * Wrapper for pthread_mutex_unlock().
- */
-#define mutex_unlock(mutex) \
-	lock_verify(pthread_mutex_unlock(mutex), errno == 0)
-
-/**
- * Wrapper for pthread_mutex_destroy().
- */
-#define mutex_destroy(mutex) \
-	lock_verify(pthread_mutex_destroy(mutex), errno == 0)
-
-/**
- * Wrapper for pthread_cond_init().
- */
-#define cond_init(cond, attr) \
-	((errno = pthread_cond_init(cond, attr)) ? -1 : 0)
-
-/**
- * Wrapper for pthread_cond_wait().
- */
-#define cond_wait(cond, mutex) \
-	lock_verify(pthread_cond_wait(cond, mutex), errno == 0)
-
-/**
- * Wrapper for pthread_cond_signal().
- */
-#define cond_signal(cond) \
-	lock_verify(pthread_cond_signal(cond), errno == 0)
-
-/**
- * Wrapper for pthread_cond_broadcast().
- */
-#define cond_broadcast(cond) \
-	lock_verify(pthread_cond_broadcast(cond), errno == 0)
-
-/**
- * Wrapper for pthread_cond_destroy().
- */
-#define cond_destroy(cond) \
-	lock_verify(pthread_cond_destroy(cond), errno == 0)
-
-#endif // BFS_LOCK_H
diff --git a/src/main.c b/src/main.c
index b7a08c1..b26be85 100644
--- a/src/main.c
+++ b/src/main.c
@@ -34,11 +34,11 @@
  *     - fsade.[ch]    (a facade over non-standard filesystem features)
  *     - ioq.[ch]      (an async I/O queue)
  *     - list.h        (linked list macros)
- *     - lock.h        (mutexes, condition variables, etc.)
  *     - mtab.[ch]     (parses the system's mount table)
  *     - pwcache.[ch]  (a cache for the user/group tables)
  *     - sanity.h      (sanitizer interfaces)
  *     - stat.[ch]     (wraps stat(), or statx() on Linux)
+ *     - thread.h      (multi-threading)
  *     - trie.[ch]     (a trie set/map implementation)
  *     - typo.[ch]     (fuzzy matching for typos)
  *     - xregex.[ch]   (regular expression support)
diff --git a/src/thread.h b/src/thread.h
new file mode 100644
index 0000000..b2edf17
--- /dev/null
+++ b/src/thread.h
@@ -0,0 +1,103 @@
+// Copyright © Tavian Barnes <tavianator@tavianator.com>
+// SPDX-License-Identifier: 0BSD
+
+/**
+ * Wrappers for POSIX threading APIs.
+ */
+
+#ifndef BFS_THREAD_H
+#define BFS_THREAD_H
+
+#include "diag.h"
+#include <errno.h>
+#include <pthread.h>
+#include <string.h>
+
+#define thread_verify(expr, cond) \
+	bfs_verify((errno = (expr), (cond)), "%s: %s", #expr, strerror(errno))
+
+/**
+ * Wrapper for pthread_create().
+ *
+ * @return
+ *         0 on success, -1 on error.
+ */
+#define thread_create(thread, attr, fn, arg) \
+	((errno = pthread_create(thread, attr, fn, arg)) ? -1 : 0)
+
+/**
+ * Wrapper for pthread_join().
+ */
+#define thread_join(thread, ret) \
+	thread_verify(pthread_join(thread, ret), errno == 0)
+
+/**
+ * Wrapper for pthread_mutex_init().
+ */
+#define mutex_init(mutex, attr) \
+	((errno = pthread_mutex_init(mutex, attr)) ? -1 : 0)
+
+/**
+ * Wrapper for pthread_mutex_lock().
+ */
+#define mutex_lock(mutex) \
+	thread_verify(pthread_mutex_lock(mutex), errno == 0)
+
+/**
+ * Wrapper for pthread_mutex_trylock().
+ *
+ * @return
+ *         Whether the mutex was locked.
+ */
+#define mutex_trylock(mutex) \
+	(thread_verify(pthread_mutex_trylock(mutex), errno == 0 || errno == EBUSY), errno == 0)
+
+/**
+ * Wrapper for pthread_mutex_unlock().
+ */
+#define mutex_unlock(mutex) \
+	thread_verify(pthread_mutex_unlock(mutex), errno == 0)
+
+/**
+ * Wrapper for pthread_mutex_destroy().
+ */
+#define mutex_destroy(mutex) \
+	thread_verify(pthread_mutex_destroy(mutex), errno == 0)
+
+/**
+ * Wrapper for pthread_cond_init().
+ */
+#define cond_init(cond, attr) \
+	((errno = pthread_cond_init(cond, attr)) ? -1 : 0)
+
+/**
+ * Wrapper for pthread_cond_wait().
+ */
+#define cond_wait(cond, mutex) \
+	thread_verify(pthread_cond_wait(cond, mutex), errno == 0)
+
+/**
+ * Wrapper for pthread_cond_signal().
+ */
+#define cond_signal(cond) \
+	thread_verify(pthread_cond_signal(cond), errno == 0)
+
+/**
+ * Wrapper for pthread_cond_broadcast().
+ */
+#define cond_broadcast(cond) \
+	thread_verify(pthread_cond_broadcast(cond), errno == 0)
+
+/**
+ * Wrapper for pthread_cond_destroy().
+ */
+#define cond_destroy(cond) \
+	thread_verify(pthread_cond_destroy(cond), errno == 0)
+
+/**
+ * Wrapper for pthread_once().
+ */
+#define call_once(once, fn) \
+	thread_verify(pthread_once(once, fn), errno == 0)
+
+#endif // BFS_THREAD_H
diff --git a/src/xregex.c b/src/xregex.c
index 88df082..beb6676 100644
--- a/src/xregex.c
+++ b/src/xregex.c
@@ -5,6 +5,7 @@
 #include "alloc.h"
 #include "config.h"
 #include "diag.h"
+#include "thread.h"
 #include "sanity.h"
 #include <errno.h>
 #include <pthread.h>
@@ -106,9 +107,7 @@ static void bfs_onig_once(void) {
 /** Initialize Oniguruma. */
 static int bfs_onig_initialize(OnigEncoding *enc) {
 	static pthread_once_t once = PTHREAD_ONCE_INIT;
-	if (pthread_once(&once, bfs_onig_once) != 0) {
-		return ONIGERR_FAIL_TO_INITIALIZE;
-	}
+	call_once(&once, bfs_onig_once);
 
 	*enc = bfs_onig_enc;
 	return bfs_onig_status;
-- 
cgit v1.2.3


From 58a36277cfaf95cba8609c1ddc823df2bc9fb60e Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Tue, 4 Jul 2023 13:31:54 -0400
Subject: ioq: Don't write to an empty slot in ioqq_trypop()

---
 src/ioq.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 1160b34..10451ac 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -240,18 +240,22 @@ static union ioq_cmd *ioqq_trypop(struct ioqq *ioqq) {
 	size_t i = load(&ioqq->tail, relaxed);
 	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
 
-	uintptr_t prev = exchange(slot, 0, acquire);
+	uintptr_t prev = load(slot, relaxed);
+	if (!(prev & ~IOQ_BLOCKED)) {
+		return NULL;
+	}
+	if (!compare_exchange_weak(slot, &prev, 0, acquire, relaxed)) {
+		return NULL;
+	}
 
 	if (prev & IOQ_BLOCKED) {
 		ioqq_wake(ioqq, i);
 	}
 	prev &= ~IOQ_BLOCKED;
 
-	if (prev) {
-		size_t j = exchange(&ioqq->tail, i + IOQ_STRIDE, relaxed);
-		bfs_assert(j == i, "ioqq_trypop() only supports a single consumer");
-		(void)j;
-	}
+	size_t j = exchange(&ioqq->tail, i + IOQ_STRIDE, relaxed);
+	bfs_assert(j == i, "ioqq_trypop() only supports a single consumer");
+	(void)j;
 
 	return (union ioq_cmd *)prev;
 }
-- 
cgit v1.2.3


From 379dee8a47480938c067fca0acd01dea9b5afa33 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Tue, 4 Jul 2023 15:01:18 -0400
Subject: ioq: New ioq_capacity() function

---
 src/ioq.c | 9 +++++++--
 src/ioq.h | 5 +++++
 2 files changed, 12 insertions(+), 2 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 10451ac..5673c77 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -355,6 +355,10 @@ fail:
 	return NULL;
 }
 
+size_t ioq_capacity(const struct ioq *ioq) {
+	return ioq->depth - ioq->size;
+}
+
 int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path, void *ptr) {
 	if (ioq->size >= ioq->depth) {
 		return -1;
@@ -382,7 +386,6 @@ struct ioq_res *ioq_pop(struct ioq *ioq) {
 	}
 
 	union ioq_cmd *cmd = ioqq_pop(ioq->ready);
-	--ioq->size;
 	return &cmd->res;
 }
 
@@ -396,11 +399,13 @@ struct ioq_res *ioq_trypop(struct ioq *ioq) {
 		return NULL;
 	}
 
-	--ioq->size;
 	return &cmd->res;
 }
 
 void ioq_free(struct ioq *ioq, struct ioq_res *res) {
+	bfs_assert(ioq->size > 0);
+	--ioq->size;
+
 	arena_free(&ioq->cmds, (union ioq_cmd *)res);
 }
 
diff --git a/src/ioq.h b/src/ioq.h
index 064e2e2..9901293 100644
--- a/src/ioq.h
+++ b/src/ioq.h
@@ -40,6 +40,11 @@ struct ioq_res {
  */
 struct ioq *ioq_create(size_t depth, size_t nthreads);
 
+/**
+ * Check the remaining capacity of a queue.
+ */
+size_t ioq_capacity(const struct ioq *ioq);
+
 /**
  * Asynchronous bfs_opendir().
  *
-- 
cgit v1.2.3


From 222ac5ba4fbab0ab880e36423d0f1338e39b02c7 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Tue, 4 Jul 2023 15:17:23 -0400
Subject: ioq: Implement async close() and closedir()

---
 src/bftw.c |  41 ++++++++-------
 src/ioq.c  | 175 ++++++++++++++++++++++++++++++++++++-------------------------
 src/ioq.h  |  81 ++++++++++++++++++++++++----
 3 files changed, 197 insertions(+), 100 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/bftw.c b/src/bftw.c
index ec67817..64f221b 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -901,34 +901,39 @@ static int bftw_ioq_pop(struct bftw_state *state, bool block) {
 		return -1;
 	}
 
-	struct ioq_res *res = block ? ioq_pop(ioq) : ioq_trypop(ioq);
-	if (!res) {
+	struct ioq_ent *ent = block ? ioq_pop(ioq) : ioq_trypop(ioq);
+	if (!ent) {
 		return -1;
 	}
 
 	struct bftw_cache *cache = &state->cache;
-	++cache->capacity;
-
-	struct bftw_file *file = res->ptr;
-	file->ioqueued = false;
+	struct bftw_file *file;
+	struct bfs_dir *dir;
 
-	if (file->parent) {
-		bftw_cache_unpin(cache, file->parent);
-	}
+	enum ioq_op op = ent->op;
+	if (op == IOQ_OPENDIR) {
+		file = ent->ptr;
+		file->ioqueued = false;
 
-	if (res->error) {
-		arena_free(&cache->dirs, res->dir);
-	} else {
-		bftw_file_set_dir(cache, file, res->dir);
-	}
+		++cache->capacity;
+		if (file->parent) {
+			bftw_cache_unpin(cache, file->parent);
+		}
 
-	ioq_free(ioq, res);
+		dir = ent->opendir.dir;
+		if (ent->ret == 0) {
+			bftw_file_set_dir(cache, file, dir);
+		} else {
+			arena_free(&cache->dirs, dir);
+		}
 
-	if (!(state->flags & BFTW_SORT)) {
-		SLIST_PREPEND(&state->dirs, file);
+		if (!(state->flags & BFTW_SORT)) {
+			SLIST_PREPEND(&state->dirs, file);
+		}
 	}
 
-	return 0;
+	ioq_free(ioq, ent);
+	return op;
 }
 
 /** Try to reserve space in the I/O queue. */
diff --git a/src/ioq.c b/src/ioq.c
index 5673c77..0544044 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -16,29 +16,6 @@
 #include <pthread.h>
 #include <stdlib.h>
 
-/**
- * An I/O queue request.
- */
-struct ioq_req {
-	/** Directory allocation. */
-	struct bfs_dir *dir;
-	/** Base file descriptor for openat(). */
-	int dfd;
-	/** Path to open, relative to dfd. */
-	const char *path;
-
-	/** Arbitrary user data. */
-	void *ptr;
-};
-
-/**
- * An I/O queue command.
- */
-union ioq_cmd {
-	struct ioq_req req;
-	struct ioq_res res;
-};
-
 /**
  * A monitor for an I/O queue slot.
  */
@@ -101,7 +78,7 @@ bfs_static_assert(IOQ_STRIDE % 2 == 1);
 
 /** Slot flag bit to indicate waiters. */
 #define IOQ_BLOCKED ((uintptr_t)1)
-bfs_static_assert(alignof(union ioq_cmd) > 1);
+bfs_static_assert(alignof(struct ioq_ent) > 1);
 
 /** Destroy an I/O command queue. */
 static void ioqq_destroy(struct ioqq *ioqq) {
@@ -189,12 +166,12 @@ static void ioqq_wake(struct ioqq *ioqq, size_t i) {
 	cond_broadcast(&monitor->cond);
 }
 
-/** Push a command onto the queue. */
-static void ioqq_push(struct ioqq *ioqq, union ioq_cmd *cmd) {
+/** Push an entry onto the queue. */
+static void ioqq_push(struct ioqq *ioqq, struct ioq_ent *ent) {
 	size_t i = fetch_add(&ioqq->head, IOQ_STRIDE, relaxed);
 	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
 
-	uintptr_t addr = (uintptr_t)cmd;
+	uintptr_t addr = (uintptr_t)ent;
 	bfs_assert(!(addr & IOQ_BLOCKED));
 
 	uintptr_t prev = load(slot, relaxed);
@@ -212,8 +189,8 @@ static void ioqq_push(struct ioqq *ioqq, union ioq_cmd *cmd) {
 	}
 }
 
-/** Pop a command from a queue. */
-static union ioq_cmd *ioqq_pop(struct ioqq *ioqq) {
+/** Pop an entry from the queue. */
+static struct ioq_ent *ioqq_pop(struct ioqq *ioqq) {
 	size_t i = fetch_add(&ioqq->tail, IOQ_STRIDE, relaxed);
 	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
 
@@ -232,11 +209,11 @@ static union ioq_cmd *ioqq_pop(struct ioqq *ioqq) {
 	}
 	prev &= ~IOQ_BLOCKED;
 
-	return (union ioq_cmd *)prev;
+	return (struct ioq_ent *)prev;
 }
 
-/** Pop a command from a queue if one is available. */
-static union ioq_cmd *ioqq_trypop(struct ioqq *ioqq) {
+/** Pop an entry from the queue if one is available. */
+static struct ioq_ent *ioqq_trypop(struct ioqq *ioqq) {
 	size_t i = load(&ioqq->tail, relaxed);
 	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
 
@@ -257,11 +234,11 @@ static union ioq_cmd *ioqq_trypop(struct ioqq *ioqq) {
 	bfs_assert(j == i, "ioqq_trypop() only supports a single consumer");
 	(void)j;
 
-	return (union ioq_cmd *)prev;
+	return (struct ioq_ent *)prev;
 }
 
 /** Sentinel stop command. */
-static union ioq_cmd IOQ_STOP;
+static struct ioq_ent IOQ_STOP;
 
 struct ioq {
 	/** The depth of the queue. */
@@ -271,8 +248,8 @@ struct ioq {
 	/** Cancellation flag. */
 	atomic bool cancel;
 
-	/** ioq_cmd command arena. */
-	struct arena cmds;
+	/** ioq_ent arena. */
+	struct arena ents;
 
 	/** Pending I/O requests. */
 	struct ioqq *pending;
@@ -290,30 +267,50 @@ static void *ioq_work(void *ptr) {
 	struct ioq *ioq = ptr;
 
 	while (true) {
-		union ioq_cmd *cmd = ioqq_pop(ioq->pending);
-		if (cmd == &IOQ_STOP) {
+		struct ioq_ent *ent = ioqq_pop(ioq->pending);
+		if (ent == &IOQ_STOP) {
 			break;
 		}
 
 		bool cancel = load(&ioq->cancel, relaxed);
 
-		struct ioq_req req = cmd->req;
-		sanitize_uninit(cmd);
+		ent->ret = -1;
+
+		switch (ent->op) {
+		case IOQ_CLOSE:
+			// Always close(), even if we're cancelled, just like a real EINTR
+			ent->ret = xclose(ent->close.fd);
+			break;
+
+		case IOQ_OPENDIR:
+			if (!cancel) {
+				struct ioq_opendir *args = &ent->opendir;
+				ent->ret = bfs_opendir(args->dir, args->dfd, args->path);
+				if (ent->ret == 0) {
+					bfs_polldir(args->dir);
+				}
+			}
+			break;
+
+		case IOQ_CLOSEDIR:
+			ent->ret = bfs_closedir(ent->closedir.dir);
+			break;
 
-		struct ioq_res *res = &cmd->res;
-		res->ptr = req.ptr;
-		res->dir = req.dir;
+		default:
+			bfs_bug("Unknown ioq_op %d", (int)ent->op);
+			errno = ENOSYS;
+			break;
+		}
 
 		if (cancel) {
-			res->error = EINTR;
-		} else if (bfs_opendir(req.dir, req.dfd, req.path) != 0) {
-			res->error = errno;
+			ent->error = EINTR;
+		} else if (ent->ret < 0) {
+			ent->error = errno;
 		} else {
-			res->error = 0;
-			bfs_polldir(res->dir);
+			ent->error = 0;
 		}
 
-		ioqq_push(ioq->ready, cmd);
+		ioqq_push(ioq->ready, ent);
 	}
 
 	return NULL;
@@ -326,7 +323,7 @@ struct ioq *ioq_create(size_t depth, size_t nthreads) {
 	}
 
 	ioq->depth = depth;
-	ARENA_INIT(&ioq->cmds, union ioq_cmd);
+	ARENA_INIT(&ioq->ents, struct ioq_ent);
 
 	ioq->pending = ioqq_create(depth);
 	if (!ioq->pending) {
@@ -359,54 +356,88 @@ size_t ioq_capacity(const struct ioq *ioq) {
 	return ioq->depth - ioq->size;
 }
 
-int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path, void *ptr) {
+static struct ioq_ent *ioq_request(struct ioq *ioq, enum ioq_op op, void *ptr) {
+	if (load(&ioq->cancel, relaxed)) {
+		errno = EINTR;
+		return NULL;
+	}
+
 	if (ioq->size >= ioq->depth) {
+		errno = EAGAIN;
+		return NULL;
+	}
+
+	struct ioq_ent *ent = arena_alloc(&ioq->ents);
+	if (!ent) {
+		return NULL;
+	}
+
+	ent->op = op;
+	ent->ptr = ptr;
+	++ioq->size;
+	return ent;
+}
+
+int ioq_close(struct ioq *ioq, int fd, void *ptr) {
+	struct ioq_ent *ent = ioq_request(ioq, IOQ_CLOSE, ptr);
+	if (!ent) {
 		return -1;
 	}
 
-	union ioq_cmd *cmd = arena_alloc(&ioq->cmds);
-	if (!cmd) {
+	ent->close.fd = fd;
+
+	ioqq_push(ioq->pending, ent);
+	return 0;
+}
+
+int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path, void *ptr) {
+	struct ioq_ent *ent = ioq_request(ioq, IOQ_OPENDIR, ptr);
+	if (!ent) {
 		return -1;
 	}
 
-	struct ioq_req *req = &cmd->req;
-	req->dir = dir;
-	req->dfd = dfd;
-	req->path = path;
-	req->ptr = ptr;
+	struct ioq_opendir *args = &ent->opendir;
+	args->dir = dir;
+	args->dfd = dfd;
+	args->path = path;
 
-	ioqq_push(ioq->pending, cmd);
-	++ioq->size;
+	ioqq_push(ioq->pending, ent);
 	return 0;
 }
 
-struct ioq_res *ioq_pop(struct ioq *ioq) {
-	if (ioq->size == 0) {
-		return NULL;
+int ioq_closedir(struct ioq *ioq, struct bfs_dir *dir, void *ptr) {
+	struct ioq_ent *ent = ioq_request(ioq, IOQ_CLOSEDIR, ptr);
+	if (!ent) {
+		return -1;
 	}
 
-	union ioq_cmd *cmd = ioqq_pop(ioq->ready);
-	return &cmd->res;
+	ent->closedir.dir = dir;
+
+	ioqq_push(ioq->pending, ent);
+	return 0;
 }
 
-struct ioq_res *ioq_trypop(struct ioq *ioq) {
+struct ioq_ent *ioq_pop(struct ioq *ioq) {
 	if (ioq->size == 0) {
 		return NULL;
 	}
 
-	union ioq_cmd *cmd = ioqq_trypop(ioq->ready);
-	if (!cmd) {
+	return ioqq_pop(ioq->ready);
+}
+
+struct ioq_ent *ioq_trypop(struct ioq *ioq) {
+	if (ioq->size == 0) {
 		return NULL;
 	}
 
-	return &cmd->res;
+	return ioqq_trypop(ioq->ready);
 }
 
-void ioq_free(struct ioq *ioq, struct ioq_res *res) {
+void ioq_free(struct ioq *ioq, struct ioq_ent *ent) {
 	bfs_assert(ioq->size > 0);
 	--ioq->size;
 
-	arena_free(&ioq->cmds, (union ioq_cmd *)res);
+	arena_free(&ioq->ents, ent);
 }
 
 void ioq_cancel(struct ioq *ioq) {
@@ -431,7 +462,7 @@ void ioq_destroy(struct ioq *ioq) {
 	ioqq_destroy(ioq->ready);
 	ioqq_destroy(ioq->pending);
 
-	arena_destroy(&ioq->cmds);
+	arena_destroy(&ioq->ents);
 
 	free(ioq);
 }
diff --git a/src/ioq.h b/src/ioq.h
index 9901293..99c18c2 100644
--- a/src/ioq.h
+++ b/src/ioq.h
@@ -16,16 +16,49 @@
 struct ioq;
 
 /**
- * An I/O queue response.
+ * I/O queue operations.
  */
-struct ioq_res {
-	/** The opened directory. */
-	struct bfs_dir *dir;
+enum ioq_op {
+	/** ioq_close(). */
+	IOQ_CLOSE,
+	/** ioq_opendir(). */
+	IOQ_OPENDIR,
+	/** ioq_closedir(). */
+	IOQ_CLOSEDIR,
+};
+
+/**
+ * An I/O queue entry.
+ */
+struct ioq_ent {
+	/** The I/O operation. */
+	enum ioq_op op;
+
+	/** The return value of the operation. */
+	int ret;
 	/** The error code, if the operation failed. */
 	int error;
 
 	/** Arbitrary user data. */
 	void *ptr;
+
+	/** Operation-specific arguments. */
+	union {
+		/** ioq_close() args. */
+		struct ioq_close {
+			int fd;
+		} close;
+		/** ioq_opendir() args. */
+		struct ioq_opendir {
+			struct bfs_dir *dir;
+			int dfd;
+			const char *path;
+		} opendir;
+		/** ioq_closedir() args. */
+		struct ioq_closedir {
+			struct bfs_dir *dir;
+		} closedir;
+	};
 };
 
 /**
@@ -45,6 +78,20 @@ struct ioq *ioq_create(size_t depth, size_t nthreads);
  */
 size_t ioq_capacity(const struct ioq *ioq);
 
+/**
+ * Asynchronous close().
+ *
+ * @param ioq
+ *         The I/O queue.
+ * @param fd
+ *         The fd to close.
+ * @param ptr
+ *         An arbitrary pointer to associate with the request.
+ * @return
+ *         0 on success, or -1 on failure.
+ */
+int ioq_close(struct ioq *ioq, int fd, void *ptr);
+
 /**
  * Asynchronous bfs_opendir().
  *
@@ -63,6 +110,20 @@ size_t ioq_capacity(const struct ioq *ioq);
  */
 int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path, void *ptr);
 
+/**
+ * Asynchronous bfs_closedir().
+ *
+ * @param ioq
+ *         The I/O queue.
+ * @param dir
+ *         The directory to close.
+ * @param ptr
+ *         An arbitrary pointer to associate with the request.
+ * @return
+ *         0 on success, or -1 on failure.
+ */
+int ioq_closedir(struct ioq *ioq, struct bfs_dir *dir, void *ptr);
+
 /**
  * Pop a response from the queue.
  *
@@ -71,7 +132,7 @@ int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path,
  * @return
  *         The next response, or NULL.
  */
-struct ioq_res *ioq_pop(struct ioq *ioq);
+struct ioq_ent *ioq_pop(struct ioq *ioq);
 
 /**
  * Pop a response from the queue, without blocking.
@@ -81,17 +142,17 @@ struct ioq_res *ioq_pop(struct ioq *ioq);
  * @return
  *         The next response, or NULL.
  */
-struct ioq_res *ioq_trypop(struct ioq *ioq);
+struct ioq_ent *ioq_trypop(struct ioq *ioq);
 
 /**
- * Free a response.
+ * Free a queue entry.
  *
  * @param ioq
  *         The I/O queue.
- * @param res
- *         The response to free.
+ * @param ent
+ *         The entry to free.
  */
-void ioq_free(struct ioq *ioq, struct ioq_res *res);
+void ioq_free(struct ioq *ioq, struct ioq_ent *ent);
 
 /**
  * Cancel any pending I/O operations.
-- 
cgit v1.2.3


From dcb6cf524a1d7279457a3e1e7fecff5f46181ab4 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Tue, 11 Jul 2023 10:21:26 -0400
Subject: ioq: Separate slot and queue operations

---
 src/ioq.c | 124 +++++++++++++++++++++++++++++++++++++-------------------------
 1 file changed, 75 insertions(+), 49 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 0544044..c66ebda 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -44,6 +44,28 @@ static void ioq_monitor_destroy(struct ioq_monitor *monitor) {
 	mutex_destroy(&monitor->mutex);
 }
 
+/** A single entry in a command queue. */
+typedef atomic uintptr_t ioq_slot;
+
+/** Slot flag bit to indicate waiters. */
+#define IOQ_BLOCKED ((uintptr_t)1)
+bfs_static_assert(alignof(struct ioq_ent) > 1);
+
+/** Check if a slot has waiters. */
+static bool ioq_slot_blocked(uintptr_t value) {
+	return value & IOQ_BLOCKED;
+}
+
+/** Extract the pointer from a slot. */
+static struct ioq_ent *ioq_slot_ptr(uintptr_t value) {
+	return (struct ioq_ent *)(value & ~IOQ_BLOCKED);
+}
+
+/** Check if a slot is empty. */
+static bool ioq_slot_empty(uintptr_t value) {
+	return !ioq_slot_ptr(value);
+}
+
 /**
  * An MPMC queue of I/O commands.
  */
@@ -62,7 +84,7 @@ struct ioqq {
 	cache_align atomic size_t tail;
 
 	/** The circular buffer itself. */
-	cache_align atomic uintptr_t slots[];
+	cache_align ioq_slot slots[];
 };
 
 // If we assign slots sequentially, threads will likely be operating on
@@ -73,13 +95,9 @@ struct ioqq {
 // still use every available slot.  Since the length is a power of two, that
 // means the stride must be odd.
 
-#define IOQ_STRIDE ((FALSE_SHARING_SIZE / sizeof(atomic uintptr_t)) | 1)
+#define IOQ_STRIDE ((FALSE_SHARING_SIZE / sizeof(ioq_slot)) | 1)
 bfs_static_assert(IOQ_STRIDE % 2 == 1);
 
-/** Slot flag bit to indicate waiters. */
-#define IOQ_BLOCKED ((uintptr_t)1)
-bfs_static_assert(alignof(struct ioq_ent) > 1);
-
 /** Destroy an I/O command queue. */
 static void ioqq_destroy(struct ioqq *ioqq) {
 	for (size_t i = 0; i < ioqq->monitor_mask + 1; ++i) {
@@ -129,9 +147,11 @@ static struct ioqq *ioqq_create(size_t size) {
 }
 
 /** Atomically wait for a slot to change. */
-static uintptr_t ioqq_wait(struct ioqq *ioqq, size_t i, uintptr_t value) {
-	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
+static uintptr_t ioq_slot_wait(struct ioqq *ioqq, ioq_slot *slot, uintptr_t value) {
+	fetch_or(slot, IOQ_BLOCKED, relaxed);
+	value |= IOQ_BLOCKED;
 
+	size_t i = slot - ioqq->slots;
 	struct ioq_monitor *monitor = &ioqq->monitors[i & ioqq->monitor_mask];
 	mutex_lock(&monitor->mutex);
 
@@ -147,11 +167,12 @@ static uintptr_t ioqq_wait(struct ioqq *ioqq, size_t i, uintptr_t value) {
 }
 
 /** Wake up any threads waiting on a slot. */
-static void ioqq_wake(struct ioqq *ioqq, size_t i) {
+static void ioq_slot_wake(struct ioqq *ioqq, ioq_slot *slot) {
+	size_t i = slot - ioqq->slots;
 	struct ioq_monitor *monitor = &ioqq->monitors[i & ioqq->monitor_mask];
 
 	// The following implementation would clearly avoid the missed wakeup
-	// issue mentioned above in ioqq_wait():
+	// issue mentioned above in ioq_slot_wait():
 	//
 	//     mutex_lock(&monitor->mutex);
 	//     cond_broadcast(&monitor->cond);
@@ -166,75 +187,80 @@ static void ioqq_wake(struct ioqq *ioqq, size_t i) {
 	cond_broadcast(&monitor->cond);
 }
 
-/** Push an entry onto the queue. */
-static void ioqq_push(struct ioqq *ioqq, struct ioq_ent *ent) {
+/** Get the next slot for writing. */
+static ioq_slot *ioqq_write(struct ioqq *ioqq) {
 	size_t i = fetch_add(&ioqq->head, IOQ_STRIDE, relaxed);
-	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
+	return &ioqq->slots[i & ioqq->slot_mask];
+}
 
+/** Push an entry into a slot. */
+static void ioq_slot_push(struct ioqq *ioqq, ioq_slot *slot, struct ioq_ent *ent) {
 	uintptr_t addr = (uintptr_t)ent;
-	bfs_assert(!(addr & IOQ_BLOCKED));
+	bfs_assert(!ioq_slot_blocked(addr));
 
 	uintptr_t prev = load(slot, relaxed);
 	do {
-		while (prev & ~IOQ_BLOCKED) {
-			prev = fetch_or(slot, IOQ_BLOCKED, relaxed);
-			if (prev & ~IOQ_BLOCKED) {
-				prev = ioqq_wait(ioqq, i, prev | IOQ_BLOCKED);
-			}
+		while (!ioq_slot_empty(prev)) {
+			prev = ioq_slot_wait(ioqq, slot, prev);
 		}
 	} while (!compare_exchange_weak(slot, &prev, addr, release, relaxed));
 
-	if (prev & IOQ_BLOCKED) {
-		ioqq_wake(ioqq, i);
+	if (ioq_slot_blocked(prev)) {
+		ioq_slot_wake(ioqq, slot);
 	}
 }
 
-/** Pop an entry from the queue. */
-static struct ioq_ent *ioqq_pop(struct ioqq *ioqq) {
+/** Push an entry onto the queue. */
+static void ioqq_push(struct ioqq *ioqq, struct ioq_ent *ent) {
+	ioq_slot *slot = ioqq_write(ioqq);
+	ioq_slot_push(ioqq, slot, ent);
+}
+
+/** Get the next slot for reading. */
+static ioq_slot *ioqq_read(struct ioqq *ioqq) {
 	size_t i = fetch_add(&ioqq->tail, IOQ_STRIDE, relaxed);
-	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
+	return &ioqq->slots[i & ioqq->slot_mask];
+}
 
+/** (Try to) pop an entry from a slot. */
+static struct ioq_ent *ioq_slot_pop(struct ioqq *ioqq, ioq_slot *slot, bool block) {
 	uintptr_t prev = load(slot, relaxed);
 	do {
-		while (!(prev & ~IOQ_BLOCKED)) {
-			prev = fetch_or(slot, IOQ_BLOCKED, relaxed);
-			if (!(prev & ~IOQ_BLOCKED)) {
-				prev = ioqq_wait(ioqq, i, IOQ_BLOCKED);
+		while (ioq_slot_empty(prev)) {
+			if (block) {
+				prev = ioq_slot_wait(ioqq, slot, prev);
+			} else {
+				return NULL;
 			}
 		}
 	} while (!compare_exchange_weak(slot, &prev, 0, acquire, relaxed));
 
-	if (prev & IOQ_BLOCKED) {
-		ioqq_wake(ioqq, i);
+	if (ioq_slot_blocked(prev)) {
+		ioq_slot_wake(ioqq, slot);
 	}
-	prev &= ~IOQ_BLOCKED;
 
-	return (struct ioq_ent *)prev;
+	return ioq_slot_ptr(prev);
+}
+
+/** Pop an entry from the queue. */
+static struct ioq_ent *ioqq_pop(struct ioqq *ioqq) {
+	ioq_slot *slot = ioqq_read(ioqq);
+	return ioq_slot_pop(ioqq, slot, true);
 }
 
 /** Pop an entry from the queue if one is available. */
 static struct ioq_ent *ioqq_trypop(struct ioqq *ioqq) {
 	size_t i = load(&ioqq->tail, relaxed);
-	atomic uintptr_t *slot = &ioqq->slots[i & ioqq->slot_mask];
+	ioq_slot *slot = &ioqq->slots[i & ioqq->slot_mask];
 
-	uintptr_t prev = load(slot, relaxed);
-	if (!(prev & ~IOQ_BLOCKED)) {
-		return NULL;
-	}
-	if (!compare_exchange_weak(slot, &prev, 0, acquire, relaxed)) {
-		return NULL;
+	struct ioq_ent *ret = ioq_slot_pop(ioqq, slot, false);
+	if (ret) {
+		size_t j = exchange(&ioqq->tail, i + IOQ_STRIDE, relaxed);
+		bfs_assert(j == i, "Detected multiple consumers");
+		(void)j;
 	}
 
-	if (prev & IOQ_BLOCKED) {
-		ioqq_wake(ioqq, i);
-	}
-	prev &= ~IOQ_BLOCKED;
-
-	size_t j = exchange(&ioqq->tail, i + IOQ_STRIDE, relaxed);
-	bfs_assert(j == i, "ioqq_trypop() only supports a single consumer");
-	(void)j;
-
-	return (struct ioq_ent *)prev;
+	return ret;
 }
 
 /** Sentinel stop command. */
-- 
cgit v1.2.3


From d9fb2e5e76e1e2de63af78f4de308c3de577c90b Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 12 Jul 2023 13:12:27 -0400
Subject: ioq: Try harder to avoid setting IOQ_BLOCKED

---
 src/ioq.c | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index c66ebda..a394e07 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -148,20 +148,30 @@ static struct ioqq *ioqq_create(size_t size) {
 
 /** Atomically wait for a slot to change. */
 static uintptr_t ioq_slot_wait(struct ioqq *ioqq, ioq_slot *slot, uintptr_t value) {
-	fetch_or(slot, IOQ_BLOCKED, relaxed);
-	value |= IOQ_BLOCKED;
-
 	size_t i = slot - ioqq->slots;
 	struct ioq_monitor *monitor = &ioqq->monitors[i & ioqq->monitor_mask];
 	mutex_lock(&monitor->mutex);
 
-	uintptr_t ret;
-	while ((ret = load(slot, relaxed)) == value) {
+	uintptr_t ret = load(slot, relaxed);
+	if (ret != value) {
+		goto done;
+	}
+
+	if (!(value & IOQ_BLOCKED)) {
+		value |= IOQ_BLOCKED;
+		if (!compare_exchange_strong(slot, &ret, value, relaxed, relaxed)) {
+			goto done;
+		}
+	}
+
+	do {
 		// To avoid missed wakeups, it is important that
 		// cond_broadcast() is not called right here
 		cond_wait(&monitor->cond, &monitor->mutex);
-	}
+		ret = load(slot, relaxed);
+	} while (ret == value);
 
+done:
 	mutex_unlock(&monitor->mutex);
 	return ret;
 }
-- 
cgit v1.2.3


From 43a59c487209293ee48b8d1f0215ceb643af8ad3 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Tue, 8 Aug 2023 16:03:11 -0400
Subject: ioq: New ioq_slot_monitor() helper

---
 src/ioq.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index a394e07..f7ca8c6 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -146,10 +146,15 @@ static struct ioqq *ioqq_create(size_t size) {
 	return ioqq;
 }
 
+/** Get the monitor associated with a slot. */
+static struct ioq_monitor *ioq_slot_monitor(struct ioqq *ioqq, ioq_slot *slot) {
+	size_t i = slot - ioqq->slots;
+	return &ioqq->monitors[i & ioqq->monitor_mask];
+}
+
 /** Atomically wait for a slot to change. */
 static uintptr_t ioq_slot_wait(struct ioqq *ioqq, ioq_slot *slot, uintptr_t value) {
-	size_t i = slot - ioqq->slots;
-	struct ioq_monitor *monitor = &ioqq->monitors[i & ioqq->monitor_mask];
+	struct ioq_monitor *monitor = ioq_slot_monitor(ioqq, slot);
 	mutex_lock(&monitor->mutex);
 
 	uintptr_t ret = load(slot, relaxed);
@@ -178,8 +183,7 @@ done:
 
 /** Wake up any threads waiting on a slot. */
 static void ioq_slot_wake(struct ioqq *ioqq, ioq_slot *slot) {
-	size_t i = slot - ioqq->slots;
-	struct ioq_monitor *monitor = &ioqq->monitors[i & ioqq->monitor_mask];
+	struct ioq_monitor *monitor = ioq_slot_monitor(ioqq, slot);
 
 	// The following implementation would clearly avoid the missed wakeup
 	// issue mentioned above in ioq_slot_wait():
-- 
cgit v1.2.3


From 52de184ba28551734e1cb13233588504ab5f62ec Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 27 Sep 2023 12:11:15 -0400
Subject: Formatting fixes

---
 src/color.c     |  6 +++---
 src/ctx.c       |  1 -
 src/darray.c    |  4 ++--
 src/diag.c      |  8 ++++----
 src/diag.h      |  2 +-
 src/dstring.c   |  4 ++--
 src/eval.c      | 15 +++++++--------
 src/exec.c      | 16 ++++++++--------
 src/fsade.c     |  6 +++---
 src/ioq.c       |  2 +-
 src/opt.c       | 11 +++++------
 src/opt.h       |  1 -
 src/parse.c     | 45 ++++++++++++++++++++++-----------------------
 src/printf.c    | 46 +++++++++++++++++++++++-----------------------
 src/pwcache.c   |  1 -
 src/stat.c      |  2 +-
 src/trie.c      |  2 +-
 src/trie.h      |  4 ++--
 src/xregex.c    |  2 +-
 src/xspawn.c    |  2 +-
 src/xtime.c     | 12 ++++++------
 tests/bfstd.c   |  2 +-
 tests/mksock.c  |  2 +-
 tests/trie.c    |  4 ++--
 tests/xtimegm.c |  6 +++---
 25 files changed, 100 insertions(+), 106 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/color.c b/src/color.c
index 8d0b995..788d35d 100644
--- a/src/color.c
+++ b/src/color.c
@@ -1143,11 +1143,11 @@ static int print_expr(CFILE *cfile, const struct bfs_expr *expr, bool verbose) {
 	if (verbose) {
 		double rate = 0.0, time = 0.0;
 		if (expr->evaluations) {
-			rate = 100.0*expr->successes/expr->evaluations;
-			time = (1.0e9*expr->elapsed.tv_sec + expr->elapsed.tv_nsec)/expr->evaluations;
+			rate = 100.0 * expr->successes / expr->evaluations;
+			time = (1.0e9 * expr->elapsed.tv_sec + expr->elapsed.tv_nsec) / expr->evaluations;
 		}
 		if (cbuff(cfile, " [${ylw}%zu${rs}/${ylw}%zu${rs}=${ylw}%g%%${rs}; ${ylw}%gns${rs}]",
-		          expr->successes, expr->evaluations, rate, time)) {
+			    expr->successes, expr->evaluations, rate, time)) {
 			return -1;
 		}
 	}
diff --git a/src/ctx.c b/src/ctx.c
index 3a44e68..9a24a33 100644
--- a/src/ctx.c
+++ b/src/ctx.c
@@ -168,7 +168,6 @@ void bfs_ctx_flush(const struct bfs_ctx *ctx) {
 		} else if (cfile == ctx->cout) {
 			bfs_error(ctx, "(standard output): %m.\n");
 		}
-
 	}
 
 	// Flush the user/group caches, in case the executed command edits the
diff --git a/src/darray.c b/src/darray.c
index 42b8397..3e66a55 100644
--- a/src/darray.c
+++ b/src/darray.c
@@ -55,7 +55,7 @@ void *darray_push(void *da, const void *item, size_t size) {
 	size_t i = header->length++;
 	if (i >= capacity) {
 		capacity *= 2;
-		header = realloc(header, sizeof(*header) + capacity*size);
+		header = realloc(header, sizeof(*header) + capacity * size);
 		if (!header) {
 			// This failure will be detected by darray_check()
 			return da;
@@ -64,7 +64,7 @@ void *darray_push(void *da, const void *item, size_t size) {
 	}
 
 	char *data = darray_data(header);
-	memcpy(data + i*size, item, size);
+	memcpy(data + i * size, item, size);
 	return data;
 }
 
diff --git a/src/diag.c b/src/diag.c
index fa9db39..fa66525 100644
--- a/src/diag.c
+++ b/src/diag.c
@@ -4,9 +4,9 @@
 #include "diag.h"
 #include "alloc.h"
 #include "bfstd.h"
-#include "ctx.h"
 #include "color.h"
 #include "config.h"
+#include "ctx.h"
 #include "dstring.h"
 #include "expr.h"
 #include <errno.h>
@@ -31,14 +31,14 @@ void bfs_perror(const struct bfs_ctx *ctx, const char *str) {
 	bfs_error(ctx, "%s: %m.\n", str);
 }
 
-void bfs_error(const struct bfs_ctx *ctx, const char *format, ...)  {
+void bfs_error(const struct bfs_ctx *ctx, const char *format, ...) {
 	va_list args;
 	va_start(args, format);
 	bfs_verror(ctx, format, args);
 	va_end(args);
 }
 
-bool bfs_warning(const struct bfs_ctx *ctx, const char *format, ...)  {
+bool bfs_warning(const struct bfs_ctx *ctx, const char *format, ...) {
 	va_list args;
 	va_start(args, format);
 	bool ret = bfs_vwarning(ctx, format, args);
@@ -46,7 +46,7 @@ bool bfs_warning(const struct bfs_ctx *ctx, const char *format, ...)  {
 	return ret;
 }
 
-bool bfs_debug(const struct bfs_ctx *ctx, enum debug_flags flag, const char *format, ...)  {
+bool bfs_debug(const struct bfs_ctx *ctx, enum debug_flags flag, const char *format, ...) {
 	va_list args;
 	va_start(args, format);
 	bool ret = bfs_vdebug(ctx, flag, format, args);
diff --git a/src/diag.h b/src/diag.h
index e019db0..fea8847 100644
--- a/src/diag.h
+++ b/src/diag.h
@@ -8,8 +8,8 @@
 #ifndef BFS_DIAG_H
 #define BFS_DIAG_H
 
-#include "ctx.h"
 #include "config.h"
+#include "ctx.h"
 #include <stdarg.h>
 
 /**
diff --git a/src/dstring.c b/src/dstring.c
index ef4e733..f947741 100644
--- a/src/dstring.c
+++ b/src/dstring.c
@@ -169,7 +169,7 @@ char *dstrprintf(const char *format, ...) {
 
 char *dstrvprintf(const char *format, va_list args) {
 	// Guess a capacity to try to avoid reallocating
-	dchar *str = dstralloc(2*strlen(format));
+	dchar *str = dstralloc(2 * strlen(format));
 	if (!str) {
 		return NULL;
 	}
@@ -195,7 +195,7 @@ int dstrcatf(dchar **str, const char *format, ...) {
 int dstrvcatf(dchar **str, const char *format, va_list args) {
 	// Guess a capacity to try to avoid calling vsnprintf() twice
 	size_t len = dstrlen(*str);
-	dstreserve(str, len + 2*strlen(format));
+	dstreserve(str, len + 2 * strlen(format));
 	size_t cap = dstrheader(*str)->capacity;
 
 	va_list copy;
diff --git a/src/eval.c b/src/eval.c
index 3550751..9f4896a 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -239,7 +239,7 @@ bool eval_time(const struct bfs_expr *expr, struct bfs_eval *state) {
 	time_t diff = timespec_diff(&expr->reftime, time);
 	switch (expr->time_unit) {
 	case BFS_DAYS:
-		diff /= 60*24;
+		diff /= 60 * 24;
 		fallthru;
 	case BFS_MINUTES:
 		diff /= 60;
@@ -271,7 +271,7 @@ bool eval_used(const struct bfs_expr *expr, struct bfs_eval *state) {
 		return false;
 	}
 
-	long long day_seconds = 60*60*24;
+	long long day_seconds = 60 * 60 * 24;
 	diff = (diff + day_seconds - 1) / day_seconds;
 	return bfs_expr_cmp(expr, diff);
 }
@@ -685,7 +685,7 @@ bool eval_fls(const struct bfs_expr *expr, struct bfs_eval *state) {
 
 	uintmax_t ino = statbuf->ino;
 	uintmax_t block_size = ctx->posixly_correct ? 512 : 1024;
-	uintmax_t blocks = ((uintmax_t)statbuf->blocks*BFS_STAT_BLKSIZE + block_size - 1)/block_size;
+	uintmax_t blocks = ((uintmax_t)statbuf->blocks * BFS_STAT_BLKSIZE + block_size - 1) / block_size;
 	char mode[11];
 	xstrmode(statbuf->mode, mode);
 	char acl = bfs_check_acl(ftwbuf) > 0 ? '+' : ' ';
@@ -721,8 +721,8 @@ bool eval_fls(const struct bfs_expr *expr, struct bfs_eval *state) {
 
 	time_t time = statbuf->mtime.tv_sec;
 	time_t now = ctx->now.tv_sec;
-	time_t six_months_ago = now - 6*30*24*60*60;
-	time_t tomorrow = now + 24*60*60;
+	time_t six_months_ago = now - 6 * 30 * 24 * 60 * 60;
+	time_t tomorrow = now + 24 * 60 * 60;
 	struct tm tm;
 	if (xlocaltime(&time, &tm) != 0) {
 		goto error;
@@ -823,7 +823,6 @@ bool eval_fprintx(const struct bfs_expr *expr, struct bfs_eval *state) {
 		++path;
 	}
 
-
 	if (fputc('\n', file) == EOF) {
 		goto error;
 	}
@@ -905,7 +904,7 @@ bool eval_size(const struct bfs_expr *expr, struct bfs_eval *state) {
 	};
 
 	off_t scale = scales[expr->size_unit];
-	off_t size = (statbuf->size + scale - 1)/scale; // Round up
+	off_t size = (statbuf->size + scale - 1) / scale; // Round up
 	return bfs_expr_cmp(expr, size);
 }
 
@@ -918,7 +917,7 @@ bool eval_sparse(const struct bfs_expr *expr, struct bfs_eval *state) {
 		return false;
 	}
 
-	blkcnt_t expected = (statbuf->size + BFS_STAT_BLKSIZE - 1)/BFS_STAT_BLKSIZE;
+	blkcnt_t expected = (statbuf->size + BFS_STAT_BLKSIZE - 1) / BFS_STAT_BLKSIZE;
 	return statbuf->blocks < expected;
 }
 
diff --git a/src/exec.c b/src/exec.c
index 7b55522..0e0d585 100644
--- a/src/exec.c
+++ b/src/exec.c
@@ -5,9 +5,9 @@
 #include "alloc.h"
 #include "bfstd.h"
 #include "bftw.h"
-#include "ctx.h"
 #include "color.h"
 #include "config.h"
+#include "ctx.h"
 #include "diag.h"
 #include "dstring.h"
 #include "xspawn.h"
@@ -348,7 +348,7 @@ static int bfs_exec_spawn(const struct bfs_exec *execbuf) {
 
 	if (execbuf->flags & BFS_EXEC_MULTI) {
 		bfs_exec_debug(execbuf, "Executing '%s' ... [%zu arguments] (size %zu)\n",
-		               execbuf->argv[0], execbuf->argc - 1, execbuf->arg_size);
+			execbuf->argv[0], execbuf->argc - 1, execbuf->arg_size);
 	} else {
 		bfs_exec_debug(execbuf, "Executing '%s' ... [%zu arguments]\n", execbuf->argv[0], execbuf->argc - 1);
 	}
@@ -471,7 +471,7 @@ static bool bfs_exec_args_remain(const struct bfs_exec *execbuf) {
 static size_t bfs_exec_estimate_max(const struct bfs_exec *execbuf) {
 	size_t min = execbuf->arg_min;
 	size_t max = execbuf->arg_max;
-	return min + (max - min)/2;
+	return min + (max - min) / 2;
 }
 
 /** Update the ARG_MAX lower bound from a successful execution. */
@@ -486,7 +486,7 @@ static void bfs_exec_update_min(struct bfs_exec *execbuf) {
 
 		size_t estimate = bfs_exec_estimate_max(execbuf);
 		bfs_exec_debug(execbuf, "ARG_MAX between [%zu, %zu], trying %zu\n",
-		               execbuf->arg_min, execbuf->arg_max, estimate);
+			execbuf->arg_min, execbuf->arg_max, estimate);
 	}
 }
 
@@ -502,7 +502,7 @@ static size_t bfs_exec_update_max(struct bfs_exec *execbuf) {
 
 	// Trim a fraction off the max size to avoid repeated failures near the
 	// top end of the working range
-	size -= size/16;
+	size -= size / 16;
 	if (size < execbuf->arg_max) {
 		execbuf->arg_max = size;
 
@@ -515,7 +515,7 @@ static size_t bfs_exec_update_max(struct bfs_exec *execbuf) {
 	// Binary search for a more precise bound
 	size_t estimate = bfs_exec_estimate_max(execbuf);
 	bfs_exec_debug(execbuf, "ARG_MAX between [%zu, %zu], trying %zu\n",
-	               execbuf->arg_min, execbuf->arg_max, estimate);
+		execbuf->arg_min, execbuf->arg_max, estimate);
 	return estimate;
 }
 
@@ -589,7 +589,7 @@ static bool bfs_exec_would_overflow(const struct bfs_exec *execbuf, const char *
 	size_t next_size = execbuf->arg_size + bfs_exec_arg_size(arg);
 	if (next_size > arg_max) {
 		bfs_exec_debug(execbuf, "Command size (%zu) would exceed maximum (%zu), executing buffered command\n",
-		               next_size, arg_max);
+			next_size, arg_max);
 		return true;
 	}
 
@@ -601,7 +601,7 @@ static int bfs_exec_push(struct bfs_exec *execbuf, char *arg) {
 	execbuf->argv[execbuf->argc] = arg;
 
 	if (execbuf->argc + 1 >= execbuf->argv_cap) {
-		size_t cap = 2*execbuf->argv_cap;
+		size_t cap = 2 * execbuf->argv_cap;
 		char **argv = realloc(execbuf->argv, sizeof_array(char *, cap));
 		if (!argv) {
 			return -1;
diff --git a/src/fsade.c b/src/fsade.c
index 8dec5a8..cbff47b 100644
--- a/src/fsade.c
+++ b/src/fsade.c
@@ -3,9 +3,9 @@
 
 #include "fsade.h"
 #include "atomic.h"
-#include "config.h"
 #include "bfstd.h"
 #include "bftw.h"
+#include "config.h"
 #include "dir.h"
 #include "dstring.h"
 #include "sanity.h"
@@ -292,7 +292,7 @@ int bfs_check_xattrs(const struct BFTW *ftwbuf) {
 	ssize_t len;
 
 #if BFS_USE_SYS_EXTATTR_H
-	ssize_t (*extattr_list)(const char *, int, void*, size_t) =
+	ssize_t (*extattr_list)(const char *, int, void *, size_t) =
 		ftwbuf->type == BFS_LNK ? extattr_list_link : extattr_list_file;
 
 	len = extattr_list(path, EXTATTR_NAMESPACE_SYSTEM, NULL, 0);
@@ -331,7 +331,7 @@ int bfs_check_xattr_named(const struct BFTW *ftwbuf, const char *name) {
 	ssize_t len;
 
 #if BFS_USE_SYS_EXTATTR_H
-	ssize_t (*extattr_get)(const char *, int, const char *, void*, size_t) =
+	ssize_t (*extattr_get)(const char *, int, const char *, void *, size_t) =
 		ftwbuf->type == BFS_LNK ? extattr_get_link : extattr_get_file;
 
 	len = extattr_get(path, EXTATTR_NAMESPACE_SYSTEM, name, NULL, 0);
diff --git a/src/ioq.c b/src/ioq.c
index f7ca8c6..d3ba2de 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -9,8 +9,8 @@
 #include "config.h"
 #include "diag.h"
 #include "dir.h"
-#include "thread.h"
 #include "sanity.h"
+#include "thread.h"
 #include <assert.h>
 #include <errno.h>
 #include <pthread.h>
diff --git a/src/opt.c b/src/opt.c
index 14de081..77c2798 100644
--- a/src/opt.c
+++ b/src/opt.c
@@ -535,8 +535,8 @@ static struct bfs_expr *optimize_and_expr(const struct opt_state *state, struct
 	expr->pure = lhs->pure && rhs->pure;
 	expr->always_true = lhs->always_true && rhs->always_true;
 	expr->always_false = lhs->always_false || rhs->always_false;
-	expr->cost = lhs->cost + lhs->probability*rhs->cost;
-	expr->probability = lhs->probability*rhs->probability;
+	expr->cost = lhs->cost + lhs->probability * rhs->cost;
+	expr->probability = lhs->probability * rhs->probability;
 
 	return expr;
 }
@@ -606,8 +606,8 @@ static struct bfs_expr *optimize_or_expr(const struct opt_state *state, struct b
 	expr->pure = lhs->pure && rhs->pure;
 	expr->always_true = lhs->always_true || rhs->always_true;
 	expr->always_false = lhs->always_false && rhs->always_false;
-	expr->cost = lhs->cost + (1 - lhs->probability)*rhs->cost;
-	expr->probability = lhs->probability + rhs->probability - lhs->probability*rhs->probability;
+	expr->cost = lhs->cost + (1 - lhs->probability) * rhs->cost;
+	expr->probability = lhs->probability + rhs->probability - lhs->probability * rhs->probability;
 
 	return expr;
 }
@@ -1078,7 +1078,6 @@ static const struct {
 	{eval_xattrname, 0.01},
 };
 
-
 /**
  * Table of simple predicates.
  */
@@ -1331,7 +1330,7 @@ static bool reorder_expr_recursive(const struct opt_state *state, struct bfs_exp
 	if (expr->eval_fn == eval_and || expr->eval_fn == eval_or) {
 		if (lhs->pure && rhs->pure) {
 			float rhs_prob = expr->eval_fn == eval_and ? rhs->probability : 1.0 - rhs->probability;
-			float swapped_cost = rhs->cost + rhs_prob*lhs->cost;
+			float swapped_cost = rhs->cost + rhs_prob * lhs->cost;
 			ret |= reorder_expr(state, expr, swapped_cost);
 		}
 	}
diff --git a/src/opt.h b/src/opt.h
index 28cadb9..4aac129 100644
--- a/src/opt.h
+++ b/src/opt.h
@@ -21,4 +21,3 @@ struct bfs_ctx;
 int bfs_optimize(struct bfs_ctx *ctx);
 
 #endif // BFS_OPT_H
-
diff --git a/src/parse.c b/src/parse.c
index 3416d9e..7766a7b 100644
--- a/src/parse.c
+++ b/src/parse.c
@@ -41,8 +41,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/time.h>
 #include <sys/stat.h>
+#include <sys/time.h>
 #include <sys/wait.h>
 #include <time.h>
 #include <unistd.h>
@@ -1797,10 +1797,10 @@ static int parse_reftime(const struct parser_state *state, struct bfs_expr *expr
 #else
 	int gmtoff = -timezone;
 #endif
-	int tz_hour = gmtoff/3600;
-	int tz_min = (labs(gmtoff)/60)%60;
+	int tz_hour = gmtoff / 3600;
+	int tz_min = (labs(gmtoff) / 60) % 60;
 	fprintf(stderr, "  - %04d-%02d-%02dT%02d:%02d:%02d%+03d:%02d\n",
-	        year, month, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, tz_hour, tz_min);
+		year, month, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, tz_hour, tz_min);
 
 	if (xgmtime(&state->now.tv_sec, &tm) != 0) {
 		parse_perror(state, "xgmtime()");
@@ -1832,8 +1832,8 @@ static struct bfs_expr *parse_newerxy(struct parser_state *state, int arg1, int
 	expr->stat_field = parse_newerxy_field(arg[6]);
 	if (!expr->stat_field) {
 		parse_expr_error(state, expr,
-		                 "For ${blu}-newer${bld}XY${rs}, ${bld}X${rs} should be ${bld}a${rs}, ${bld}c${rs}, ${bld}m${rs}, or ${bld}B${rs}, not ${err}%c${rs}.\n",
-		                 arg[6]);
+			"For ${blu}-newer${bld}XY${rs}, ${bld}X${rs} should be ${bld}a${rs}, ${bld}c${rs}, ${bld}m${rs}, or ${bld}B${rs}, not ${err}%c${rs}.\n",
+			arg[6]);
 		goto fail;
 	}
 
@@ -1845,8 +1845,8 @@ static struct bfs_expr *parse_newerxy(struct parser_state *state, int arg1, int
 		enum bfs_stat_field field = parse_newerxy_field(arg[7]);
 		if (!field) {
 			parse_expr_error(state, expr,
-			                 "For ${blu}-newer${bld}XY${rs}, ${bld}Y${rs} should be ${bld}a${rs}, ${bld}c${rs}, ${bld}m${rs}, ${bld}B${rs}, or ${bld}t${rs}, not ${err}%c${rs}.\n",
-			                 arg[7]);
+				"For ${blu}-newer${bld}XY${rs}, ${bld}Y${rs} should be ${bld}a${rs}, ${bld}c${rs}, ${bld}m${rs}, ${bld}B${rs}, or ${bld}t${rs}, not ${err}%c${rs}.\n",
+				arg[7]);
 			goto fail;
 		}
 
@@ -1855,7 +1855,6 @@ static struct bfs_expr *parse_newerxy(struct parser_state *state, int arg1, int
 			goto fail;
 		}
 
-
 		const struct timespec *reftime = bfs_stat_time(&sb, field);
 		if (!reftime) {
 			parse_expr_error(state, expr, "Couldn't get file %s.\n", bfs_stat_field_name(field));
@@ -1905,7 +1904,7 @@ static struct bfs_expr *parse_nohidden(struct parser_state *state, int arg1, int
  */
 static struct bfs_expr *parse_noleaf(struct parser_state *state, int arg1, int arg2) {
 	parse_warning(state, "${ex}%s${rs} does not apply the optimization that ${blu}%s${rs} inhibits.\n\n",
-	              BFS_COMMAND, state->argv[0]);
+		BFS_COMMAND, state->argv[0]);
 	return parse_nullary_option(state);
 }
 
@@ -2730,12 +2729,12 @@ static struct bfs_expr *parse_help(struct parser_state *state, int arg1, int arg
 	}
 
 	cfprintf(cout, "Usage: ${ex}%s${rs} [${cyn}flags${rs}...] [${mag}paths${rs}...] [${blu}expression${rs}...]\n\n",
-		 state->command);
+		state->command);
 
 	cfprintf(cout, "${ex}%s${rs} is compatible with ${ex}find${rs}, with some extensions. "
-		       "${cyn}Flags${rs} (${cyn}-H${rs}/${cyn}-L${rs}/${cyn}-P${rs} etc.), ${mag}paths${rs},\n"
-		       "and ${blu}expressions${rs} may be freely mixed in any order.\n\n",
-		 BFS_COMMAND);
+	               "${cyn}Flags${rs} (${cyn}-H${rs}/${cyn}-L${rs}/${cyn}-P${rs} etc.), ${mag}paths${rs},\n"
+	               "and ${blu}expressions${rs} may be freely mixed in any order.\n\n",
+		BFS_COMMAND);
 
 	cfprintf(cout, "${bld}Flags:${rs}\n\n");
 
@@ -2807,7 +2806,7 @@ static struct bfs_expr *parse_help(struct parser_state *state, int arg1, int arg
 	cfprintf(cout, "  ${blu}-ignore_readdir_race${rs}\n");
 	cfprintf(cout, "  ${blu}-noignore_readdir_race${rs}\n");
 	cfprintf(cout, "      Whether to report an error if ${ex}%s${rs} detects that the file tree is modified\n",
-	         BFS_COMMAND);
+		BFS_COMMAND);
 	cfprintf(cout, "      during the search (default: ${blu}-noignore_readdir_race${rs})\n");
 	cfprintf(cout, "  ${blu}-maxdepth${rs} ${bld}N${rs}\n");
 	cfprintf(cout, "  ${blu}-mindepth${rs} ${bld}N${rs}\n");
@@ -3462,14 +3461,14 @@ static struct bfs_expr *parse_whole_expr(struct parser_state *state) {
 
 	if (state->mount_arg && state->xdev_arg) {
 		parse_conflict_warning(state, state->mount_arg, 1, state->xdev_arg, 1,
-		                       "${blu}%s${rs} is redundant in the presence of ${blu}%s${rs}.\n\n",
-		                       state->xdev_arg[0], state->mount_arg[0]);
+			"${blu}%s${rs} is redundant in the presence of ${blu}%s${rs}.\n\n",
+			state->xdev_arg[0], state->mount_arg[0]);
 	}
 
 	if (state->ctx->warn && state->depth_arg && state->prune_arg) {
 		parse_conflict_warning(state, state->depth_arg, 1, state->prune_arg, 1,
-		                       "${blu}%s${rs} does not work in the presence of ${blu}%s${rs}.\n",
-		                       state->prune_arg[0], state->depth_arg[0]);
+			"${blu}%s${rs} does not work in the presence of ${blu}%s${rs}.\n",
+			state->prune_arg[0], state->depth_arg[0]);
 
 		if (state->interactive) {
 			bfs_warning(state->ctx, "Do you want to continue? ");
@@ -3483,8 +3482,8 @@ static struct bfs_expr *parse_whole_expr(struct parser_state *state) {
 
 	if (state->ok_expr && state->files0_stdin_arg) {
 		parse_conflict_error(state, state->ok_expr->argv, state->ok_expr->argc, state->files0_stdin_arg, 2,
-		                     "${blu}%s${rs} conflicts with ${blu}%s${rs} ${bld}%s${rs}.\n",
-		                     state->ok_expr->argv[0], state->files0_stdin_arg[0], state->files0_stdin_arg[1]);
+			"${blu}%s${rs} conflicts with ${blu}%s${rs} ${bld}%s${rs}.\n",
+			state->ok_expr->argv[0], state->files0_stdin_arg[0], state->files0_stdin_arg[1]);
 		goto fail;
 	}
 
@@ -3644,7 +3643,7 @@ void bfs_ctx_dump(const struct bfs_ctx *ctx, enum debug_flags flag) {
 static void dump_costs(const struct bfs_ctx *ctx) {
 	const struct bfs_expr *expr = ctx->expr;
 	bfs_debug(ctx, DEBUG_COST, "       Cost: ~${ylw}%g${rs}\n", expr->cost);
-	bfs_debug(ctx, DEBUG_COST, "Probability: ~${ylw}%g%%${rs}\n", 100.0*expr->probability);
+	bfs_debug(ctx, DEBUG_COST, "Probability: ~${ylw}%g%%${rs}\n", 100.0 * expr->probability);
 }
 
 struct bfs_ctx *bfs_parse_cmdline(int argc, char *argv[]) {
@@ -3654,7 +3653,7 @@ struct bfs_ctx *bfs_parse_cmdline(int argc, char *argv[]) {
 		goto fail;
 	}
 
-	static char* default_argv[] = {BFS_COMMAND, NULL};
+	static char *default_argv[] = {BFS_COMMAND, NULL};
 	if (argc < 1) {
 		argc = 1;
 		argv = default_argv;
diff --git a/src/printf.c b/src/printf.c
index 5de5a28..98bcb0f 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -113,14 +113,14 @@ static int bfs_printf_ctime(CFILE *cfile, const struct bfs_printf *directive, co
 	}
 
 	BFS_PRINTF_BUF(buf, "%s %s %2d %.2d:%.2d:%.2d.%09ld0 %4d",
-	               days[tm.tm_wday],
-	               months[tm.tm_mon],
-	               tm.tm_mday,
-	               tm.tm_hour,
-	               tm.tm_min,
-	               tm.tm_sec,
-	               (long)ts->tv_nsec,
-	               1900 + tm.tm_year);
+		days[tm.tm_wday],
+		months[tm.tm_mon],
+		tm.tm_mday,
+		tm.tm_hour,
+		tm.tm_min,
+		tm.tm_sec,
+		(long)ts->tv_nsec,
+		1900 + tm.tm_year);
 
 	return dyn_fprintf(cfile->file, directive, buf);
 }
@@ -152,19 +152,19 @@ static int bfs_printf_strftime(CFILE *cfile, const struct bfs_printf *directive,
 		break;
 	case '+':
 		ret = snprintf(buf, sizeof(buf), "%4d-%.2d-%.2d+%.2d:%.2d:%.2d.%09ld0",
-		               1900 + tm.tm_year,
-		               tm.tm_mon + 1,
-		               tm.tm_mday,
-		               tm.tm_hour,
-		               tm.tm_min,
-		               tm.tm_sec,
-		               (long)ts->tv_nsec);
+			1900 + tm.tm_year,
+			tm.tm_mon + 1,
+			tm.tm_mday,
+			tm.tm_hour,
+			tm.tm_min,
+			tm.tm_sec,
+			(long)ts->tv_nsec);
 		break;
 	case 'k':
 		ret = snprintf(buf, sizeof(buf), "%2d", tm.tm_hour);
 		break;
 	case 'l':
-		ret = snprintf(buf, sizeof(buf), "%2d", (tm.tm_hour + 11)%12 + 1);
+		ret = snprintf(buf, sizeof(buf), "%2d", (tm.tm_hour + 11) % 12 + 1);
 		break;
 	case 's':
 		ret = snprintf(buf, sizeof(buf), "%lld", (long long)ts->tv_sec);
@@ -174,10 +174,10 @@ static int bfs_printf_strftime(CFILE *cfile, const struct bfs_printf *directive,
 		break;
 	case 'T':
 		ret = snprintf(buf, sizeof(buf), "%.2d:%.2d:%.2d.%09ld0",
-			       tm.tm_hour,
-			       tm.tm_min,
-			       tm.tm_sec,
-			       (long)ts->tv_nsec);
+			tm.tm_hour,
+			tm.tm_min,
+			tm.tm_sec,
+			(long)ts->tv_nsec);
 		break;
 
 	// POSIX strftime() features
@@ -202,7 +202,7 @@ static int bfs_printf_b(CFILE *cfile, const struct bfs_printf *directive, const
 		return -1;
 	}
 
-	uintmax_t blocks = ((uintmax_t)statbuf->blocks*BFS_STAT_BLKSIZE + 511)/512;
+	uintmax_t blocks = ((uintmax_t)statbuf->blocks * BFS_STAT_BLKSIZE + 511) / 512;
 	BFS_PRINTF_BUF(buf, "%ju", blocks);
 	return dyn_fprintf(cfile->file, directive, buf);
 }
@@ -338,7 +338,7 @@ static int bfs_printf_k(CFILE *cfile, const struct bfs_printf *directive, const
 		return -1;
 	}
 
-	uintmax_t blocks = ((uintmax_t)statbuf->blocks*BFS_STAT_BLKSIZE + 1023)/1024;
+	uintmax_t blocks = ((uintmax_t)statbuf->blocks * BFS_STAT_BLKSIZE + 1023) / 1024;
 	BFS_PRINTF_BUF(buf, "%ju", blocks);
 	return dyn_fprintf(cfile->file, directive, buf);
 }
@@ -452,7 +452,7 @@ static int bfs_printf_S(CFILE *cfile, const struct bfs_printf *directive, const
 	if (statbuf->size == 0 && statbuf->blocks == 0) {
 		sparsity = 1.0;
 	} else {
-		sparsity = (double)BFS_STAT_BLKSIZE*statbuf->blocks/statbuf->size;
+		sparsity = (double)BFS_STAT_BLKSIZE * statbuf->blocks / statbuf->size;
 	}
 	return dyn_fprintf(cfile->file, directive, sparsity);
 }
diff --git a/src/pwcache.c b/src/pwcache.c
index 0e2f5c1..c728ba9 100644
--- a/src/pwcache.c
+++ b/src/pwcache.c
@@ -4,7 +4,6 @@
 #include "pwcache.h"
 #include "alloc.h"
 #include "config.h"
-#include "darray.h"
 #include "trie.h"
 #include <errno.h>
 #include <grp.h>
diff --git a/src/stat.c b/src/stat.c
index e8f48ee..d7387c6 100644
--- a/src/stat.c
+++ b/src/stat.c
@@ -10,8 +10,8 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <string.h>
-#include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/types.h>
 
 #if defined(STATX_BASIC_STATS) && (!__ANDROID__ || __ANDROID_API__ >= 30)
 #  define BFS_HAS_LIBC_STATX true
diff --git a/src/trie.c b/src/trie.c
index 77aa2d0..55544e6 100644
--- a/src/trie.c
+++ b/src/trie.c
@@ -700,7 +700,7 @@ static void trie_remove_impl(struct trie *trie, struct trie_leaf *leaf) {
 	}
 
 	if (child_index < parent_size) {
-		memmove(child, child + 1, (parent_size - child_index)*sizeof(*child));
+		memmove(child, child + 1, (parent_size - child_index) * sizeof(*child));
 	}
 
 	if (has_single_bit(parent_size)) {
diff --git a/src/trie.h b/src/trie.h
index 2f51db5..02088f1 100644
--- a/src/trie.h
+++ b/src/trie.h
@@ -4,8 +4,8 @@
 #ifndef BFS_TRIE_H
 #define BFS_TRIE_H
 
-#include "config.h"
 #include "alloc.h"
+#include "config.h"
 #include "list.h"
 #include <stddef.h>
 #include <stdint.h>
@@ -143,6 +143,6 @@ void trie_destroy(struct trie *trie);
  * Iterate over the leaves of a trie.
  */
 #define for_trie(leaf, trie) \
-	for_list(struct trie_leaf, leaf, trie)
+	for_list (struct trie_leaf, leaf, trie)
 
 #endif // BFS_TRIE_H
diff --git a/src/xregex.c b/src/xregex.c
index beb6676..b9c04bf 100644
--- a/src/xregex.c
+++ b/src/xregex.c
@@ -5,8 +5,8 @@
 #include "alloc.h"
 #include "config.h"
 #include "diag.h"
-#include "thread.h"
 #include "sanity.h"
+#include "thread.h"
 #include <errno.h>
 #include <pthread.h>
 #include <stdlib.h>
diff --git a/src/xspawn.c b/src/xspawn.c
index 80bafef..7fb63e0 100644
--- a/src/xspawn.c
+++ b/src/xspawn.c
@@ -187,7 +187,7 @@ fail:
 
 	// In case of a write error, the parent will still see that we exited
 	// unsuccessfully, but won't know why
-	(void) xwrite(pipefd[1], &error, sizeof(error));
+	(void)xwrite(pipefd[1], &error, sizeof(error));
 
 	xclose(pipefd[1]);
 	_Exit(127);
diff --git a/src/xtime.c b/src/xtime.c
index 79dafad..e90bdb1 100644
--- a/src/xtime.c
+++ b/src/xtime.c
@@ -81,7 +81,7 @@ static int safe_add(int *value, int delta) {
 
 static int floor_div(int n, int d) {
 	int a = n < 0;
-	return (n + a)/d - a;
+	return (n + a) / d - a;
 }
 
 static int wrap(int *value, int max, int *next) {
@@ -93,7 +93,7 @@ static int wrap(int *value, int max, int *next) {
 static int month_length(int year, int month) {
 	static const int month_lengths[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
 	int ret = month_lengths[month];
-	if (month == 1 && year%4 == 0 && (year%100 != 0 || (year + 300)%400 == 0)) {
+	if (month == 1 && year % 4 == 0 && (year % 100 != 0 || (year + 300) % 400 == 0)) {
 		++ret;
 	}
 	return ret;
@@ -156,13 +156,13 @@ int xtimegm(struct tm *tm, time_t *timep) {
 		leap_days = floor_div(tm->tm_year + 3, 4) - floor_div(tm->tm_year + 99, 100) + floor_div(tm->tm_year + 299, 400) - 17;
 	}
 
-	long long epoch_days = 365LL*(tm->tm_year - 70) + leap_days + tm->tm_yday;
-	tm->tm_wday = (epoch_days + 4)%7;
+	long long epoch_days = 365LL * (tm->tm_year - 70) + leap_days + tm->tm_yday;
+	tm->tm_wday = (epoch_days + 4) % 7;
 	if (tm->tm_wday < 0) {
 		tm->tm_wday += 7;
 	}
 
-	long long epoch_time = tm->tm_sec + 60*(tm->tm_min + 60*(tm->tm_hour + 24*epoch_days));
+	long long epoch_time = tm->tm_sec + 60 * (tm->tm_min + 60 * (tm->tm_hour + 24 * epoch_days));
 	*timep = (time_t)epoch_time;
 	if ((long long)*timep != epoch_time) {
 		goto overflow;
@@ -296,7 +296,7 @@ end:
 			goto error;
 		}
 
-		int offset = 60*tz_hour + tz_min;
+		int offset = 60 * tz_hour + tz_min;
 		if (tz_negative) {
 			result->tv_sec -= offset;
 		} else {
diff --git a/tests/bfstd.c b/tests/bfstd.c
index 83964e5..33b3792 100644
--- a/tests/bfstd.c
+++ b/tests/bfstd.c
@@ -7,8 +7,8 @@
 #include <errno.h>
 #include <langinfo.h>
 #include <locale.h>
-#include <stdio.h>
 #include <stdint.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
diff --git a/tests/mksock.c b/tests/mksock.c
index 42ef322..7023b4f 100644
--- a/tests/mksock.c
+++ b/tests/mksock.c
@@ -9,8 +9,8 @@
 #include "../src/bfstd.h"
 #include <errno.h>
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #include <sys/socket.h>
 #include <sys/un.h>
 #include <unistd.h>
diff --git a/tests/trie.c b/tests/trie.c
index 6ea94a2..656fd85 100644
--- a/tests/trie.c
+++ b/tests/trie.c
@@ -120,11 +120,11 @@ int main(void) {
 	bfs_verify(!trie_find_mem(&trie, longstr, longsize));
 	bfs_verify(trie_insert_mem(&trie, longstr, longsize));
 
-	memset(longstr + longsize/2, 0xAB, longsize/2);
+	memset(longstr + longsize / 2, 0xAB, longsize / 2);
 	bfs_verify(!trie_find_mem(&trie, longstr, longsize));
 	bfs_verify(trie_insert_mem(&trie, longstr, longsize));
 
-	memset(longstr, 0xAA, longsize/2);
+	memset(longstr, 0xAA, longsize / 2);
 	bfs_verify(!trie_find_mem(&trie, longstr, longsize));
 	bfs_verify(trie_insert_mem(&trie, longstr, longsize));
 
diff --git a/tests/xtimegm.c b/tests/xtimegm.c
index b2479b7..973b2eb 100644
--- a/tests/xtimegm.c
+++ b/tests/xtimegm.c
@@ -42,9 +42,9 @@ static bool tm_equal(const struct tm *tma, const struct tm *tmb) {
 
 static void tm_print(FILE *file, const struct tm *tm) {
 	fprintf(file, "Y%d M%d D%d  h%d m%d s%d  wd%d yd%d%s\n",
-	        tm->tm_year, tm->tm_mon, tm->tm_mday,
-	        tm->tm_hour, tm->tm_min, tm->tm_sec,
-	        tm->tm_wday, tm->tm_yday,
+		tm->tm_year, tm->tm_mon, tm->tm_mday,
+		tm->tm_hour, tm->tm_min, tm->tm_sec,
+		tm->tm_wday, tm->tm_yday,
 		tm->tm_isdst ? (tm->tm_isdst < 0 ? " (DST?)" : " (DST)") : "");
 }
 
-- 
cgit v1.2.3


From 1afa241472709b32baf5e3e1fd3ba6ebd5fd1bf6 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Tue, 11 Jul 2023 14:04:40 -0400
Subject: ioq: Use io_uring

Closes #65.
---
 src/bftw.c |  23 ++++-
 src/ioq.c  | 307 +++++++++++++++++++++++++++++++++++++++++++++++++++++--------
 2 files changed, 288 insertions(+), 42 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/bftw.c b/src/bftw.c
index 5e5f4a5..902a3fa 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -470,21 +470,34 @@ static int bftw_state_init(struct bftw_state *state, const struct bftw_args *arg
 
 	state->error = 0;
 
-	if (args->nopenfd < 1) {
+	if (args->nopenfd < 2) {
 		errno = EMFILE;
 		return -1;
 	}
-	bftw_cache_init(&state->cache, args->nopenfd);
 
-	state->nthreads = args->nthreads;
-	if (state->nthreads > 0) {
-		state->ioq = ioq_create(4096, state->nthreads);
+	size_t nopenfd = args->nopenfd;
+	size_t qdepth = 4096;
+	size_t nthreads = args->nthreads;
+
+#if BFS_USE_LIBURING
+	// io_uring uses one fd per ring, ioq uses one ring per thread
+	if (nthreads >= nopenfd - 1) {
+		nthreads = nopenfd - 2;
+	}
+	nopenfd -= nthreads;
+#endif
+
+	bftw_cache_init(&state->cache, nopenfd);
+
+	if (nthreads > 0) {
+		state->ioq = ioq_create(qdepth, nthreads);
 		if (!state->ioq) {
 			return -1;
 		}
 	} else {
 		state->ioq = NULL;
 	}
+	state->nthreads = nthreads;
 
 	SLIST_INIT(&state->to_open);
 	SLIST_INIT(&state->to_read);
diff --git a/src/ioq.c b/src/ioq.c
index d3ba2de..04b9c0d 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -16,6 +16,10 @@
 #include <pthread.h>
 #include <stdlib.h>
 
+#if BFS_USE_LIBURING
+#  include <liburing.h>
+#endif
+
 /**
  * A monitor for an I/O queue slot.
  */
@@ -280,6 +284,21 @@ static struct ioq_ent *ioqq_trypop(struct ioqq *ioqq) {
 /** Sentinel stop command. */
 static struct ioq_ent IOQ_STOP;
 
+/** I/O queue thread-specific data. */
+struct ioq_thread {
+	/** The thread handle. */
+	pthread_t id;
+	/** Pointer back to the I/O queue. */
+	struct ioq *parent;
+
+#if BFS_USE_LIBURING
+	/** io_uring instance. */
+	struct io_uring ring;
+	/** Any error that occurred initializing the ring. */
+	int ring_err;
+#endif
+};
+
 struct ioq {
 	/** The depth of the queue. */
 	size_t depth;
@@ -299,60 +318,247 @@ struct ioq {
 	/** The number of background threads. */
 	size_t nthreads;
 	/** The background threads themselves. */
-	pthread_t threads[];
+	struct ioq_thread threads[];
 };
 
-/** Background thread entry point. */
-static void *ioq_work(void *ptr) {
-	struct ioq *ioq = ptr;
+/** Cancel a request if we need to. */
+static bool ioq_check_cancel(struct ioq *ioq, struct ioq_ent *ent) {
+	if (!load(&ioq->cancel, relaxed)) {
+		return false;
+	}
 
-	while (true) {
-		struct ioq_ent *ent = ioqq_pop(ioq->pending);
-		if (ent == &IOQ_STOP) {
-			break;
+	// Always close(), even if we're cancelled, just like a real EINTR
+	if (ent->op == IOQ_CLOSE || ent->op == IOQ_CLOSEDIR) {
+		return false;
+	}
+
+	ent->ret = -1;
+	ent->error = EINTR;
+	ioqq_push(ioq->ready, ent);
+	return true;
+}
+
+/** Handle a single request synchronously. */
+static void ioq_handle(struct ioq *ioq, struct ioq_ent *ent) {
+	int ret;
+
+	switch (ent->op) {
+	case IOQ_CLOSE:
+		ret = xclose(ent->close.fd);
+		break;
+
+	case IOQ_OPENDIR:
+		ret = bfs_opendir(ent->opendir.dir, ent->opendir.dfd, ent->opendir.path);
+		if (ret == 0) {
+			bfs_polldir(ent->opendir.dir);
 		}
+		break;
+
+	case IOQ_CLOSEDIR:
+		ret = bfs_closedir(ent->closedir.dir);
+		break;
+
+	default:
+		bfs_bug("Unknown ioq_op %d", (int)ent->op);
+		ret = -1;
+		errno = ENOSYS;
+		break;
+	}
+
+	ent->ret = ret;
+	ent->error = ret == 0 ? 0 : errno;
+
+	ioqq_push(ioq->ready, ent);
+}
 
-		bool cancel = load(&ioq->cancel, relaxed);
+#if BFS_USE_LIBURING
+/** io_uring worker state. */
+struct ioq_ring_state {
+	/** The I/O queue. */
+	struct ioq *ioq;
+	/** The io_uring. */
+	struct io_uring *ring;
+	/** The current ioq->pending slot. */
+	ioq_slot *slot;
+	/** Number of prepped, unsubmitted SQEs. */
+	size_t prepped;
+	/** Number of submitted, unreaped SQEs. */
+	size_t submitted;
+	/** Whether to stop the loop. */
+	bool stop;
+};
+
+/** Pop a request for ioq_ring_prep(). */
+static struct ioq_ent *ioq_ring_pop(struct ioq_ring_state *state) {
+	if (state->stop) {
+		return NULL;
+	}
+
+	// Advance to the next slot if necessary
+	struct ioq *ioq = state->ioq;
+	if (!state->slot) {
+		state->slot = ioqq_read(ioq->pending);
+	}
+
+	// Block if we have nothing else to do
+	bool block = !state->prepped && !state->submitted;
+	struct ioq_ent *ret = ioq_slot_pop(ioq->pending, state->slot, block);
+
+	if (ret) {
+		// Got an entry, move to the next slot next time
+		state->slot = NULL;
+	}
+
+	if (ret == &IOQ_STOP) {
+		state->stop = true;
+		ret = NULL;
+	}
+
+	return ret;
+}
 
-		ent->ret = -1;
+/** Prep a single SQE. */
+static void ioq_prep_sqe(struct io_uring_sqe *sqe, struct ioq_ent *ent) {
+	switch (ent->op) {
+	case IOQ_CLOSE:
+		io_uring_prep_close(sqe, ent->close.fd);
+		break;
+
+	case IOQ_OPENDIR:
+		io_uring_prep_openat(sqe, ent->opendir.dfd, ent->opendir.path, O_RDONLY | O_CLOEXEC | O_DIRECTORY, 0);
+		break;
+
+#if BFS_USE_UNWRAPDIR
+	case IOQ_CLOSEDIR:
+		io_uring_prep_close(sqe, bfs_unwrapdir(ent->closedir.dir));
+		break;
+#endif
+
+	default:
+		bfs_bug("Unknown ioq_op %d", (int)ent->op);
+		io_uring_prep_nop(sqe);
+		break;
+	}
 
-		switch (ent->op) {
-		case IOQ_CLOSE:
-			// Always close(), even if we're cancelled, just like a real EINTR
-			ent->ret = xclose(ent->close.fd);
+	io_uring_sqe_set_data(sqe, ent);
+}
+
+/** Prep a batch of SQEs. */
+static bool ioq_ring_prep(struct ioq_ring_state *state) {
+	struct ioq *ioq = state->ioq;
+	struct io_uring *ring = state->ring;
+
+	while (io_uring_sq_space_left(ring)) {
+		struct ioq_ent *ent = ioq_ring_pop(state);
+		if (!ent) {
 			break;
+		}
+
+		if (ioq_check_cancel(ioq, ent)) {
+			continue;
+		}
 
-		case IOQ_OPENDIR:
-			if (!cancel) {
-				struct ioq_opendir *args = &ent->opendir;
-				ent->ret = bfs_opendir(args->dir, args->dfd, args->path);
-				if (ent->ret == 0) {
-					bfs_polldir(args->dir);
-				}
+#if !BFS_USE_UNWRAPDIR
+		if (ent->op == IOQ_CLOSEDIR) {
+			ioq_handle(ioq, ent);
+			continue;
+		}
+#endif
+
+		struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
+		ioq_prep_sqe(sqe, ent);
+		++state->prepped;
+	}
+
+	return state->prepped || state->submitted;
+}
+
+/** Reap a batch of SQEs. */
+static void ioq_ring_reap(struct ioq_ring_state *state) {
+	struct ioq *ioq = state->ioq;
+	struct io_uring *ring = state->ring;
+
+	while (state->prepped) {
+		int ret = io_uring_submit_and_wait(ring, 1);
+		if (ret > 0) {
+			state->prepped -= ret;
+			state->submitted += ret;
+		}
+	}
+
+	while (state->submitted) {
+		struct io_uring_cqe *cqe;
+		if (io_uring_wait_cqe(ring, &cqe) < 0) {
+			continue;
+		}
+
+		struct ioq_ent *ent = io_uring_cqe_get_data(cqe);
+		ent->ret = cqe->res >= 0 ? cqe->res : -1;
+		ent->error = cqe->res < 0 ? -cqe->res : 0;
+		io_uring_cqe_seen(ring, cqe);
+		--state->submitted;
+
+		if (ent->op == IOQ_OPENDIR && ent->ret >= 0) {
+			int fd = ent->ret;
+			if (ioq_check_cancel(ioq, ent)) {
+				xclose(fd);
+				continue;
 			}
-			break;
 
-		case IOQ_CLOSEDIR:
-			ent->ret = bfs_closedir(ent->closedir.dir);
-			break;
+			ent->ret = bfs_opendir(ent->opendir.dir, fd, NULL);
+			if (ent->ret == 0) {
+				// TODO: io_uring_prep_getdents()
+				bfs_polldir(ent->opendir.dir);
+			} else {
+				ent->error = errno;
+			}
+		}
+
+		ioqq_push(ioq->ready, ent);
+	}
+}
+
+/** io_uring worker loop. */
+static void ioq_ring_work(struct ioq_thread *thread) {
+	struct ioq_ring_state state = {
+		.ioq = thread->parent,
+		.ring = &thread->ring,
+	};
 
-		default:
-			bfs_bug("Unknown ioq_op %d", (int)ent->op);
-			errno = ENOSYS;
+	while (ioq_ring_prep(&state)) {
+		ioq_ring_reap(&state);
+	}
+}
+#endif
+
+/** Synchronous syscall loop. */
+static void ioq_sync_work(struct ioq_thread *thread) {
+	struct ioq *ioq = thread->parent;
+
+	while (true) {
+		struct ioq_ent *ent = ioqq_pop(ioq->pending);
+		if (ent == &IOQ_STOP) {
 			break;
 		}
 
-		if (cancel) {
-			ent->error = EINTR;
-		} else if (ent->ret < 0) {
-			ent->error = errno;
-		} else {
-			ent->error = 0;
+		if (!ioq_check_cancel(ioq, ent)) {
+			ioq_handle(ioq, ent);
 		}
+	}
+}
 
-		ioqq_push(ioq->ready, ent);
+/** Background thread entry point. */
+static void *ioq_work(void *ptr) {
+	struct ioq_thread *thread = ptr;
+
+#if BFS_USE_LIBURING
+	if (thread->ring_err == 0) {
+		ioq_ring_work(thread);
+		return NULL;
 	}
+#endif
 
+	ioq_sync_work(thread);
 	return NULL;
 }
 
@@ -376,7 +582,30 @@ struct ioq *ioq_create(size_t depth, size_t nthreads) {
 	}
 
 	for (size_t i = 0; i < nthreads; ++i) {
-		if (thread_create(&ioq->threads[i], NULL, ioq_work, ioq) != 0) {
+		struct ioq_thread *thread = &ioq->threads[i];
+		thread->parent = ioq;
+
+#if BFS_USE_LIBURING
+		struct ioq_thread *prev = i ? &ioq->threads[i - 1] : NULL;
+		if (prev && prev->ring_err) {
+			thread->ring_err = prev->ring_err;
+		} else {
+			// Share io-wq workers between rings
+			struct io_uring_params params = {0};
+			if (prev) {
+				params.flags |= IORING_SETUP_ATTACH_WQ;
+				params.wq_fd = prev->ring.ring_fd;
+			}
+
+			size_t entries = depth / nthreads;
+			if (entries < 16) {
+				entries = 16;
+			}
+			thread->ring_err = -io_uring_queue_init_params(entries, &thread->ring, &params);
+		}
+#endif
+
+		if (thread_create(&thread->id, NULL, ioq_work, thread) != 0) {
 			goto fail;
 		}
 		++ioq->nthreads;
@@ -496,7 +725,11 @@ void ioq_destroy(struct ioq *ioq) {
 	ioq_cancel(ioq);
 
 	for (size_t i = 0; i < ioq->nthreads; ++i) {
-		thread_join(ioq->threads[i], NULL);
+		struct ioq_thread *thread = &ioq->threads[i];
+		thread_join(thread->id, NULL);
+#if BFS_USE_LIBURING
+		io_uring_queue_exit(&thread->ring);
+#endif
 	}
 
 	ioqq_destroy(ioq->ready);
-- 
cgit v1.2.3


From 214a1f9215d33d4b9f34a3d258da1e1f4e3eb01f Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Mon, 16 Oct 2023 16:44:46 -0400
Subject: dir: Add a flags parameter to bfs_opendir()

---
 src/bftw.c |  4 ++--
 src/dir.c  | 31 ++++++++++++++++++++-----------
 src/dir.h  | 12 +++++++++++-
 src/eval.c |  6 +++---
 src/ioq.c  |  7 ++++---
 src/ioq.h  |  6 +++++-
 6 files changed, 45 insertions(+), 21 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/bftw.c b/src/bftw.c
index 06a0085..e2f1a66 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -856,7 +856,7 @@ static int bftw_ioq_opendir(struct bftw_state *state, struct bftw_file *file) {
 		goto unpin;
 	}
 
-	if (ioq_opendir(state->ioq, dir, dfd, file->name, file) != 0) {
+	if (ioq_opendir(state->ioq, dir, dfd, file->name, 0, file) != 0) {
 		goto free;
 	}
 
@@ -1018,7 +1018,7 @@ static struct bfs_dir *bftw_file_opendir(struct bftw_state *state, struct bftw_f
 		return NULL;
 	}
 
-	if (bfs_opendir(dir, fd, NULL) != 0) {
+	if (bfs_opendir(dir, fd, NULL, 0) != 0) {
 		bftw_freedir(cache, dir);
 		return NULL;
 	}
diff --git a/src/dir.c b/src/dir.c
index a7423e9..dee02e5 100644
--- a/src/dir.c
+++ b/src/dir.c
@@ -96,18 +96,26 @@ enum bfs_type bfs_mode_to_type(mode_t mode) {
 	}
 }
 
+/**
+ * Private directory flags.
+ */
+enum {
+	/** We've reached the end of the directory. */
+	BFS_DIR_EOF = BFS_DIR_PRIVATE << 0,
+};
+
 struct bfs_dir {
+	unsigned int flags;
+
 #if BFS_USE_GETDENTS
-	alignas(sys_dirent) int fd;
+	int fd;
 	unsigned short pos;
 	unsigned short size;
-	// sys_dirent buf[];
+	alignas(sys_dirent) char buf[];
 #else
 	DIR *dir;
 	struct dirent *de;
 #endif
-
-	bool eof;
 };
 
 #if BFS_USE_GETDENTS
@@ -125,7 +133,7 @@ void bfs_dir_arena(struct arena *arena) {
 	arena_init(arena, alignof(struct bfs_dir), DIR_SIZE);
 }
 
-int bfs_opendir(struct bfs_dir *dir, int at_fd, const char *at_path) {
+int bfs_opendir(struct bfs_dir *dir, int at_fd, const char *at_path, enum bfs_dir_flags flags) {
 	int fd;
 	if (at_path) {
 		fd = openat(at_fd, at_path, O_RDONLY | O_CLOEXEC | O_DIRECTORY);
@@ -139,6 +147,8 @@ int bfs_opendir(struct bfs_dir *dir, int at_fd, const char *at_path) {
 		return -1;
 	}
 
+	dir->flags = flags;
+
 #if BFS_USE_GETDENTS
 	dir->fd = fd;
 	dir->pos = 0;
@@ -154,7 +164,6 @@ int bfs_opendir(struct bfs_dir *dir, int at_fd, const char *at_path) {
 	dir->de = NULL;
 #endif
 
-	dir->eof = false;
 	return 0;
 }
 
@@ -170,14 +179,14 @@ int bfs_polldir(struct bfs_dir *dir) {
 #if BFS_USE_GETDENTS
 	if (dir->pos < dir->size) {
 		return 1;
-	} else if (dir->eof) {
+	} else if (dir->flags & BFS_DIR_EOF) {
 		return 0;
 	}
 
 	char *buf = (char *)(dir + 1);
 	ssize_t size = bfs_getdents(dir->fd, buf, BUF_SIZE);
 	if (size == 0) {
-		dir->eof = true;
+		dir->flags |= BFS_DIR_EOF;
 		return 0;
 	} else if (size < 0) {
 		return -1;
@@ -194,7 +203,7 @@ int bfs_polldir(struct bfs_dir *dir) {
 		if (size > 0) {
 			dir->size += size;
 		} else if (size == 0) {
-			dir->eof = true;
+			dir->flags |= BFS_DIR_EOF;
 		}
 	}
 
@@ -202,7 +211,7 @@ int bfs_polldir(struct bfs_dir *dir) {
 #else // !BFS_USE_GETDENTS
 	if (dir->de) {
 		return 1;
-	} else if (dir->eof) {
+	} else if (dir->flags & BFS_DIR_EOF) {
 		return 0;
 	}
 
@@ -211,7 +220,7 @@ int bfs_polldir(struct bfs_dir *dir) {
 	if (dir->de) {
 		return 1;
 	} else if (errno == 0) {
-		dir->eof = true;
+		dir->flags |= BFS_DIR_EOF;
 		return 0;
 	} else {
 		return -1;
diff --git a/src/dir.h b/src/dir.h
index 1137ff5..7e0cbba 100644
--- a/src/dir.h
+++ b/src/dir.h
@@ -86,6 +86,14 @@ struct bfs_dir *bfs_allocdir(void);
  */
 void bfs_dir_arena(struct arena *arena);
 
+/**
+ * bfs_opendir() flags.
+ */
+enum bfs_dir_flags {
+	/** @internal Start of private flags. */
+	BFS_DIR_PRIVATE = 1 << 0,
+};
+
 /**
  * Open a directory.
  *
@@ -96,10 +104,12 @@ void bfs_dir_arena(struct arena *arena);
  * @param at_path
  *         The path of the directory to open, relative to at_fd.  Pass NULL to
  *         open at_fd itself.
+ * @param flags
+ *         Flags that control which directory entries are listed.
  * @return
  *         0 on success, or -1 on failure.
  */
-int bfs_opendir(struct bfs_dir *dir, int at_fd, const char *at_path);
+int bfs_opendir(struct bfs_dir *dir, int at_fd, const char *at_path, enum bfs_dir_flags flags);
 
 /**
  * Get the file descriptor for a directory.
diff --git a/src/eval.c b/src/eval.c
index e0dd97b..adf7a0b 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -427,7 +427,7 @@ bool eval_empty(const struct bfs_expr *expr, struct bfs_eval *state) {
 			return ret;
 		}
 
-		if (bfs_opendir(dir, ftwbuf->at_fd, ftwbuf->at_path) != 0) {
+		if (bfs_opendir(dir, ftwbuf->at_fd, ftwbuf->at_path, 0) != 0) {
 			eval_report_error(state);
 			return ret;
 		}
@@ -1517,8 +1517,8 @@ static int infer_fdlimit(const struct bfs_ctx *ctx, int limit) {
 		goto done;
 	}
 
-	if (bfs_opendir(dir, AT_FDCWD, "/proc/self/fd") != 0
-	    && bfs_opendir(dir, AT_FDCWD, "/dev/fd") != 0) {
+	if (bfs_opendir(dir, AT_FDCWD, "/proc/self/fd", 0) != 0
+	    && bfs_opendir(dir, AT_FDCWD, "/dev/fd", 0) != 0) {
 		goto done;
 	}
 
diff --git a/src/ioq.c b/src/ioq.c
index 04b9c0d..8c1bdbe 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -348,7 +348,7 @@ static void ioq_handle(struct ioq *ioq, struct ioq_ent *ent) {
 		break;
 
 	case IOQ_OPENDIR:
-		ret = bfs_opendir(ent->opendir.dir, ent->opendir.dfd, ent->opendir.path);
+		ret = bfs_opendir(ent->opendir.dir, ent->opendir.dfd, ent->opendir.path, ent->opendir.flags);
 		if (ret == 0) {
 			bfs_polldir(ent->opendir.dir);
 		}
@@ -505,7 +505,7 @@ static void ioq_ring_reap(struct ioq_ring_state *state) {
 				continue;
 			}
 
-			ent->ret = bfs_opendir(ent->opendir.dir, fd, NULL);
+			ent->ret = bfs_opendir(ent->opendir.dir, fd, NULL, ent->opendir.flags);
 			if (ent->ret == 0) {
 				// TODO: io_uring_prep_getdents()
 				bfs_polldir(ent->opendir.dir);
@@ -659,7 +659,7 @@ int ioq_close(struct ioq *ioq, int fd, void *ptr) {
 	return 0;
 }
 
-int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path, void *ptr) {
+int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path, enum bfs_dir_flags flags, void *ptr) {
 	struct ioq_ent *ent = ioq_request(ioq, IOQ_OPENDIR, ptr);
 	if (!ent) {
 		return -1;
@@ -669,6 +669,7 @@ int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path,
 	args->dir = dir;
 	args->dfd = dfd;
 	args->path = path;
+	args->flags = flags;
 
 	ioqq_push(ioq->pending, ent);
 	return 0;
diff --git a/src/ioq.h b/src/ioq.h
index 99c18c2..eab89ec 100644
--- a/src/ioq.h
+++ b/src/ioq.h
@@ -8,6 +8,7 @@
 #ifndef BFS_IOQ_H
 #define BFS_IOQ_H
 
+#include "dir.h"
 #include <stddef.h>
 
 /**
@@ -53,6 +54,7 @@ struct ioq_ent {
 			struct bfs_dir *dir;
 			int dfd;
 			const char *path;
+			enum bfs_dir_flags flags;
 		} opendir;
 		/** ioq_closedir() args. */
 		struct ioq_closedir {
@@ -103,12 +105,14 @@ int ioq_close(struct ioq *ioq, int fd, void *ptr);
  *         The base file descriptor.
  * @param path
  *         The path to open, relative to dfd.
+ * @param flags
+ *         Flags that control which directory entries are listed.
  * @param ptr
  *         An arbitrary pointer to associate with the request.
  * @return
  *         0 on success, or -1 on failure.
  */
-int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path, void *ptr);
+int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path, enum bfs_dir_flags flags, void *ptr);
 
 /**
  * Asynchronous bfs_closedir().
-- 
cgit v1.2.3


From 5fe11b94b38bfb4d43637e05ac24da0d7d72b9ea Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Tue, 7 Nov 2023 16:43:35 -0500
Subject: ioq: Implement a better non-blocking pop

---
 src/bftw.c |   2 +-
 src/ioq.c  | 276 +++++++++++++++++++++++++++++++++++++++++--------------------
 src/ioq.h  |  13 +--
 3 files changed, 189 insertions(+), 102 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/bftw.c b/src/bftw.c
index 5a55037..0b74cd9 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -588,7 +588,7 @@ static int bftw_ioq_pop(struct bftw_state *state, bool block) {
 		return -1;
 	}
 
-	struct ioq_ent *ent = block ? ioq_pop(ioq) : ioq_trypop(ioq);
+	struct ioq_ent *ent = ioq_pop(ioq, block);
 	if (!ent) {
 		return -1;
 	}
diff --git a/src/ioq.c b/src/ioq.c
index 8c1bdbe..244b2cc 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -1,6 +1,127 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+/**
+ * An asynchronous I/O queue implementation.
+ *
+ * struct ioq is composed of two separate queues:
+ *
+ *     struct ioqq *pending; // Pending I/O requests
+ *     struct ioqq *ready;   // Ready I/O responses
+ *
+ * Worker threads pop requests from `pending`, execute them, and push them back
+ * to the `ready` queue.  The main thread pushes requests to `pending` and pops
+ * them from `ready`.
+ *
+ * struct ioqq is a blocking MPMC queue (though it could be SPMC/MPSC for
+ * pending/ready respectively).  It is implemented as a circular buffer:
+ *
+ *     size_t mask;            // (1 << N) - 1
+ *     [padding]
+ *     size_t head;            // Writer index
+ *     [padding]
+ *     size_t tail;            // Reader index
+ *     [padding]
+ *     ioq_slot slots[1 << N]; // Queue contents
+ *
+ * Pushes are implemented with an unconditional
+ *
+ *     fetch_add(&ioqq->head, IOQ_STRIDE)
+ *
+ * which scales better on many architectures than compare-and-swap (see [1] for
+ * details).  Pops are implemented similarly.  We add IOQ_STRIDE rather than 1
+ * so that successive queue elements are on different cache lines, but the
+ * exposition below uses 1 for simplicity.
+ *
+ * Since the fetch-and-adds are unconditional, non-blocking readers can get
+ * ahead of writers:
+ *
+ *     Reader              Writer
+ *     ────────────────    ──────────────────────
+ *     head:     0 → 1
+ *     slots[0]: empty
+ *                         tail:     0     → 1
+ *                         slots[0]: empty → full
+ *     head:     1 → 2
+ *     slots[1]: empty!
+ *
+ * To avoid this, non-blocking reads (ioqq_pop(ioqq, false)) must mark the slots
+ * somehow so that writers can skip them:
+ *
+ *     Reader                     Writer
+ *     ───────────────────────    ───────────────────────
+ *     head:         0 → 1
+ *     slots[0]: empty → skip
+ *                                tail:         0 → 1
+ *                                slots[0]:  skip → empty
+ *                                tail:         1 → 2
+ *                                slots[1]: empty → full
+ *     head:         1 → 2
+ *     slots[1]:  full → empty
+ *
+ * As well, a reader might "lap" a writer (or another reader), so slots need to
+ * count how many times they should be skipped:
+ *
+ *     Reader                        Writer
+ *     ──────────────────────────    ─────────────────────────
+ *     head:          0 → 1
+ *     slots[0]:  empty → skip(1)
+ *     head:          1 → 2
+ *     slots[1]:  empty → skip(1)
+ *     ...
+ *     head:          M → 0
+ *     slots[M]:  empty → skip(1)
+ *     head:          0 → 1
+ *     slots[0]: skip(1 → 2)
+ *                                   tail:           0 → 1
+ *                                   slots[0]:  skip(2 → 1)
+ *                                   tail:           1 → 2
+ *                                   slots[1]: skip(1) → empty
+ *                                   ...
+ *                                   tail:           M → 0
+ *                                   slots[M]: skip(1) → empty
+ *                                   tail:           0 → 1
+ *                                   slots[0]: skip(1) → empty
+ *                                   tail:           1 → 2
+ *                                   slots[1]:   empty → full
+ *     head:          1 → 2
+ *     slots[1]:   full → empty
+ *
+ * As described in [1], this approach is susceptible to livelock if readers stay
+ * ahead of writers.  This is okay for us because we don't retry failed non-
+ * blocking reads.
+ *
+ * The slot representation uses tag bits to hold either a pointer or skip(N):
+ *
+ *     IOQ_SKIP (highest bit)    IOQ_BLOCKED (lowest bit)
+ *        ↓                         ↓
+ *        0 0 0       ...       0 0 0
+ *          └──────────┬──────────┘
+ *                     │
+ *                value bits
+ *
+ * If IOQ_SKIP is unset, the value bits hold a pointer (or zero/NULL for empty).
+ * If IOQ_SKIP is set, the value bits hold a negative skip count.  Writers can
+ * reduce the skip count by adding 1 to the value bits, and when the count hits
+ * zero, the carry will automatically clear IOQ_SKIP:
+ *
+ *     IOQ_SKIP                  IOQ_BLOCKED
+ *        ↓                         ↓
+ *        1 1 1       ...       1 0 0    skip(2)
+ *        1 1 1       ...       1 1 0    skip(1)
+ *        0 0 0       ...       0 0 0    empty
+ *
+ * The IOQ_BLOCKED flag is used to track sleeping waiters, futex-style.  To wait
+ * for a slot to change, waiters call ioq_slot_wait() which sets IOQ_BLOCKED and
+ * goes to sleep.  Whenever a slot is updated, if the old value had IOQ_BLOCKED
+ * set, ioq_slot_wake() must be called to wake up that waiter.
+ *
+ * Blocking/waking uses a pool of monitors (mutex, condition variable pairs).
+ * Slots are assigned round-robin to a monitor from the pool.
+ *
+ * [1]: https://arxiv.org/abs/2201.02179
+ */
+
 #include "ioq.h"
 #include "alloc.h"
 #include "atomic.h"
@@ -51,24 +172,15 @@ static void ioq_monitor_destroy(struct ioq_monitor *monitor) {
 /** A single entry in a command queue. */
 typedef atomic uintptr_t ioq_slot;
 
-/** Slot flag bit to indicate waiters. */
+/** Someone might be waiting on this slot. */
 #define IOQ_BLOCKED ((uintptr_t)1)
-bfs_static_assert(alignof(struct ioq_ent) > 1);
-
-/** Check if a slot has waiters. */
-static bool ioq_slot_blocked(uintptr_t value) {
-	return value & IOQ_BLOCKED;
-}
-
-/** Extract the pointer from a slot. */
-static struct ioq_ent *ioq_slot_ptr(uintptr_t value) {
-	return (struct ioq_ent *)(value & ~IOQ_BLOCKED);
-}
+/** The next push(es) should skip this slot. */
+#define IOQ_SKIP ((uintptr_t)1 << (UINTPTR_WIDTH - 1))
+/** Amount to add for an additional skip. */
+#define IOQ_SKIP_ONE (~IOQ_BLOCKED)
 
-/** Check if a slot is empty. */
-static bool ioq_slot_empty(uintptr_t value) {
-	return !ioq_slot_ptr(value);
-}
+// Need room for two flag bits
+bfs_static_assert(alignof(struct ioq_ent) > 2);
 
 /**
  * An MPMC queue of I/O commands.
@@ -205,80 +317,85 @@ static void ioq_slot_wake(struct ioqq *ioqq, ioq_slot *slot) {
 	cond_broadcast(&monitor->cond);
 }
 
-/** Get the next slot for writing. */
-static ioq_slot *ioqq_write(struct ioqq *ioqq) {
-	size_t i = fetch_add(&ioqq->head, IOQ_STRIDE, relaxed);
-	return &ioqq->slots[i & ioqq->slot_mask];
-}
-
 /** Push an entry into a slot. */
-static void ioq_slot_push(struct ioqq *ioqq, ioq_slot *slot, struct ioq_ent *ent) {
-	uintptr_t addr = (uintptr_t)ent;
-	bfs_assert(!ioq_slot_blocked(addr));
-
+static bool ioq_slot_push(struct ioqq *ioqq, ioq_slot *slot, struct ioq_ent *ent) {
 	uintptr_t prev = load(slot, relaxed);
-	do {
-		while (!ioq_slot_empty(prev)) {
+	while (true) {
+		uintptr_t next;
+		if (prev & IOQ_SKIP) {
+			// skip(1) → empty
+			// skip(n) → skip(n - 1)
+			next = (prev - IOQ_SKIP_ONE) & ~IOQ_BLOCKED;
+		} else if (prev > IOQ_BLOCKED) {
+			// full(ptr) → wait
 			prev = ioq_slot_wait(ioqq, slot, prev);
+			continue;
+		} else {
+			// empty → full(ptr)
+			next = (uintptr_t)ent >> 1;
 		}
-	} while (!compare_exchange_weak(slot, &prev, addr, release, relaxed));
 
-	if (ioq_slot_blocked(prev)) {
+		if (compare_exchange_weak(slot, &prev, next, release, relaxed)) {
+			break;
+		}
+	}
+
+	if (prev & IOQ_BLOCKED) {
 		ioq_slot_wake(ioqq, slot);
 	}
+
+	return !(prev & IOQ_SKIP);
 }
 
 /** Push an entry onto the queue. */
 static void ioqq_push(struct ioqq *ioqq, struct ioq_ent *ent) {
-	ioq_slot *slot = ioqq_write(ioqq);
-	ioq_slot_push(ioqq, slot, ent);
-}
-
-/** Get the next slot for reading. */
-static ioq_slot *ioqq_read(struct ioqq *ioqq) {
-	size_t i = fetch_add(&ioqq->tail, IOQ_STRIDE, relaxed);
-	return &ioqq->slots[i & ioqq->slot_mask];
+	while (true) {
+		size_t i = fetch_add(&ioqq->head, IOQ_STRIDE, relaxed);
+		ioq_slot *slot = &ioqq->slots[i & ioqq->slot_mask];
+		if (ioq_slot_push(ioqq, slot, ent)) {
+			break;
+		}
+	}
 }
 
 /** (Try to) pop an entry from a slot. */
 static struct ioq_ent *ioq_slot_pop(struct ioqq *ioqq, ioq_slot *slot, bool block) {
 	uintptr_t prev = load(slot, relaxed);
-	do {
-		while (ioq_slot_empty(prev)) {
-			if (block) {
-				prev = ioq_slot_wait(ioqq, slot, prev);
-			} else {
-				return NULL;
-			}
+	while (true) {
+		// empty     → skip(1)
+		// skip(n)   → skip(n + 1)
+		// full(ptr) → full(ptr - 1)
+		uintptr_t next = prev + IOQ_SKIP_ONE;
+		// skip(n)   → ~IOQ_BLOCKED
+		// full(ptr) → 0
+		next &= (next & IOQ_SKIP) ? ~IOQ_BLOCKED : 0;
+
+		if (block && next) {
+			prev = ioq_slot_wait(ioqq, slot, prev);
+			continue;
+		}
+
+		if (compare_exchange_weak(slot, &prev, next, acquire, relaxed)) {
+			break;
 		}
-	} while (!compare_exchange_weak(slot, &prev, 0, acquire, relaxed));
+	}
 
-	if (ioq_slot_blocked(prev)) {
+	if (prev & IOQ_BLOCKED) {
 		ioq_slot_wake(ioqq, slot);
 	}
 
-	return ioq_slot_ptr(prev);
+	// empty     → 0
+	// skip(n)   → 0
+	// full(ptr) → ptr
+	prev &= (prev & IOQ_SKIP) ? 0 : ~IOQ_BLOCKED;
+	return (struct ioq_ent *)(prev << 1);
 }
 
 /** Pop an entry from the queue. */
-static struct ioq_ent *ioqq_pop(struct ioqq *ioqq) {
-	ioq_slot *slot = ioqq_read(ioqq);
-	return ioq_slot_pop(ioqq, slot, true);
-}
-
-/** Pop an entry from the queue if one is available. */
-static struct ioq_ent *ioqq_trypop(struct ioqq *ioqq) {
-	size_t i = load(&ioqq->tail, relaxed);
+static struct ioq_ent *ioqq_pop(struct ioqq *ioqq, bool block) {
+	size_t i = fetch_add(&ioqq->tail, IOQ_STRIDE, relaxed);
 	ioq_slot *slot = &ioqq->slots[i & ioqq->slot_mask];
-
-	struct ioq_ent *ret = ioq_slot_pop(ioqq, slot, false);
-	if (ret) {
-		size_t j = exchange(&ioqq->tail, i + IOQ_STRIDE, relaxed);
-		bfs_assert(j == i, "Detected multiple consumers");
-		(void)j;
-	}
-
-	return ret;
+	return ioq_slot_pop(ioqq, slot, block);
 }
 
 /** Sentinel stop command. */
@@ -378,8 +495,6 @@ struct ioq_ring_state {
 	struct ioq *ioq;
 	/** The io_uring. */
 	struct io_uring *ring;
-	/** The current ioq->pending slot. */
-	ioq_slot *slot;
 	/** Number of prepped, unsubmitted SQEs. */
 	size_t prepped;
 	/** Number of submitted, unreaped SQEs. */
@@ -394,20 +509,9 @@ static struct ioq_ent *ioq_ring_pop(struct ioq_ring_state *state) {
 		return NULL;
 	}
 
-	// Advance to the next slot if necessary
-	struct ioq *ioq = state->ioq;
-	if (!state->slot) {
-		state->slot = ioqq_read(ioq->pending);
-	}
-
 	// Block if we have nothing else to do
 	bool block = !state->prepped && !state->submitted;
-	struct ioq_ent *ret = ioq_slot_pop(ioq->pending, state->slot, block);
-
-	if (ret) {
-		// Got an entry, move to the next slot next time
-		state->slot = NULL;
-	}
+	struct ioq_ent *ret = ioqq_pop(state->ioq->pending, block);
 
 	if (ret == &IOQ_STOP) {
 		state->stop = true;
@@ -536,7 +640,7 @@ static void ioq_sync_work(struct ioq_thread *thread) {
 	struct ioq *ioq = thread->parent;
 
 	while (true) {
-		struct ioq_ent *ent = ioqq_pop(ioq->pending);
+		struct ioq_ent *ent = ioqq_pop(ioq->pending, true);
 		if (ent == &IOQ_STOP) {
 			break;
 		}
@@ -687,20 +791,12 @@ int ioq_closedir(struct ioq *ioq, struct bfs_dir *dir, void *ptr) {
 	return 0;
 }
 
-struct ioq_ent *ioq_pop(struct ioq *ioq) {
-	if (ioq->size == 0) {
-		return NULL;
-	}
-
-	return ioqq_pop(ioq->ready);
-}
-
-struct ioq_ent *ioq_trypop(struct ioq *ioq) {
+struct ioq_ent *ioq_pop(struct ioq *ioq, bool block) {
 	if (ioq->size == 0) {
 		return NULL;
 	}
 
-	return ioqq_trypop(ioq->ready);
+	return ioqq_pop(ioq->ready, block);
 }
 
 void ioq_free(struct ioq *ioq, struct ioq_ent *ent) {
diff --git a/src/ioq.h b/src/ioq.h
index eab89ec..87727cb 100644
--- a/src/ioq.h
+++ b/src/ioq.h
@@ -8,6 +8,7 @@
 #ifndef BFS_IOQ_H
 #define BFS_IOQ_H
 
+#include "config.h"
 #include "dir.h"
 #include <stddef.h>
 
@@ -136,17 +137,7 @@ int ioq_closedir(struct ioq *ioq, struct bfs_dir *dir, void *ptr);
  * @return
  *         The next response, or NULL.
  */
-struct ioq_ent *ioq_pop(struct ioq *ioq);
-
-/**
- * Pop a response from the queue, without blocking.
- *
- * @param ioq
- *         The I/O queue.
- * @return
- *         The next response, or NULL.
- */
-struct ioq_ent *ioq_trypop(struct ioq *ioq);
+struct ioq_ent *ioq_pop(struct ioq *ioq, bool block);
 
 /**
  * Free a queue entry.
-- 
cgit v1.2.3


From c745df94a182b8a569cb833ecfbe8da33bf01f98 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Thu, 9 Nov 2023 14:34:21 -0500
Subject: config: New attr_noinline and attr_cold macros

---
 src/alloc.c  |  1 +
 src/config.h | 18 ++++++++++++++++++
 src/diag.h   | 15 +++++++++++++++
 src/ioq.c    |  2 ++
 4 files changed, 36 insertions(+)

(limited to 'src/ioq.c')

diff --git a/src/alloc.c b/src/alloc.c
index 0b978ba..3b9972f 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -107,6 +107,7 @@ void arena_init(struct arena *arena, size_t align, size_t size) {
 }
 
 /** Allocate a new slab. */
+attr_cold
 static int slab_alloc(struct arena *arena) {
 	void **slabs = realloc(arena->slabs, sizeof_array(void *, arena->nslabs + 1));
 	if (!slabs) {
diff --git a/src/config.h b/src/config.h
index 3100cec..b95abaa 100644
--- a/src/config.h
+++ b/src/config.h
@@ -184,6 +184,24 @@
 #  define fallthru ((void)0)
 #endif
 
+/**
+ * Hint to avoid inlining a function.
+ */
+#if __has_attribute(noinline)
+#  define attr_noinline __attribute__((noinline))
+#else
+#  define attr_noinline
+#endif
+
+/**
+ * Hint that a function is unlikely to be called.
+ */
+#if __has_attribute(cold)
+#  define attr_cold attr_noinline __attribute__((cold))
+#else
+#  define attr_cold attr_noinline
+#endif
+
 /**
  * Adds compiler warnings for bad printf()-style function calls, if supported.
  */
diff --git a/src/diag.h b/src/diag.h
index 8c7ed57..870264e 100644
--- a/src/diag.h
+++ b/src/diag.h
@@ -44,6 +44,7 @@ struct bfs_loc {
 /**
  * Print a message to standard error and abort.
  */
+attr_cold
 attr_format(2, 3)
 noreturn void bfs_abortf(const struct bfs_loc *loc, const char *format, ...);
 
@@ -116,11 +117,13 @@ const char *debug_flag_name(enum debug_flags flag);
 /**
  * Like perror(), but decorated like bfs_error().
  */
+attr_cold
 void bfs_perror(const struct bfs_ctx *ctx, const char *str);
 
 /**
  * Shorthand for printing error messages.
  */
+attr_cold
 attr_format(2, 3)
 void bfs_error(const struct bfs_ctx *ctx, const char *format, ...);
 
@@ -129,6 +132,7 @@ void bfs_error(const struct bfs_ctx *ctx, const char *format, ...);
  *
  * @return Whether a warning was printed.
  */
+attr_cold
 attr_format(2, 3)
 bool bfs_warning(const struct bfs_ctx *ctx, const char *format, ...);
 
@@ -137,60 +141,71 @@ bool bfs_warning(const struct bfs_ctx *ctx, const char *format, ...);
  *
  * @return Whether a debug message was printed.
  */
+attr_cold
 attr_format(3, 4)
 bool bfs_debug(const struct bfs_ctx *ctx, enum debug_flags flag, const char *format, ...);
 
 /**
  * bfs_error() variant that takes a va_list.
  */
+attr_cold
 attr_format(2, 0)
 void bfs_verror(const struct bfs_ctx *ctx, const char *format, va_list args);
 
 /**
  * bfs_warning() variant that takes a va_list.
  */
+attr_cold
 attr_format(2, 0)
 bool bfs_vwarning(const struct bfs_ctx *ctx, const char *format, va_list args);
 
 /**
  * bfs_debug() variant that takes a va_list.
  */
+attr_cold
 attr_format(3, 0)
 bool bfs_vdebug(const struct bfs_ctx *ctx, enum debug_flags flag, const char *format, va_list args);
 
 /**
  * Print the error message prefix.
  */
+attr_cold
 void bfs_error_prefix(const struct bfs_ctx *ctx);
 
 /**
  * Print the warning message prefix.
  */
+attr_cold
 bool bfs_warning_prefix(const struct bfs_ctx *ctx);
 
 /**
  * Print the debug message prefix.
  */
+attr_cold
 bool bfs_debug_prefix(const struct bfs_ctx *ctx, enum debug_flags flag);
 
 /**
  * Highlight parts of the command line in an error message.
  */
+attr_cold
 void bfs_argv_error(const struct bfs_ctx *ctx, const bool args[]);
 
 /**
  * Highlight parts of an expression in an error message.
  */
+attr_cold
 void bfs_expr_error(const struct bfs_ctx *ctx, const struct bfs_expr *expr);
 
 /**
  * Highlight parts of the command line in a warning message.
  */
+attr_cold
 bool bfs_argv_warning(const struct bfs_ctx *ctx, const bool args[]);
 
 /**
  * Highlight parts of an expression in a warning message.
  */
+attr_cold
 bool bfs_expr_warning(const struct bfs_ctx *ctx, const struct bfs_expr *expr);
 
 #endif // BFS_DIAG_H
diff --git a/src/ioq.c b/src/ioq.c
index 244b2cc..ef89fa8 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -269,6 +269,7 @@ static struct ioq_monitor *ioq_slot_monitor(struct ioqq *ioqq, ioq_slot *slot) {
 }
 
 /** Atomically wait for a slot to change. */
+attr_noinline
 static uintptr_t ioq_slot_wait(struct ioqq *ioqq, ioq_slot *slot, uintptr_t value) {
 	struct ioq_monitor *monitor = ioq_slot_monitor(ioqq, slot);
 	mutex_lock(&monitor->mutex);
@@ -298,6 +299,7 @@ done:
 }
 
 /** Wake up any threads waiting on a slot. */
+attr_noinline
 static void ioq_slot_wake(struct ioqq *ioqq, ioq_slot *slot) {
 	struct ioq_monitor *monitor = ioq_slot_monitor(ioqq, slot);
 
-- 
cgit v1.2.3


From 949a197112e2de23c31f2afb1247bf3568097b69 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 15 Nov 2023 16:09:47 -0500
Subject: ioq: Don't crash on allocation failures

---
 src/ioq.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index ef89fa8..bd8d111 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -216,6 +216,10 @@ bfs_static_assert(IOQ_STRIDE % 2 == 1);
 
 /** Destroy an I/O command queue. */
 static void ioqq_destroy(struct ioqq *ioqq) {
+	if (!ioqq) {
+		return;
+	}
+
 	for (size_t i = 0; i < ioqq->monitor_mask + 1; ++i) {
 		ioq_monitor_destroy(&ioqq->monitors[i]);
 	}
-- 
cgit v1.2.3


From 5b38f658ee42bef05cecb6cadec65b25d9e94993 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Thu, 16 Nov 2023 12:25:09 -0500
Subject: config: New variadic attr(...) macro

---
 src/alloc.c   |  2 +-
 src/alloc.h   | 10 ++++----
 src/bar.c     |  2 +-
 src/color.c   |  4 ++--
 src/color.h   |  4 ++--
 src/config.h  | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/diag.h    | 37 ++++++++++++------------------
 src/dstring.h | 18 +++++++--------
 src/eval.c    |  2 +-
 src/exec.c    |  2 +-
 src/fsade.c   |  6 ++---
 src/ioq.c     |  4 ++--
 src/mtab.c    |  2 +-
 src/opt.c     |  4 ++--
 src/parse.c   | 14 +++++------
 src/trie.c    |  2 +-
 src/xspawn.c  |  2 +-
 17 files changed, 127 insertions(+), 62 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/alloc.c b/src/alloc.c
index 467f0f0..b65d0c5 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -176,7 +176,7 @@ void arena_init(struct arena *arena, size_t align, size_t size) {
 }
 
 /** Allocate a new slab. */
-attr_cold
+attr(cold)
 static int slab_alloc(struct arena *arena) {
 	// Make the initial allocation size ~4K
 	size_t size = 4096;
diff --git a/src/alloc.h b/src/alloc.h
index 7132470..f21ddb9 100644
--- a/src/alloc.h
+++ b/src/alloc.h
@@ -114,8 +114,7 @@ static inline size_t flex_size(size_t align, size_t min, size_t offset, size_t s
  * @return
  *         The allocated memory, or NULL on failure.
  */
-attr_malloc(free, 1)
-attr_aligned_alloc(1, 2)
+attr(malloc(free, 1), aligned_alloc(1, 2))
 void *alloc(size_t align, size_t size);
 
 /**
@@ -128,8 +127,7 @@ void *alloc(size_t align, size_t size);
  * @return
  *         The allocated memory, or NULL on failure.
  */
-attr_malloc(free, 1)
-attr_aligned_alloc(1, 2)
+attr(malloc(free, 1), aligned_alloc(1, 2))
 void *zalloc(size_t align, size_t size);
 
 /** Allocate memory for the given type. */
@@ -252,7 +250,7 @@ void arena_free(struct arena *arena, void *ptr);
 /**
  * Allocate an object out of the arena.
  */
-attr_malloc(arena_free, 2)
+attr(malloc(arena_free, 2))
 void *arena_alloc(struct arena *arena);
 
 /**
@@ -334,7 +332,7 @@ void varena_free(struct varena *varena, void *ptr, size_t count);
  * @return
  *         The allocated struct, or NULL on failure.
  */
-attr_malloc(varena_free, 2)
+attr(malloc(varena_free, 2))
 void *varena_alloc(struct varena *varena, size_t count);
 
 /**
diff --git a/src/bar.c b/src/bar.c
index 2e50dbe..babadc3 100644
--- a/src/bar.c
+++ b/src/bar.c
@@ -127,7 +127,7 @@ static void reset_before_death_by(int sig) {
 }
 
 /** printf() to the status bar with a single write(). */
-attr_format(2, 3)
+attr(format(2, 3))
 static int bfs_bar_printf(struct bfs_bar *bar, const char *format, ...) {
 	va_list args;
 	va_start(args, format);
diff --git a/src/color.c b/src/color.c
index 05ba1a3..4c2c8ca 100644
--- a/src/color.c
+++ b/src/color.c
@@ -1105,7 +1105,7 @@ static int print_link_target(CFILE *cfile, const struct BFTW *ftwbuf) {
 }
 
 /** Format some colored output to the buffer. */
-attr_format(2, 3)
+attr(format(2, 3))
 static int cbuff(CFILE *cfile, const char *format, ...);
 
 /** Dump a parsed expression tree, for debugging. */
@@ -1177,7 +1177,7 @@ static int print_expr(CFILE *cfile, const struct bfs_expr *expr, bool verbose) {
 	return 0;
 }
 
-attr_format(2, 0)
+attr(format(2, 0))
 static int cvbuff(CFILE *cfile, const char *format, va_list args) {
 	const struct colors *colors = cfile->colors;
 	int error = errno;
diff --git a/src/color.h b/src/color.h
index 8a81573..81f0e2a 100644
--- a/src/color.h
+++ b/src/color.h
@@ -100,13 +100,13 @@ int cfclose(CFILE *cfile);
  * @return
  *         0 on success, -1 on failure.
  */
-attr_format(2, 3)
+attr(format(2, 3))
 int cfprintf(CFILE *cfile, const char *format, ...);
 
 /**
  * cfprintf() variant that takes a va_list.
  */
-attr_format(2, 0)
+attr(format(2, 0))
 int cvfprintf(CFILE *cfile, const char *format, va_list args);
 
 #endif // BFS_COLOR_H
diff --git a/src/config.h b/src/config.h
index 9f95674..aa03552 100644
--- a/src/config.h
+++ b/src/config.h
@@ -301,4 +301,78 @@ typedef long double max_align_t;
 #  define attr_target_clones(...)
 #endif
 
+/**
+ * Shorthand for multiple attributes at once. attr(a, b(c), d) is equivalent to
+ *
+ *     attr_a
+ *     attr_b(c)
+ *     attr_d
+ */
+#define attr(...) \
+	attr__(attr_##__VA_ARGS__, none, none, none, none, none, none, none, none, none)
+
+/**
+ * attr() helper.  For exposition, pretend we support only 2 args, instead of 9.
+ * There are a few cases:
+ *
+ *     attr()
+ *         => attr__(attr_, none, none)
+ *         => attr_                =>
+ *            attr_none            =>
+ *            attr_too_many_none() =>
+ *
+ *     attr(a)
+ *         => attr__(attr_a, none, none)
+ *         => attr_a               => __attribute__((a))
+ *            attr_none            =>
+ *            attr_too_many_none() =>
+ *
+ *     attr(a, b(c))
+ *         => attr__(attr_a, b(c), none, none)
+ *         => attr_a                   => __attribute__((a))
+ *            attr_b(c)                => __attribute__((b(c)))
+ *            attr_too_many_none(none) =>
+ *
+ *     attr(a, b(c), d)
+ *         => attr__(attr_a, b(c), d, none, none)
+ *         => attr_a                      => __attribute__((a))
+ *            attr_b(c)                   => __attribute__((b(c)))
+ *            attr_too_many_d(none, none) => error
+ *
+ * Some attribute names are the same as standard library functions, e.g. printf.
+ * Standard libraries are permitted to define these functions as macros, like
+ *
+ *     #define printf(...) __builtin_printf(__VA_ARGS__)
+ *
+ * The token paste in
+ *
+ *     #define attr(...) attr__(attr_##__VA_ARGS__, none, none)
+ *
+ * is necessary to prevent macro expansion before evaluating attr__().
+ * Otherwise, we could get
+ *
+ *     attr(printf(1, 2))
+ *         => attr__(__builtin_printf(1, 2), none, none)
+ *         => attr____builtin_printf(1, 2)
+ *         => error
+ */
+#define attr__(a1, a2, a3, a4, a5, a6, a7, a8, a9, none, ...) \
+	a1 \
+	attr_##a2 \
+	attr_##a3 \
+	attr_##a4 \
+	attr_##a5 \
+	attr_##a6 \
+	attr_##a7 \
+	attr_##a8 \
+	attr_##a9 \
+	attr_too_many_##none(__VA_ARGS__)
+
+// Ignore `attr_none` from expanding 1-9 argument attr(a1, a2, ...)
+#define attr_none
+// Ignore `attr_` from expanding 0-argument attr()
+#define attr_
+// Only trigger an error on more than 9 arguments
+#define attr_too_many_none(...)
+
 #endif // BFS_CONFIG_H
diff --git a/src/diag.h b/src/diag.h
index 981419e..aa6a44f 100644
--- a/src/diag.h
+++ b/src/diag.h
@@ -44,8 +44,7 @@ struct bfs_loc {
 /**
  * Print a message to standard error and abort.
  */
-attr_cold
-attr_format(2, 3)
+attr(cold, format(2, 3))
 noreturn void bfs_abortf(const struct bfs_loc *loc, const char *format, ...);
 
 /**
@@ -117,14 +116,13 @@ const char *debug_flag_name(enum debug_flags flag);
 /**
  * Like perror(), but decorated like bfs_error().
  */
-attr_cold
+attr(cold)
 void bfs_perror(const struct bfs_ctx *ctx, const char *str);
 
 /**
  * Shorthand for printing error messages.
  */
-attr_cold
-attr_format(2, 3)
+attr(cold, format(2, 3))
 void bfs_error(const struct bfs_ctx *ctx, const char *format, ...);
 
 /**
@@ -132,8 +130,7 @@ void bfs_error(const struct bfs_ctx *ctx, const char *format, ...);
  *
  * @return Whether a warning was printed.
  */
-attr_cold
-attr_format(2, 3)
+attr(cold, format(2, 3))
 bool bfs_warning(const struct bfs_ctx *ctx, const char *format, ...);
 
 /**
@@ -141,71 +138,67 @@ bool bfs_warning(const struct bfs_ctx *ctx, const char *format, ...);
  *
  * @return Whether a debug message was printed.
  */
-attr_cold
-attr_format(3, 4)
+attr(cold, format(3, 4))
 bool bfs_debug(const struct bfs_ctx *ctx, enum debug_flags flag, const char *format, ...);
 
 /**
  * bfs_error() variant that takes a va_list.
  */
-attr_cold
-attr_format(2, 0)
+attr(cold, format(2, 0))
 void bfs_verror(const struct bfs_ctx *ctx, const char *format, va_list args);
 
 /**
  * bfs_warning() variant that takes a va_list.
  */
-attr_cold
-attr_format(2, 0)
+attr(cold, format(2, 0))
 bool bfs_vwarning(const struct bfs_ctx *ctx, const char *format, va_list args);
 
 /**
  * bfs_debug() variant that takes a va_list.
  */
-attr_cold
-attr_format(3, 0)
+attr(cold, format(3, 0))
 bool bfs_vdebug(const struct bfs_ctx *ctx, enum debug_flags flag, const char *format, va_list args);
 
 /**
  * Print the error message prefix.
  */
-attr_cold
+attr(cold)
 void bfs_error_prefix(const struct bfs_ctx *ctx);
 
 /**
  * Print the warning message prefix.
  */
-attr_cold
+attr(cold)
 bool bfs_warning_prefix(const struct bfs_ctx *ctx);
 
 /**
  * Print the debug message prefix.
  */
-attr_cold
+attr(cold)
 bool bfs_debug_prefix(const struct bfs_ctx *ctx, enum debug_flags flag);
 
 /**
  * Highlight parts of the command line in an error message.
  */
-attr_cold
+attr(cold)
 void bfs_argv_error(const struct bfs_ctx *ctx, const bool args[]);
 
 /**
  * Highlight parts of an expression in an error message.
  */
-attr_cold
+attr(cold)
 void bfs_expr_error(const struct bfs_ctx *ctx, const struct bfs_expr *expr);
 
 /**
  * Highlight parts of the command line in a warning message.
  */
-attr_cold
+attr(cold)
 bool bfs_argv_warning(const struct bfs_ctx *ctx, const bool args[]);
 
 /**
  * Highlight parts of an expression in a warning message.
  */
-attr_cold
+attr(cold)
 bool bfs_expr_warning(const struct bfs_ctx *ctx, const struct bfs_expr *expr);
 
 #endif // BFS_DIAG_H
diff --git a/src/dstring.h b/src/dstring.h
index 2a94438..1be1185 100644
--- a/src/dstring.h
+++ b/src/dstring.h
@@ -41,7 +41,7 @@ void dstrfree(dchar *dstr);
  * @param cap
  *         The initial capacity of the string.
  */
-attr_malloc(dstrfree, 1)
+attr(malloc(dstrfree, 1))
 dchar *dstralloc(size_t cap);
 
 /**
@@ -50,7 +50,7 @@ dchar *dstralloc(size_t cap);
  * @param str
  *         The NUL-terminated string to copy.
  */
-attr_malloc(dstrfree, 1)
+attr(malloc(dstrfree, 1))
 dchar *dstrdup(const char *str);
 
 /**
@@ -61,7 +61,7 @@ dchar *dstrdup(const char *str);
  * @param n
  *         The maximum number of characters to copy from str.
  */
-attr_malloc(dstrfree, 1)
+attr(malloc(dstrfree, 1))
 dchar *dstrndup(const char *str, size_t n);
 
 /**
@@ -70,7 +70,7 @@ dchar *dstrndup(const char *str, size_t n);
  * @param dstr
  *         The dynamic string to copy.
  */
-attr_malloc(dstrfree, 1)
+attr(malloc(dstrfree, 1))
 dchar *dstrddup(const dchar *dstr);
 
 /**
@@ -81,7 +81,7 @@ dchar *dstrddup(const dchar *dstr);
  * @param len
  *         The length of the string, which may include internal NUL bytes.
  */
-attr_malloc(dstrfree, 1)
+attr(malloc(dstrfree, 1))
 dchar *dstrxdup(const char *str, size_t len);
 
 /**
@@ -243,7 +243,7 @@ int dstrxcpy(dchar **dest, const char *str, size_t len);
  * @return
  *         The created string, or NULL on failure.
  */
-attr_format(1, 2)
+attr(format(1, 2))
 char *dstrprintf(const char *format, ...);
 
 /**
@@ -256,7 +256,7 @@ char *dstrprintf(const char *format, ...);
  * @return
  *         The created string, or NULL on failure.
  */
-attr_format(1, 0)
+attr(format(1, 0))
 char *dstrvprintf(const char *format, va_list args);
 
 /**
@@ -271,7 +271,7 @@ char *dstrvprintf(const char *format, va_list args);
  * @return
  *         0 on success, -1 on failure.
  */
-attr_format(2, 3)
+attr(format(2, 3))
 int dstrcatf(dchar **str, const char *format, ...);
 
 /**
@@ -286,7 +286,7 @@ int dstrcatf(dchar **str, const char *format, ...);
  * @return
  *         0 on success, -1 on failure.
  */
-attr_format(2, 0)
+attr(format(2, 0))
 int dstrvcatf(dchar **str, const char *format, va_list args);
 
 /**
diff --git a/src/eval.c b/src/eval.c
index a990fd4..859ad7e 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -57,7 +57,7 @@ struct bfs_eval {
 /**
  * Print an error message.
  */
-attr_format(2, 3)
+attr(format(2, 3))
 static void eval_error(struct bfs_eval *state, const char *format, ...) {
 	// By POSIX, any errors should be accompanied by a non-zero exit status
 	*state->ret = EXIT_FAILURE;
diff --git a/src/exec.c b/src/exec.c
index 90b3598..ba2fec8 100644
--- a/src/exec.c
+++ b/src/exec.c
@@ -22,7 +22,7 @@
 #include <unistd.h>
 
 /** Print some debugging info. */
-attr_format(2, 3)
+attr(format(2, 3))
 static void bfs_exec_debug(const struct bfs_exec *execbuf, const char *format, ...) {
 	const struct bfs_ctx *ctx = execbuf->ctx;
 
diff --git a/src/fsade.c b/src/fsade.c
index 4d22d99..a48eeb0 100644
--- a/src/fsade.c
+++ b/src/fsade.c
@@ -32,7 +32,7 @@
  * Many of the APIs used here don't have *at() variants, but we can try to
  * emulate something similar if /proc/self/fd is available.
  */
-attr_maybe_unused
+attr(maybe_unused)
 static const char *fake_at(const struct BFTW *ftwbuf) {
 	static atomic int proc_works = -1;
 
@@ -66,7 +66,7 @@ fail:
 	return ftwbuf->path;
 }
 
-attr_maybe_unused
+attr(maybe_unused)
 static void free_fake_at(const struct BFTW *ftwbuf, const char *path) {
 	if (path != ftwbuf->path) {
 		dstrfree((dchar *)path);
@@ -76,7 +76,7 @@ static void free_fake_at(const struct BFTW *ftwbuf, const char *path) {
 /**
  * Check if an error was caused by the absence of support or data for a feature.
  */
-attr_maybe_unused
+attr(maybe_unused)
 static bool is_absence_error(int error) {
 	// If the OS doesn't support the feature, it's obviously not enabled for
 	// any files
diff --git a/src/ioq.c b/src/ioq.c
index bd8d111..2739338 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -273,7 +273,7 @@ static struct ioq_monitor *ioq_slot_monitor(struct ioqq *ioqq, ioq_slot *slot) {
 }
 
 /** Atomically wait for a slot to change. */
-attr_noinline
+attr(noinline)
 static uintptr_t ioq_slot_wait(struct ioqq *ioqq, ioq_slot *slot, uintptr_t value) {
 	struct ioq_monitor *monitor = ioq_slot_monitor(ioqq, slot);
 	mutex_lock(&monitor->mutex);
@@ -303,7 +303,7 @@ done:
 }
 
 /** Wake up any threads waiting on a slot. */
-attr_noinline
+attr(noinline)
 static void ioq_slot_wake(struct ioqq *ioqq, ioq_slot *slot) {
 	struct ioq_monitor *monitor = ioq_slot_monitor(ioqq, slot);
 
diff --git a/src/mtab.c b/src/mtab.c
index 082150c..cc726a2 100644
--- a/src/mtab.c
+++ b/src/mtab.c
@@ -61,7 +61,7 @@ struct bfs_mtab {
 /**
  * Add an entry to the mount table.
  */
-attr_maybe_unused
+attr(maybe_unused)
 static int bfs_mtab_add(struct bfs_mtab *mtab, const char *path, const char *type) {
 	struct bfs_mount *mount = RESERVE(struct bfs_mount, &mtab->mounts, &mtab->nmounts);
 	if (!mount) {
diff --git a/src/opt.c b/src/opt.c
index 5da73d8..3ee5e81 100644
--- a/src/opt.c
+++ b/src/opt.c
@@ -305,7 +305,7 @@ struct opt_state {
 };
 
 /** Log an optimization. */
-attr_format(3, 4)
+attr(format(3, 4))
 static bool opt_debug(const struct opt_state *state, int level, const char *format, ...) {
 	bfs_assert(state->ctx->optlevel >= level);
 
@@ -321,7 +321,7 @@ static bool opt_debug(const struct opt_state *state, int level, const char *form
 }
 
 /** Warn about an expression. */
-attr_format(3, 4)
+attr(format(3, 4))
 static void opt_warning(const struct opt_state *state, const struct bfs_expr *expr, const char *format, ...) {
 	if (bfs_expr_warning(state->ctx, expr)) {
 		va_list args;
diff --git a/src/parse.c b/src/parse.c
index 778fc68..35d22fb 100644
--- a/src/parse.c
+++ b/src/parse.c
@@ -246,7 +246,7 @@ static void highlight_args(const struct bfs_ctx *ctx, char **argv, size_t argc,
 /**
  * Print an error message during parsing.
  */
-attr_format(2, 3)
+attr(format(2, 3))
 static void parse_error(const struct parser_state *state, const char *format, ...) {
 	int error = errno;
 	const struct bfs_ctx *ctx = state->ctx;
@@ -266,7 +266,7 @@ static void parse_error(const struct parser_state *state, const char *format, ..
 /**
  * Print an error about some command line arguments.
  */
-attr_format(4, 5)
+attr(format(4, 5))
 static void parse_argv_error(const struct parser_state *state, char **argv, size_t argc, const char *format, ...) {
 	int error = errno;
 	const struct bfs_ctx *ctx = state->ctx;
@@ -286,7 +286,7 @@ static void parse_argv_error(const struct parser_state *state, char **argv, size
 /**
  * Print an error about conflicting command line arguments.
  */
-attr_format(6, 7)
+attr(format(6, 7))
 static void parse_conflict_error(const struct parser_state *state, char **argv1, size_t argc1, char **argv2, size_t argc2, const char *format, ...) {
 	int error = errno;
 	const struct bfs_ctx *ctx = state->ctx;
@@ -307,7 +307,7 @@ static void parse_conflict_error(const struct parser_state *state, char **argv1,
 /**
  * Print an error about an expression.
  */
-attr_format(3, 4)
+attr(format(3, 4))
 static void parse_expr_error(const struct parser_state *state, const struct bfs_expr *expr, const char *format, ...) {
 	int error = errno;
 	const struct bfs_ctx *ctx = state->ctx;
@@ -324,7 +324,7 @@ static void parse_expr_error(const struct parser_state *state, const struct bfs_
 /**
  * Print a warning message during parsing.
  */
-attr_format(2, 3)
+attr(format(2, 3))
 static bool parse_warning(const struct parser_state *state, const char *format, ...) {
 	int error = errno;
 	const struct bfs_ctx *ctx = state->ctx;
@@ -347,7 +347,7 @@ static bool parse_warning(const struct parser_state *state, const char *format,
 /**
  * Print a warning about conflicting command line arguments.
  */
-attr_format(6, 7)
+attr(format(6, 7))
 static bool parse_conflict_warning(const struct parser_state *state, char **argv1, size_t argc1, char **argv2, size_t argc2, const char *format, ...) {
 	int error = errno;
 	const struct bfs_ctx *ctx = state->ctx;
@@ -371,7 +371,7 @@ static bool parse_conflict_warning(const struct parser_state *state, char **argv
 /**
  * Print a warning about an expression.
  */
-attr_format(3, 4)
+attr(format(3, 4))
 static bool parse_expr_warning(const struct parser_state *state, const struct bfs_expr *expr, const char *format, ...) {
 	int error = errno;
 	const struct bfs_ctx *ctx = state->ctx;
diff --git a/src/trie.c b/src/trie.c
index cf55cee..bd5300d 100644
--- a/src/trie.c
+++ b/src/trie.c
@@ -95,7 +95,7 @@
 bfs_static_assert(CHAR_WIDTH == 8);
 
 #if __i386__ || __x86_64__
-#  define trie_clones attr_target_clones("popcnt", "default")
+#  define trie_clones attr(target_clones("popcnt", "default"))
 #else
 #  define trie_clones
 #endif
diff --git a/src/xspawn.c b/src/xspawn.c
index 03287c3..40115a1 100644
--- a/src/xspawn.c
+++ b/src/xspawn.c
@@ -83,7 +83,7 @@ int bfs_spawn_destroy(struct bfs_spawn *ctx) {
 
 #if _POSIX_SPAWN > 0
 /** Set some posix_spawnattr flags. */
-attr_maybe_unused
+attr(maybe_unused)
 static int bfs_spawn_addflags(struct bfs_spawn *ctx, short flags) {
 	short prev;
 	errno = posix_spawnattr_getflags(&ctx->attr, &prev);
-- 
cgit v1.2.3


From e766015d60c927aae03b8e43d956c6976c16b2ba Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Sat, 13 Jan 2024 15:54:46 -0500
Subject: ioq: Use the negative errno convention

---
 src/bfstd.c |  9 +++++++++
 src/bfstd.h | 10 ++++++++++
 src/bftw.c  |  2 +-
 src/ioq.c   | 32 +++++++++++---------------------
 src/ioq.h   |  6 ++----
 5 files changed, 33 insertions(+), 26 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/bfstd.c b/src/bfstd.c
index 15e8667..0a9b87c 100644
--- a/src/bfstd.c
+++ b/src/bfstd.c
@@ -59,6 +59,15 @@ bool errno_is_like(int category) {
 	return error_is_like(errno, category);
 }
 
+int try(int ret) {
+	if (ret >= 0) {
+		return ret;
+	} else {
+		bfs_assert(errno > 0, "errno should be positive, was %d\n", errno);
+		return -errno;
+	}
+}
+
 char *xdirname(const char *path) {
 	size_t i = xbaseoff(path);
 
diff --git a/src/bfstd.h b/src/bfstd.h
index 8953b9b..d160c88 100644
--- a/src/bfstd.h
+++ b/src/bfstd.h
@@ -63,6 +63,16 @@ bool error_is_like(int error, int category);
  */
 bool errno_is_like(int category);
 
+/**
+ * Apply the "negative errno" convention.
+ *
+ * @param ret
+ *         The return value of the attempted operation.
+ * @return
+ *         ret, if non-negative, otherwise -errno.
+ */
+int try(int ret);
+
 #include <fcntl.h>
 
 #ifndef O_EXEC
diff --git a/src/bftw.c b/src/bftw.c
index 49f07df..952b090 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -621,7 +621,7 @@ static int bftw_ioq_pop(struct bftw_state *state, bool block) {
 		}
 
 		dir = ent->opendir.dir;
-		if (ent->ret == 0) {
+		if (ent->result >= 0) {
 			bftw_file_set_dir(cache, file, dir);
 		} else {
 			bftw_freedir(cache, dir);
diff --git a/src/ioq.c b/src/ioq.c
index 2739338..89ebb3e 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -455,42 +455,35 @@ static bool ioq_check_cancel(struct ioq *ioq, struct ioq_ent *ent) {
 		return false;
 	}
 
-	ent->ret = -1;
-	ent->error = EINTR;
+	ent->result = -EINTR;
 	ioqq_push(ioq->ready, ent);
 	return true;
 }
 
 /** Handle a single request synchronously. */
 static void ioq_handle(struct ioq *ioq, struct ioq_ent *ent) {
-	int ret;
-
 	switch (ent->op) {
 	case IOQ_CLOSE:
-		ret = xclose(ent->close.fd);
+		ent->result = try(xclose(ent->close.fd));
 		break;
 
 	case IOQ_OPENDIR:
-		ret = bfs_opendir(ent->opendir.dir, ent->opendir.dfd, ent->opendir.path, ent->opendir.flags);
-		if (ret == 0) {
+		ent->result = try(bfs_opendir(ent->opendir.dir, ent->opendir.dfd, ent->opendir.path, ent->opendir.flags));
+		if (ent->result >= 0) {
 			bfs_polldir(ent->opendir.dir);
 		}
 		break;
 
 	case IOQ_CLOSEDIR:
-		ret = bfs_closedir(ent->closedir.dir);
+		ent->result = try(bfs_closedir(ent->closedir.dir));
 		break;
 
 	default:
 		bfs_bug("Unknown ioq_op %d", (int)ent->op);
-		ret = -1;
-		errno = ENOSYS;
+		ent->result = -ENOSYS;
 		break;
 	}
 
-	ent->ret = ret;
-	ent->error = ret == 0 ? 0 : errno;
-
 	ioqq_push(ioq->ready, ent);
 }
 
@@ -603,24 +596,21 @@ static void ioq_ring_reap(struct ioq_ring_state *state) {
 		}
 
 		struct ioq_ent *ent = io_uring_cqe_get_data(cqe);
-		ent->ret = cqe->res >= 0 ? cqe->res : -1;
-		ent->error = cqe->res < 0 ? -cqe->res : 0;
+		ent->result = cqe->res;
 		io_uring_cqe_seen(ring, cqe);
 		--state->submitted;
 
-		if (ent->op == IOQ_OPENDIR && ent->ret >= 0) {
-			int fd = ent->ret;
+		if (ent->op == IOQ_OPENDIR && ent->result >= 0) {
+			int fd = ent->result;
 			if (ioq_check_cancel(ioq, ent)) {
 				xclose(fd);
 				continue;
 			}
 
-			ent->ret = bfs_opendir(ent->opendir.dir, fd, NULL, ent->opendir.flags);
-			if (ent->ret == 0) {
+			ent->result = try(bfs_opendir(ent->opendir.dir, fd, NULL, ent->opendir.flags));
+			if (ent->result >= 0) {
 				// TODO: io_uring_prep_getdents()
 				bfs_polldir(ent->opendir.dir);
-			} else {
-				ent->error = errno;
 			}
 		}
 
diff --git a/src/ioq.h b/src/ioq.h
index 87727cb..e1e5052 100644
--- a/src/ioq.h
+++ b/src/ioq.h
@@ -36,10 +36,8 @@ struct ioq_ent {
 	/** The I/O operation. */
 	enum ioq_op op;
 
-	/** The return value of the operation. */
-	int ret;
-	/** The error code, if the operation failed. */
-	int error;
+	/** The return value (on success) or negative error code (on failure). */
+	int result;
 
 	/** Arbitrary user data. */
 	void *ptr;
-- 
cgit v1.2.3


From 58de21264be085b96c52d48db2cddd49ce40bde3 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 17 Jan 2024 12:11:18 -0500
Subject: ioq: Refactor to take advantage of -Wswitch

---
 src/ioq.c | 162 +++++++++++++++++++++++++++++++++++---------------------------
 1 file changed, 91 insertions(+), 71 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 89ebb3e..3d32dfe 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -460,34 +460,39 @@ static bool ioq_check_cancel(struct ioq *ioq, struct ioq_ent *ent) {
 	return true;
 }
 
-/** Handle a single request synchronously. */
-static void ioq_handle(struct ioq *ioq, struct ioq_ent *ent) {
+/** Dispatch a single request synchronously. */
+static void ioq_dispatch_sync(struct ioq *ioq, struct ioq_ent *ent) {
 	switch (ent->op) {
-	case IOQ_CLOSE:
-		ent->result = try(xclose(ent->close.fd));
-		break;
+		case IOQ_CLOSE:
+			ent->result = try(xclose(ent->close.fd));
+			return;
 
-	case IOQ_OPENDIR:
-		ent->result = try(bfs_opendir(ent->opendir.dir, ent->opendir.dfd, ent->opendir.path, ent->opendir.flags));
-		if (ent->result >= 0) {
-			bfs_polldir(ent->opendir.dir);
+		case IOQ_OPENDIR: {
+			struct ioq_opendir *args = &ent->opendir;
+			ent->result = try(bfs_opendir(args->dir, args->dfd, args->path, args->flags));
+			if (ent->result >= 0) {
+				bfs_polldir(args->dir);
+			}
+			return;
 		}
-		break;
 
-	case IOQ_CLOSEDIR:
-		ent->result = try(bfs_closedir(ent->closedir.dir));
-		break;
-
-	default:
-		bfs_bug("Unknown ioq_op %d", (int)ent->op);
-		ent->result = -ENOSYS;
-		break;
+		case IOQ_CLOSEDIR:
+			ent->result = try(bfs_closedir(ent->closedir.dir));
+			return;
 	}
 
+	bfs_bug("Unknown ioq_op %d", (int)ent->op);
+	ent->result = -ENOSYS;
+}
+
+/** Complete a single request synchronously. */
+static void ioq_complete(struct ioq *ioq, struct ioq_ent *ent) {
+	ioq_dispatch_sync(ioq, ent);
 	ioqq_push(ioq->ready, ent);
 }
 
 #if BFS_USE_LIBURING
+
 /** io_uring worker state. */
 struct ioq_ring_state {
 	/** The I/O queue. */
@@ -520,35 +525,54 @@ static struct ioq_ent *ioq_ring_pop(struct ioq_ring_state *state) {
 	return ret;
 }
 
-/** Prep a single SQE. */
-static void ioq_prep_sqe(struct io_uring_sqe *sqe, struct ioq_ent *ent) {
-	switch (ent->op) {
-	case IOQ_CLOSE:
-		io_uring_prep_close(sqe, ent->close.fd);
-		break;
+/** Dispatch a single request asynchronously. */
+static struct io_uring_sqe *ioq_dispatch_async(struct io_uring *ring, struct ioq_ent *ent) {
+	struct io_uring_sqe *sqe = NULL;
 
-	case IOQ_OPENDIR:
-		io_uring_prep_openat(sqe, ent->opendir.dfd, ent->opendir.path, O_RDONLY | O_CLOEXEC | O_DIRECTORY, 0);
-		break;
+	switch (ent->op) {
+		case IOQ_CLOSE:
+			sqe = io_uring_get_sqe(ring);
+			io_uring_prep_close(sqe, ent->close.fd);
+			return sqe;
+
+		case IOQ_OPENDIR: {
+			sqe = io_uring_get_sqe(ring);
+			struct ioq_opendir *args = &ent->opendir;
+			int flags = O_RDONLY | O_CLOEXEC | O_DIRECTORY;
+			io_uring_prep_openat(sqe, args->dfd, args->path, flags, 0);
+			return sqe;
+		}
 
+		case IOQ_CLOSEDIR:
 #if BFS_USE_UNWRAPDIR
-	case IOQ_CLOSEDIR:
-		io_uring_prep_close(sqe, bfs_unwrapdir(ent->closedir.dir));
-		break;
+			sqe = io_uring_get_sqe(ring);
+			io_uring_prep_close(sqe, bfs_unwrapdir(ent->closedir.dir));
 #endif
+			return sqe;
+	}
 
-	default:
-		bfs_bug("Unknown ioq_op %d", (int)ent->op);
-		io_uring_prep_nop(sqe);
-		break;
+	bfs_bug("Unknown ioq_op %d", (int)ent->op);
+	return NULL;
+}
+
+/** Prep a single SQE. */
+static void ioq_prep_sqe(struct ioq_ring_state *state, struct ioq_ent *ent) {
+	struct ioq *ioq = state->ioq;
+	if (ioq_check_cancel(ioq, ent)) {
+		return;
 	}
 
-	io_uring_sqe_set_data(sqe, ent);
+	struct io_uring_sqe *sqe = ioq_dispatch_async(state->ring, ent);
+	if (sqe) {
+		io_uring_sqe_set_data(sqe, ent);
+		++state->prepped;
+	} else {
+		ioq_complete(ioq, ent);
+	}
 }
 
 /** Prep a batch of SQEs. */
 static bool ioq_ring_prep(struct ioq_ring_state *state) {
-	struct ioq *ioq = state->ioq;
 	struct io_uring *ring = state->ring;
 
 	while (io_uring_sq_space_left(ring)) {
@@ -557,28 +581,42 @@ static bool ioq_ring_prep(struct ioq_ring_state *state) {
 			break;
 		}
 
+		ioq_prep_sqe(state, ent);
+	}
+
+	return state->prepped || state->submitted;
+}
+
+/** Reap a single CQE. */
+static void ioq_reap_cqe(struct ioq_ring_state *state, struct io_uring_cqe *cqe) {
+	struct ioq *ioq = state->ioq;
+	struct io_uring *ring = state->ring;
+
+	struct ioq_ent *ent = io_uring_cqe_get_data(cqe);
+	ent->result = cqe->res;
+	io_uring_cqe_seen(ring, cqe);
+	--state->submitted;
+
+	if (ent->op == IOQ_OPENDIR && ent->result >= 0) {
+		int fd = ent->result;
 		if (ioq_check_cancel(ioq, ent)) {
-			continue;
+			xclose(fd);
+			return;
 		}
 
-#if !BFS_USE_UNWRAPDIR
-		if (ent->op == IOQ_CLOSEDIR) {
-			ioq_handle(ioq, ent);
-			continue;
+		struct ioq_opendir *args = &ent->opendir;
+		ent->result = try(bfs_opendir(args->dir, fd, NULL, args->flags));
+		if (ent->result >= 0) {
+			// TODO: io_uring_prep_getdents()
+			bfs_polldir(args->dir);
 		}
-#endif
-
-		struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
-		ioq_prep_sqe(sqe, ent);
-		++state->prepped;
 	}
 
-	return state->prepped || state->submitted;
+	ioqq_push(ioq->ready, ent);
 }
 
-/** Reap a batch of SQEs. */
+/** Reap a batch of CQEs. */
 static void ioq_ring_reap(struct ioq_ring_state *state) {
-	struct ioq *ioq = state->ioq;
 	struct io_uring *ring = state->ring;
 
 	while (state->prepped) {
@@ -595,26 +633,7 @@ static void ioq_ring_reap(struct ioq_ring_state *state) {
 			continue;
 		}
 
-		struct ioq_ent *ent = io_uring_cqe_get_data(cqe);
-		ent->result = cqe->res;
-		io_uring_cqe_seen(ring, cqe);
-		--state->submitted;
-
-		if (ent->op == IOQ_OPENDIR && ent->result >= 0) {
-			int fd = ent->result;
-			if (ioq_check_cancel(ioq, ent)) {
-				xclose(fd);
-				continue;
-			}
-
-			ent->result = try(bfs_opendir(ent->opendir.dir, fd, NULL, ent->opendir.flags));
-			if (ent->result >= 0) {
-				// TODO: io_uring_prep_getdents()
-				bfs_polldir(ent->opendir.dir);
-			}
-		}
-
-		ioqq_push(ioq->ready, ent);
+		ioq_reap_cqe(state, cqe);
 	}
 }
 
@@ -629,7 +648,8 @@ static void ioq_ring_work(struct ioq_thread *thread) {
 		ioq_ring_reap(&state);
 	}
 }
-#endif
+
+#endif // BFS_USE_LIBURING
 
 /** Synchronous syscall loop. */
 static void ioq_sync_work(struct ioq_thread *thread) {
@@ -642,7 +662,7 @@ static void ioq_sync_work(struct ioq_thread *thread) {
 		}
 
 		if (!ioq_check_cancel(ioq, ent)) {
-			ioq_handle(ioq, ent);
+			ioq_complete(ioq, ent);
 		}
 	}
 }
-- 
cgit v1.2.3


From 18597572f62ab18824df6bab6c30d2b645921969 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 23 Aug 2023 13:10:10 -0400
Subject: ioq: Implement ioq_stat()

---
 src/bftw.c |   3 ++
 src/ioq.c  | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-------
 src/ioq.h  |  31 +++++++++++++++++++
 3 files changed, 124 insertions(+), 10 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/bftw.c b/src/bftw.c
index 952b090..02dd051 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -631,6 +631,9 @@ static int bftw_ioq_pop(struct bftw_state *state, bool block) {
 			SLIST_APPEND(&state->to_read, file, to_read);
 		}
 		break;
+
+	case IOQ_STAT:
+		break;
 	}
 
 	ioq_free(ioq, ent);
diff --git a/src/ioq.c b/src/ioq.c
index 3d32dfe..3172f0a 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -131,6 +131,7 @@
 #include "diag.h"
 #include "dir.h"
 #include "sanity.h"
+#include "stat.h"
 #include "thread.h"
 #include <assert.h>
 #include <errno.h>
@@ -432,6 +433,10 @@ struct ioq {
 
 	/** ioq_ent arena. */
 	struct arena ents;
+#if BFS_USE_LIBURING && BFS_USE_STATX
+	/** struct statx arena. */
+	struct arena xbufs;
+#endif
 
 	/** Pending I/O requests. */
 	struct ioqq *pending;
@@ -479,6 +484,12 @@ static void ioq_dispatch_sync(struct ioq *ioq, struct ioq_ent *ent) {
 		case IOQ_CLOSEDIR:
 			ent->result = try(bfs_closedir(ent->closedir.dir));
 			return;
+
+		case IOQ_STAT: {
+			struct ioq_stat *args = &ent->stat;
+			ent->result = try(bfs_stat(args->dfd, args->path, args->flags, args->buf));
+			return;
+		}
 	}
 
 	bfs_bug("Unknown ioq_op %d", (int)ent->op);
@@ -549,6 +560,17 @@ static struct io_uring_sqe *ioq_dispatch_async(struct io_uring *ring, struct ioq
 			io_uring_prep_close(sqe, bfs_unwrapdir(ent->closedir.dir));
 #endif
 			return sqe;
+
+		case IOQ_STAT: {
+#if BFS_USE_STATX
+			sqe = io_uring_get_sqe(ring);
+			struct ioq_stat *args = &ent->stat;
+			int flags = bfs_statx_flags(args->flags);
+			unsigned int mask = STATX_BASIC_STATS | STATX_BTIME;
+			io_uring_prep_statx(sqe, args->dfd, args->path, flags, mask, args->xbuf);
+#endif
+			return sqe;
+		}
 	}
 
 	bfs_bug("Unknown ioq_op %d", (int)ent->op);
@@ -597,21 +619,41 @@ static void ioq_reap_cqe(struct ioq_ring_state *state, struct io_uring_cqe *cqe)
 	io_uring_cqe_seen(ring, cqe);
 	--state->submitted;
 
-	if (ent->op == IOQ_OPENDIR && ent->result >= 0) {
-		int fd = ent->result;
-		if (ioq_check_cancel(ioq, ent)) {
-			xclose(fd);
-			return;
+	if (ent->result < 0) {
+		goto push;
+	}
+
+	switch (ent->op) {
+		case IOQ_OPENDIR: {
+			int fd = ent->result;
+			if (ioq_check_cancel(ioq, ent)) {
+				xclose(fd);
+				return;
+			}
+
+			struct ioq_opendir *args = &ent->opendir;
+			ent->result = try(bfs_opendir(args->dir, fd, NULL, args->flags));
+			if (ent->result >= 0) {
+				// TODO: io_uring_prep_getdents()
+				bfs_polldir(args->dir);
+			}
+
+			break;
 		}
 
-		struct ioq_opendir *args = &ent->opendir;
-		ent->result = try(bfs_opendir(args->dir, fd, NULL, args->flags));
-		if (ent->result >= 0) {
-			// TODO: io_uring_prep_getdents()
-			bfs_polldir(args->dir);
+#if BFS_USE_STATX
+		case IOQ_STAT: {
+			struct ioq_stat *args = &ent->stat;
+			ent->result = try(bfs_statx_convert(args->buf, args->xbuf));
+			break;
 		}
+#endif
+
+		default:
+			break;
 	}
 
+push:
 	ioqq_push(ioq->ready, ent);
 }
 
@@ -689,8 +731,13 @@ struct ioq *ioq_create(size_t depth, size_t nthreads) {
 	}
 
 	ioq->depth = depth;
+
 	ARENA_INIT(&ioq->ents, struct ioq_ent);
 
+#if BFS_USE_LIBURING && BFS_USE_STATX
+	ARENA_INIT(&ioq->xbufs, struct statx);
+#endif
+
 	ioq->pending = ioqq_create(depth);
 	if (!ioq->pending) {
 		goto fail;
@@ -807,6 +854,30 @@ int ioq_closedir(struct ioq *ioq, struct bfs_dir *dir, void *ptr) {
 	return 0;
 }
 
+int ioq_stat(struct ioq *ioq, int dfd, const char *path, enum bfs_stat_flags flags, struct bfs_stat *buf, void *ptr) {
+	struct ioq_ent *ent = ioq_request(ioq, IOQ_STAT, ptr);
+	if (!ent) {
+		return -1;
+	}
+
+	struct ioq_stat *args = &ent->stat;
+	args->dfd = dfd;
+	args->path = path;
+	args->flags = flags;
+	args->buf = buf;
+
+#if BFS_USE_LIBURING && BFS_USE_STATX
+	args->xbuf = arena_alloc(&ioq->xbufs);
+	if (!args->xbuf) {
+		ioq_free(ioq, ent);
+		return -1;
+	}
+#endif
+
+	ioqq_push(ioq->pending, ent);
+	return 0;
+}
+
 struct ioq_ent *ioq_pop(struct ioq *ioq, bool block) {
 	if (ioq->size == 0) {
 		return NULL;
@@ -819,6 +890,12 @@ void ioq_free(struct ioq *ioq, struct ioq_ent *ent) {
 	bfs_assert(ioq->size > 0);
 	--ioq->size;
 
+#if BFS_USE_LIBURING && BFS_USE_STATX
+	if (ent->op == IOQ_STAT) {
+		arena_free(&ioq->xbufs, ent->stat.xbuf);
+	}
+#endif
+
 	arena_free(&ioq->ents, ent);
 }
 
@@ -848,6 +925,9 @@ void ioq_destroy(struct ioq *ioq) {
 	ioqq_destroy(ioq->ready);
 	ioqq_destroy(ioq->pending);
 
+#if BFS_USE_LIBURING && BFS_USE_STATX
+	arena_destroy(&ioq->xbufs);
+#endif
 	arena_destroy(&ioq->ents);
 
 	free(ioq);
diff --git a/src/ioq.h b/src/ioq.h
index e1e5052..77aabaa 100644
--- a/src/ioq.h
+++ b/src/ioq.h
@@ -10,6 +10,7 @@
 
 #include "config.h"
 #include "dir.h"
+#include "stat.h"
 #include <stddef.h>
 
 /**
@@ -27,6 +28,8 @@ enum ioq_op {
 	IOQ_OPENDIR,
 	/** ioq_closedir(). */
 	IOQ_CLOSEDIR,
+	/** ioq_stat(). */
+	IOQ_STAT,
 };
 
 /**
@@ -59,6 +62,14 @@ struct ioq_ent {
 		struct ioq_closedir {
 			struct bfs_dir *dir;
 		} closedir;
+		/** ioq_stat() args. */
+		struct ioq_stat {
+			int dfd;
+			const char *path;
+			enum bfs_stat_flags flags;
+			struct bfs_stat *buf;
+			void *xbuf;
+		} stat;
 	};
 };
 
@@ -127,6 +138,26 @@ int ioq_opendir(struct ioq *ioq, struct bfs_dir *dir, int dfd, const char *path,
  */
 int ioq_closedir(struct ioq *ioq, struct bfs_dir *dir, void *ptr);
 
+/**
+ * Asynchronous bfs_stat().
+ *
+ * @param ioq
+ *         The I/O queue.
+ * @param dfd
+ *         The base file descriptor.
+ * @param path
+ *         The path to stat, relative to dfd.
+ * @param flags
+ *         Flags that affect the lookup.
+ * @param buf
+ *         A place to store the stat buffer, if successful.
+ * @param ptr
+ *         An arbitrary pointer to associate with the request.
+ * @return
+ *         0 on success, or -1 on failure.
+ */
+int ioq_stat(struct ioq *ioq, int dfd, const char *path, enum bfs_stat_flags flags, struct bfs_stat *buf, void *ptr);
+
 /**
  * Pop a response from the queue.
  *
-- 
cgit v1.2.3


From 1e5cd0a3d4585793c5a4d16ab60473a57e18af23 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Mon, 12 Feb 2024 13:24:11 -0500
Subject: ioq: Shrink the io_urings

---
 src/ioq.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 3172f0a..50550ed 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -764,10 +764,8 @@ struct ioq *ioq_create(size_t depth, size_t nthreads) {
 				params.wq_fd = prev->ring.ring_fd;
 			}
 
-			size_t entries = depth / nthreads;
-			if (entries < 16) {
-				entries = 16;
-			}
+			// Use a page for each SQE ring
+			size_t entries = 4096 / sizeof(struct io_uring_sqe);
 			thread->ring_err = -io_uring_queue_init_params(entries, &thread->ring, &params);
 		}
 #endif
-- 
cgit v1.2.3


From a98fe72db88350fcec030487208e6c50c9de1974 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Mon, 12 Feb 2024 13:26:10 -0500
Subject: ioq: Get rid of IOQ_STRIDE

Benchmarks show it hurts more than it helps.
---
 src/ioq.c | 25 +++++--------------------
 1 file changed, 5 insertions(+), 20 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 50550ed..438601d 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -26,15 +26,11 @@
  *
  * Pushes are implemented with an unconditional
  *
- *     fetch_add(&ioqq->head, IOQ_STRIDE)
+ *     fetch_add(&ioqq->head, 1)
  *
  * which scales better on many architectures than compare-and-swap (see [1] for
- * details).  Pops are implemented similarly.  We add IOQ_STRIDE rather than 1
- * so that successive queue elements are on different cache lines, but the
- * exposition below uses 1 for simplicity.
- *
- * Since the fetch-and-adds are unconditional, non-blocking readers can get
- * ahead of writers:
+ * details).  Pops are implemented similarly.  Since the fetch-and-adds are
+ * unconditional, non-blocking readers can get ahead of writers:
  *
  *     Reader              Writer
  *     ────────────────    ──────────────────────
@@ -204,17 +200,6 @@ struct ioqq {
 	cache_align ioq_slot slots[];
 };
 
-// If we assign slots sequentially, threads will likely be operating on
-// consecutive slots.  If these slots are in the same cache line, that will
-// result in false sharing.  We can mitigate this by assigning slots with a
-// stride larger than a cache line e.g. 0, 9, 18, ..., 1, 10, 19, ...
-// As long as the stride is relatively prime to circular buffer length, we'll
-// still use every available slot.  Since the length is a power of two, that
-// means the stride must be odd.
-
-#define IOQ_STRIDE ((FALSE_SHARING_SIZE / sizeof(ioq_slot)) | 1)
-bfs_static_assert(IOQ_STRIDE % 2 == 1);
-
 /** Destroy an I/O command queue. */
 static void ioqq_destroy(struct ioqq *ioqq) {
 	if (!ioqq) {
@@ -357,7 +342,7 @@ static bool ioq_slot_push(struct ioqq *ioqq, ioq_slot *slot, struct ioq_ent *ent
 /** Push an entry onto the queue. */
 static void ioqq_push(struct ioqq *ioqq, struct ioq_ent *ent) {
 	while (true) {
-		size_t i = fetch_add(&ioqq->head, IOQ_STRIDE, relaxed);
+		size_t i = fetch_add(&ioqq->head, 1, relaxed);
 		ioq_slot *slot = &ioqq->slots[i & ioqq->slot_mask];
 		if (ioq_slot_push(ioqq, slot, ent)) {
 			break;
@@ -400,7 +385,7 @@ static struct ioq_ent *ioq_slot_pop(struct ioqq *ioqq, ioq_slot *slot, bool bloc
 
 /** Pop an entry from the queue. */
 static struct ioq_ent *ioqq_pop(struct ioqq *ioqq, bool block) {
-	size_t i = fetch_add(&ioqq->tail, IOQ_STRIDE, relaxed);
+	size_t i = fetch_add(&ioqq->tail, 1, relaxed);
 	ioq_slot *slot = &ioqq->slots[i & ioqq->slot_mask];
 	return ioq_slot_pop(ioqq, slot, block);
 }
-- 
cgit v1.2.3


From 5f7cc43ba12c87b70fe1e8f8f225258e718048ba Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 14 Feb 2024 14:03:26 -0500
Subject: ioq: Replay IOQ_STOP messages rather than spam them

---
 src/ioq.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 438601d..ede2413 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -511,9 +511,11 @@ static struct ioq_ent *ioq_ring_pop(struct ioq_ring_state *state) {
 
 	// Block if we have nothing else to do
 	bool block = !state->prepped && !state->submitted;
-	struct ioq_ent *ret = ioqq_pop(state->ioq->pending, block);
+	struct ioqq *pending = state->ioq->pending;
+	struct ioq_ent *ret = ioqq_pop(pending, block);
 
 	if (ret == &IOQ_STOP) {
+		ioqq_push(pending, &IOQ_STOP);
 		state->stop = true;
 		ret = NULL;
 	}
@@ -685,6 +687,7 @@ static void ioq_sync_work(struct ioq_thread *thread) {
 	while (true) {
 		struct ioq_ent *ent = ioqq_pop(ioq->pending, true);
 		if (ent == &IOQ_STOP) {
+			ioqq_push(ioq->pending, &IOQ_STOP);
 			break;
 		}
 
@@ -884,9 +887,7 @@ void ioq_free(struct ioq *ioq, struct ioq_ent *ent) {
 
 void ioq_cancel(struct ioq *ioq) {
 	if (!exchange(&ioq->cancel, true, relaxed)) {
-		for (size_t i = 0; i < ioq->nthreads; ++i) {
-			ioqq_push(ioq->pending, &IOQ_STOP);
-		}
+		ioqq_push(ioq->pending, &IOQ_STOP);
 	}
 }
 
-- 
cgit v1.2.3


From fc2ce8a878bd5acb8881e9408f6e0484d9b446c5 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 14 Feb 2024 14:37:23 -0500
Subject: ioq: Make -j also limit the io_uring worker threads

---
 src/ioq.c | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index ede2413..0126f5c 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -755,6 +755,15 @@ struct ioq *ioq_create(size_t depth, size_t nthreads) {
 			// Use a page for each SQE ring
 			size_t entries = 4096 / sizeof(struct io_uring_sqe);
 			thread->ring_err = -io_uring_queue_init_params(entries, &thread->ring, &params);
+
+			if (!prev && thread->ring_err == 0) {
+				// Limit the number of io_uring workers
+				unsigned int values[] = {
+					[IO_WQ_BOUND] = nthreads,
+					[IO_WQ_UNBOUND] = 0,
+				};
+				io_uring_register_iowq_max_workers(&thread->ring, values);
+			}
 		}
 #endif
 
-- 
cgit v1.2.3


From 80fbd584adf480f62e3282d570f67767c74c13d3 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 14 Feb 2024 15:05:57 -0500
Subject: ioq: Factor out io_uring initialization

---
 src/ioq.c | 110 +++++++++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 73 insertions(+), 37 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 0126f5c..3c26814 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -712,6 +712,75 @@ static void *ioq_work(void *ptr) {
 	return NULL;
 }
 
+/** Initialize io_uring thread state. */
+static int ioq_ring_init(struct ioq *ioq, struct ioq_thread *thread) {
+#if BFS_USE_LIBURING
+	struct ioq_thread *prev = NULL;
+	if (thread > ioq->threads) {
+		prev = thread - 1;
+	}
+
+	if (prev && prev->ring_err) {
+		thread->ring_err = prev->ring_err;
+		return -1;
+	}
+
+	// Share io-wq workers between rings
+	struct io_uring_params params = {0};
+	if (prev) {
+		params.flags |= IORING_SETUP_ATTACH_WQ;
+		params.wq_fd = prev->ring.ring_fd;
+	}
+
+	// Use a page for each SQE ring
+	size_t entries = 4096 / sizeof(struct io_uring_sqe);
+	thread->ring_err = -io_uring_queue_init_params(entries, &thread->ring, &params);
+	if (thread->ring_err) {
+		return -1;
+	}
+
+	if (!prev) {
+		// Limit the number of io_uring workers
+		unsigned int values[] = {
+			[IO_WQ_BOUND] = ioq->nthreads,
+			[IO_WQ_UNBOUND] = 0,
+		};
+		io_uring_register_iowq_max_workers(&thread->ring, values);
+	}
+#endif
+
+	return 0;
+}
+
+/** Destroy an io_uring. */
+static void ioq_ring_exit(struct ioq_thread *thread) {
+#if BFS_USE_LIBURING
+	if (thread->ring_err == 0) {
+		io_uring_queue_exit(&thread->ring);
+	}
+#endif
+}
+
+/** Create an I/O queue thread. */
+static int ioq_thread_create(struct ioq *ioq, struct ioq_thread *thread) {
+	thread->parent = ioq;
+
+	ioq_ring_init(ioq, thread);
+
+	if (thread_create(&thread->id, NULL, ioq_work, thread) != 0) {
+		ioq_ring_exit(thread);
+		return -1;
+	}
+
+	return 0;
+}
+
+/** Join an I/O queue thread. */
+static void ioq_thread_join(struct ioq_thread *thread) {
+	thread_join(thread->id, NULL);
+	ioq_ring_exit(thread);
+}
+
 struct ioq *ioq_create(size_t depth, size_t nthreads) {
 	struct ioq *ioq = ZALLOC_FLEX(struct ioq, threads, nthreads);
 	if (!ioq) {
@@ -736,41 +805,12 @@ struct ioq *ioq_create(size_t depth, size_t nthreads) {
 		goto fail;
 	}
 
+	ioq->nthreads = nthreads;
 	for (size_t i = 0; i < nthreads; ++i) {
-		struct ioq_thread *thread = &ioq->threads[i];
-		thread->parent = ioq;
-
-#if BFS_USE_LIBURING
-		struct ioq_thread *prev = i ? &ioq->threads[i - 1] : NULL;
-		if (prev && prev->ring_err) {
-			thread->ring_err = prev->ring_err;
-		} else {
-			// Share io-wq workers between rings
-			struct io_uring_params params = {0};
-			if (prev) {
-				params.flags |= IORING_SETUP_ATTACH_WQ;
-				params.wq_fd = prev->ring.ring_fd;
-			}
-
-			// Use a page for each SQE ring
-			size_t entries = 4096 / sizeof(struct io_uring_sqe);
-			thread->ring_err = -io_uring_queue_init_params(entries, &thread->ring, &params);
-
-			if (!prev && thread->ring_err == 0) {
-				// Limit the number of io_uring workers
-				unsigned int values[] = {
-					[IO_WQ_BOUND] = nthreads,
-					[IO_WQ_UNBOUND] = 0,
-				};
-				io_uring_register_iowq_max_workers(&thread->ring, values);
-			}
-		}
-#endif
-
-		if (thread_create(&thread->id, NULL, ioq_work, thread) != 0) {
+		if (ioq_thread_create(ioq, &ioq->threads[i]) != 0) {
+			ioq->nthreads = i;
 			goto fail;
 		}
-		++ioq->nthreads;
 	}
 
 	return ioq;
@@ -908,11 +948,7 @@ void ioq_destroy(struct ioq *ioq) {
 	ioq_cancel(ioq);
 
 	for (size_t i = 0; i < ioq->nthreads; ++i) {
-		struct ioq_thread *thread = &ioq->threads[i];
-		thread_join(thread->id, NULL);
-#if BFS_USE_LIBURING
-		io_uring_queue_exit(&thread->ring);
-#endif
+		ioq_thread_join(&ioq->threads[i]);
 	}
 
 	ioqq_destroy(ioq->ready);
-- 
cgit v1.2.3


From b3cdf12d71798a26c1f6d46dceff7ed50b9eed83 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 14 Feb 2024 16:29:12 -0500
Subject: ioq: Don't use the symbolic IO_WQ_[UN]BOUND indices

They are only available since liburing 2.2, which is newer than CI.
---
 src/ioq.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 3c26814..0936adf 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -742,8 +742,8 @@ static int ioq_ring_init(struct ioq *ioq, struct ioq_thread *thread) {
 	if (!prev) {
 		// Limit the number of io_uring workers
 		unsigned int values[] = {
-			[IO_WQ_BOUND] = ioq->nthreads,
-			[IO_WQ_UNBOUND] = 0,
+			ioq->nthreads, // [IO_WQ_BOUND]
+			0,             // [IO_WQ_UNBOUND]
 		};
 		io_uring_register_iowq_max_workers(&thread->ring, values);
 	}
-- 
cgit v1.2.3


From 08e23800dcccc0bc302dabc18ba5f5b8f78c846d Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Thu, 15 Feb 2024 12:43:27 -0500
Subject: ioq: Add a missing close() if bfs_opendir() fails

---
 src/ioq.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 0936adf..2558d62 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -623,6 +623,8 @@ static void ioq_reap_cqe(struct ioq_ring_state *state, struct io_uring_cqe *cqe)
 			if (ent->result >= 0) {
 				// TODO: io_uring_prep_getdents()
 				bfs_polldir(args->dir);
+			} else {
+				xclose(fd);
 			}
 
 			break;
-- 
cgit v1.2.3


From 71cffe72d83dd3ba31198d66a97cea83ba6b352e Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Thu, 15 Feb 2024 12:46:23 -0500
Subject: ioq: Don't push immediately in ioq_check_cancel()

---
 src/ioq.c | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 2558d62..b57daba 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -446,7 +446,6 @@ static bool ioq_check_cancel(struct ioq *ioq, struct ioq_ent *ent) {
 	}
 
 	ent->result = -EINTR;
-	ioqq_push(ioq->ready, ent);
 	return true;
 }
 
@@ -481,12 +480,6 @@ static void ioq_dispatch_sync(struct ioq *ioq, struct ioq_ent *ent) {
 	ent->result = -ENOSYS;
 }
 
-/** Complete a single request synchronously. */
-static void ioq_complete(struct ioq *ioq, struct ioq_ent *ent) {
-	ioq_dispatch_sync(ioq, ent);
-	ioqq_push(ioq->ready, ent);
-}
-
 #if BFS_USE_LIBURING
 
 /** io_uring worker state. */
@@ -568,6 +561,7 @@ static struct io_uring_sqe *ioq_dispatch_async(struct io_uring *ring, struct ioq
 static void ioq_prep_sqe(struct ioq_ring_state *state, struct ioq_ent *ent) {
 	struct ioq *ioq = state->ioq;
 	if (ioq_check_cancel(ioq, ent)) {
+		ioqq_push(ioq->ready, ent);
 		return;
 	}
 
@@ -576,7 +570,8 @@ static void ioq_prep_sqe(struct ioq_ring_state *state, struct ioq_ent *ent) {
 		io_uring_sqe_set_data(sqe, ent);
 		++state->prepped;
 	} else {
-		ioq_complete(ioq, ent);
+		ioq_dispatch_sync(ioq, ent);
+		ioqq_push(ioq->ready, ent);
 	}
 }
 
@@ -615,7 +610,7 @@ static void ioq_reap_cqe(struct ioq_ring_state *state, struct io_uring_cqe *cqe)
 			int fd = ent->result;
 			if (ioq_check_cancel(ioq, ent)) {
 				xclose(fd);
-				return;
+				goto push;
 			}
 
 			struct ioq_opendir *args = &ent->opendir;
@@ -694,8 +689,9 @@ static void ioq_sync_work(struct ioq_thread *thread) {
 		}
 
 		if (!ioq_check_cancel(ioq, ent)) {
-			ioq_complete(ioq, ent);
+			ioq_dispatch_sync(ioq, ent);
 		}
+		ioqq_push(ioq->ready, ent);
 	}
 }
 
-- 
cgit v1.2.3


From 9c40099611099c567d26c7a8b2782fa9a47d6849 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Thu, 15 Feb 2024 12:53:31 -0500
Subject: ioq: Add batched ioqq_push/pop operations

---
 src/ioq.c | 176 +++++++++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 127 insertions(+), 49 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index b57daba..cf0b927 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -339,17 +339,6 @@ static bool ioq_slot_push(struct ioqq *ioqq, ioq_slot *slot, struct ioq_ent *ent
 	return !(prev & IOQ_SKIP);
 }
 
-/** Push an entry onto the queue. */
-static void ioqq_push(struct ioqq *ioqq, struct ioq_ent *ent) {
-	while (true) {
-		size_t i = fetch_add(&ioqq->head, 1, relaxed);
-		ioq_slot *slot = &ioqq->slots[i & ioqq->slot_mask];
-		if (ioq_slot_push(ioqq, slot, ent)) {
-			break;
-		}
-	}
-}
-
 /** (Try to) pop an entry from a slot. */
 static struct ioq_ent *ioq_slot_pop(struct ioqq *ioqq, ioq_slot *slot, bool block) {
 	uintptr_t prev = load(slot, relaxed);
@@ -383,6 +372,32 @@ static struct ioq_ent *ioq_slot_pop(struct ioqq *ioqq, ioq_slot *slot, bool bloc
 	return (struct ioq_ent *)(prev << 1);
 }
 
+/** Push an entry onto the queue. */
+static void ioqq_push(struct ioqq *ioqq, struct ioq_ent *ent) {
+	while (true) {
+		size_t i = fetch_add(&ioqq->head, 1, relaxed);
+		ioq_slot *slot = &ioqq->slots[i & ioqq->slot_mask];
+		if (ioq_slot_push(ioqq, slot, ent)) {
+			break;
+		}
+	}
+}
+
+/** Push a batch of entries to the queue. */
+static void ioqq_push_batch(struct ioqq *ioqq, struct ioq_ent *batch[], size_t size) {
+	size_t mask = ioqq->slot_mask;
+	do {
+		size_t i = fetch_add(&ioqq->head, size, relaxed);
+		for (size_t j = i + size; i != j; ++i) {
+			ioq_slot *slot = &ioqq->slots[i & mask];
+			if (ioq_slot_push(ioqq, slot, *batch)) {
+				++batch;
+				--size;
+			}
+		}
+	} while (size > 0);
+}
+
 /** Pop an entry from the queue. */
 static struct ioq_ent *ioqq_pop(struct ioqq *ioqq, bool block) {
 	size_t i = fetch_add(&ioqq->tail, 1, relaxed);
@@ -390,6 +405,47 @@ static struct ioq_ent *ioqq_pop(struct ioqq *ioqq, bool block) {
 	return ioq_slot_pop(ioqq, slot, block);
 }
 
+/** Pop a batch of entries from the queue. */
+static void ioqq_pop_batch(struct ioqq *ioqq, struct ioq_ent *batch[], size_t size, bool block) {
+	size_t mask = ioqq->slot_mask;
+	size_t i = fetch_add(&ioqq->tail, size, relaxed);
+	for (size_t j = i + size; i != j; ++i) {
+		ioq_slot *slot = &ioqq->slots[i & mask];
+		*batch++ = ioq_slot_pop(ioqq, slot, block);
+		block = false;
+	}
+}
+
+/** Use cache-line-sized batches. */
+#define IOQ_BATCH (FALSE_SHARING_SIZE / sizeof(ioq_slot))
+
+/**
+ * A batch of entries to send all at once.
+ */
+struct ioq_batch {
+	/** The current batch size. */
+	size_t size;
+	/** The array of entries. */
+	struct ioq_ent *entries[IOQ_BATCH];
+};
+
+/** Send the batch to a queue. */
+static void ioq_batch_flush(struct ioqq *ioqq, struct ioq_batch *batch) {
+	if (batch->size > 0) {
+		ioqq_push_batch(ioqq, batch->entries, batch->size);
+		batch->size = 0;
+	}
+}
+
+/** An an entry to a batch, flushing if necessary. */
+static void ioq_batch_push(struct ioqq *ioqq, struct ioq_batch *batch, struct ioq_ent *ent) {
+	if (batch->size >= IOQ_BATCH) {
+		ioq_batch_flush(ioqq, batch);
+	}
+
+	batch->entries[batch->size++] = ent;
+}
+
 /** Sentinel stop command. */
 static struct ioq_ent IOQ_STOP;
 
@@ -494,28 +550,10 @@ struct ioq_ring_state {
 	size_t submitted;
 	/** Whether to stop the loop. */
 	bool stop;
+	/** A batch of ready entries. */
+	struct ioq_batch ready;
 };
 
-/** Pop a request for ioq_ring_prep(). */
-static struct ioq_ent *ioq_ring_pop(struct ioq_ring_state *state) {
-	if (state->stop) {
-		return NULL;
-	}
-
-	// Block if we have nothing else to do
-	bool block = !state->prepped && !state->submitted;
-	struct ioqq *pending = state->ioq->pending;
-	struct ioq_ent *ret = ioqq_pop(pending, block);
-
-	if (ret == &IOQ_STOP) {
-		ioqq_push(pending, &IOQ_STOP);
-		state->stop = true;
-		ret = NULL;
-	}
-
-	return ret;
-}
-
 /** Dispatch a single request asynchronously. */
 static struct io_uring_sqe *ioq_dispatch_async(struct io_uring *ring, struct ioq_ent *ent) {
 	struct io_uring_sqe *sqe = NULL;
@@ -557,11 +595,16 @@ static struct io_uring_sqe *ioq_dispatch_async(struct io_uring *ring, struct ioq
 	return NULL;
 }
 
+/** Check if ioq_ring_reap() has work to do. */
+static bool ioq_ring_empty(struct ioq_ring_state *state) {
+	return !state->prepped && !state->submitted && !state->ready.size;
+}
+
 /** Prep a single SQE. */
 static void ioq_prep_sqe(struct ioq_ring_state *state, struct ioq_ent *ent) {
 	struct ioq *ioq = state->ioq;
 	if (ioq_check_cancel(ioq, ent)) {
-		ioqq_push(ioq->ready, ent);
+		ioq_batch_push(ioq->ready, &state->ready, ent);
 		return;
 	}
 
@@ -571,24 +614,44 @@ static void ioq_prep_sqe(struct ioq_ring_state *state, struct ioq_ent *ent) {
 		++state->prepped;
 	} else {
 		ioq_dispatch_sync(ioq, ent);
-		ioqq_push(ioq->ready, ent);
+		ioq_batch_push(ioq->ready, &state->ready, ent);
 	}
 }
 
 /** Prep a batch of SQEs. */
 static bool ioq_ring_prep(struct ioq_ring_state *state) {
+	if (state->stop) {
+		return false;
+	}
+
+	struct ioq *ioq = state->ioq;
 	struct io_uring *ring = state->ring;
+	struct ioq_ent *pending[IOQ_BATCH];
+
+	while (io_uring_sq_space_left(ring) >= IOQ_BATCH) {
+		bool block = ioq_ring_empty(state);
+		ioqq_pop_batch(ioq->pending, pending, IOQ_BATCH, block);
+
+		bool any = false;
+		for (size_t i = 0; i < IOQ_BATCH; ++i) {
+			struct ioq_ent *ent = pending[i];
+			if (ent == &IOQ_STOP) {
+				ioqq_push(ioq->pending, &IOQ_STOP);
+				state->stop = true;
+				goto done;
+			} else if (ent) {
+				ioq_prep_sqe(state, ent);
+				any = true;
+			}
+		}
 
-	while (io_uring_sq_space_left(ring)) {
-		struct ioq_ent *ent = ioq_ring_pop(state);
-		if (!ent) {
+		if (!any) {
 			break;
 		}
-
-		ioq_prep_sqe(state, ent);
 	}
 
-	return state->prepped || state->submitted;
+done:
+	return !ioq_ring_empty(state);
 }
 
 /** Reap a single CQE. */
@@ -638,11 +701,12 @@ static void ioq_reap_cqe(struct ioq_ring_state *state, struct io_uring_cqe *cqe)
 	}
 
 push:
-	ioqq_push(ioq->ready, ent);
+	ioq_batch_push(ioq->ready, &state->ready, ent);
 }
 
 /** Reap a batch of CQEs. */
 static void ioq_ring_reap(struct ioq_ring_state *state) {
+	struct ioq *ioq = state->ioq;
 	struct io_uring *ring = state->ring;
 
 	while (state->prepped) {
@@ -661,6 +725,8 @@ static void ioq_ring_reap(struct ioq_ring_state *state) {
 
 		ioq_reap_cqe(state, cqe);
 	}
+
+	ioq_batch_flush(ioq->ready, &state->ready);
 }
 
 /** io_uring worker loop. */
@@ -681,17 +747,29 @@ static void ioq_ring_work(struct ioq_thread *thread) {
 static void ioq_sync_work(struct ioq_thread *thread) {
 	struct ioq *ioq = thread->parent;
 
-	while (true) {
-		struct ioq_ent *ent = ioqq_pop(ioq->pending, true);
-		if (ent == &IOQ_STOP) {
-			ioqq_push(ioq->pending, &IOQ_STOP);
-			break;
+	bool stop = false;
+	while (!stop) {
+		struct ioq_ent *pending[IOQ_BATCH];
+		ioqq_pop_batch(ioq->pending, pending, IOQ_BATCH, true);
+
+		struct ioq_batch ready;
+		ready.size = 0;
+
+		for (size_t i = 0; i < IOQ_BATCH; ++i) {
+			struct ioq_ent *ent = pending[i];
+			if (ent == &IOQ_STOP) {
+				ioqq_push(ioq->pending, &IOQ_STOP);
+				stop = true;
+				break;
+			} else if (ent) {
+				if (!ioq_check_cancel(ioq, ent)) {
+					ioq_dispatch_sync(ioq, ent);
+				}
+				ioq_batch_push(ioq->ready, &ready, ent);
+			}
 		}
 
-		if (!ioq_check_cancel(ioq, ent)) {
-			ioq_dispatch_sync(ioq, ent);
-		}
-		ioqq_push(ioq->ready, ent);
+		ioq_batch_flush(ioq->ready, &ready);
 	}
 }
 
-- 
cgit v1.2.3


From c749c11b04444ca40941dd2ddc5802faed148f6a Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Fri, 16 Feb 2024 13:44:04 -0500
Subject: ioq: Ensure ioq_ent is sufficiently aligned

The natural alignment of struct ioq_ent is only 2 on m68k, so over-align
it to at least 4 bytes on all platforms.

Link: https://buildd.debian.org/status/fetch.php?pkg=bfs&arch=m68k&ver=3.1-1&stamp=1707699583
---
 src/ioq.c | 2 +-
 src/ioq.h | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index cf0b927..f23b62f 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -177,7 +177,7 @@ typedef atomic uintptr_t ioq_slot;
 #define IOQ_SKIP_ONE (~IOQ_BLOCKED)
 
 // Need room for two flag bits
-bfs_static_assert(alignof(struct ioq_ent) > 2);
+bfs_static_assert(alignof(struct ioq_ent) >= (1 << 2));
 
 /**
  * An MPMC queue of I/O commands.
diff --git a/src/ioq.h b/src/ioq.h
index 30e90e0..818eea6 100644
--- a/src/ioq.h
+++ b/src/ioq.h
@@ -32,12 +32,19 @@ enum ioq_op {
 	IOQ_STAT,
 };
 
+/**
+ * The I/O queue implementation needs two tag bits in each pointer to a struct
+ * ioq_ent, so we need to ensure at least 4-byte alignment.  The natural
+ * alignment is enough on most architectures, but not m68k, so over-align it.
+ */
+#define IOQ_ENT_ALIGN alignas(4)
+
 /**
  * An I/O queue entry.
  */
 struct ioq_ent {
 	/** The I/O operation. */
-	enum ioq_op op;
+	IOQ_ENT_ALIGN enum ioq_op op;
 
 	/** The return value (on success) or negative error code (on failure). */
 	int result;
-- 
cgit v1.2.3


From b83343fa0dba85dc7e8d357ba7aaeca991315e96 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Sun, 18 Feb 2024 21:45:12 -0500
Subject: ioq: Remove some branches from ioq_slot_{push,pop}()

---
 src/ioq.c | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index f23b62f..74e2587 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -171,8 +171,11 @@ typedef atomic uintptr_t ioq_slot;
 
 /** Someone might be waiting on this slot. */
 #define IOQ_BLOCKED ((uintptr_t)1)
+
+/** Bit for IOQ_SKIP. */
+#define IOQ_SKIP_BIT (UINTPTR_WIDTH - 1)
 /** The next push(es) should skip this slot. */
-#define IOQ_SKIP ((uintptr_t)1 << (UINTPTR_WIDTH - 1))
+#define IOQ_SKIP ((uintptr_t)1 << IOQ_SKIP_BIT)
 /** Amount to add for an additional skip. */
 #define IOQ_SKIP_ONE (~IOQ_BLOCKED)
 
@@ -309,24 +312,30 @@ static void ioq_slot_wake(struct ioqq *ioqq, ioq_slot *slot) {
 	cond_broadcast(&monitor->cond);
 }
 
+/** Branch-free (slot & IOQ_SKIP) ? ~IOQ_BLOCKED : 0 */
+static uintptr_t ioq_skip_mask(uintptr_t slot) {
+	return -(slot >> IOQ_SKIP_BIT) << 1;
+}
+
 /** Push an entry into a slot. */
 static bool ioq_slot_push(struct ioqq *ioqq, ioq_slot *slot, struct ioq_ent *ent) {
 	uintptr_t prev = load(slot, relaxed);
+
 	while (true) {
-		uintptr_t next;
-		if (prev & IOQ_SKIP) {
-			// skip(1) → empty
-			// skip(n) → skip(n - 1)
-			next = (prev - IOQ_SKIP_ONE) & ~IOQ_BLOCKED;
-		} else if (prev > IOQ_BLOCKED) {
+		size_t skip_mask = ioq_skip_mask(prev);
+		size_t full_mask = ~skip_mask & ~IOQ_BLOCKED;
+		if (prev & full_mask) {
 			// full(ptr) → wait
 			prev = ioq_slot_wait(ioqq, slot, prev);
 			continue;
-		} else {
-			// empty → full(ptr)
-			next = (uintptr_t)ent >> 1;
 		}
 
+		// empty   → full(ptr)
+		uintptr_t next = ((uintptr_t)ent >> 1) & full_mask;
+		// skip(1) → empty
+		// skip(n) → skip(n - 1)
+		next |= (prev - IOQ_SKIP_ONE) & skip_mask;
+
 		if (compare_exchange_weak(slot, &prev, next, release, relaxed)) {
 			break;
 		}
@@ -349,7 +358,7 @@ static struct ioq_ent *ioq_slot_pop(struct ioqq *ioqq, ioq_slot *slot, bool bloc
 		uintptr_t next = prev + IOQ_SKIP_ONE;
 		// skip(n)   → ~IOQ_BLOCKED
 		// full(ptr) → 0
-		next &= (next & IOQ_SKIP) ? ~IOQ_BLOCKED : 0;
+		next &= ioq_skip_mask(next);
 
 		if (block && next) {
 			prev = ioq_slot_wait(ioqq, slot, prev);
@@ -368,7 +377,7 @@ static struct ioq_ent *ioq_slot_pop(struct ioqq *ioqq, ioq_slot *slot, bool bloc
 	// empty     → 0
 	// skip(n)   → 0
 	// full(ptr) → ptr
-	prev &= (prev & IOQ_SKIP) ? 0 : ~IOQ_BLOCKED;
+	prev &= ioq_skip_mask(~prev);
 	return (struct ioq_ent *)(prev << 1);
 }
 
-- 
cgit v1.2.3


From 8bc72d6c20c5e38783c4956c4d9fde9b3ee9140c Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 28 Feb 2024 11:30:20 -0500
Subject: ioq: Probe for supported io_uring operations

---
 src/ioq.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 67 insertions(+), 18 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 74e2587..f71ee6e 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -458,6 +458,17 @@ static void ioq_batch_push(struct ioqq *ioqq, struct ioq_batch *batch, struct io
 /** Sentinel stop command. */
 static struct ioq_ent IOQ_STOP;
 
+#if BFS_USE_LIBURING
+/**
+ * Supported io_uring operations.
+ */
+enum ioq_ring_ops {
+	IOQ_RING_OPENAT = 1 << 0,
+	IOQ_RING_CLOSE  = 1 << 1,
+	IOQ_RING_STATX  = 1 << 2,
+};
+#endif
+
 /** I/O queue thread-specific data. */
 struct ioq_thread {
 	/** The thread handle. */
@@ -470,6 +481,8 @@ struct ioq_thread {
 	struct io_uring ring;
 	/** Any error that occurred initializing the ring. */
 	int ring_err;
+	/** Bitmask of supported io_uring operations. */
+	enum ioq_ring_ops ring_ops;
 #endif
 };
 
@@ -553,6 +566,8 @@ struct ioq_ring_state {
 	struct ioq *ioq;
 	/** The io_uring. */
 	struct io_uring *ring;
+	/** Supported io_uring operations. */
+	enum ioq_ring_ops ops;
 	/** Number of prepped, unsubmitted SQEs. */
 	size_t prepped;
 	/** Number of submitted, unreaped SQEs. */
@@ -564,40 +579,48 @@ struct ioq_ring_state {
 };
 
 /** Dispatch a single request asynchronously. */
-static struct io_uring_sqe *ioq_dispatch_async(struct io_uring *ring, struct ioq_ent *ent) {
+static struct io_uring_sqe *ioq_dispatch_async(struct ioq_ring_state *state, struct ioq_ent *ent) {
+	struct io_uring *ring = state->ring;
+	enum ioq_ring_ops ops = state->ops;
 	struct io_uring_sqe *sqe = NULL;
 
 	switch (ent->op) {
-		case IOQ_CLOSE:
+	case IOQ_CLOSE:
+		if (ops & IOQ_RING_CLOSE) {
 			sqe = io_uring_get_sqe(ring);
 			io_uring_prep_close(sqe, ent->close.fd);
-			return sqe;
+		}
+		return sqe;
 
-		case IOQ_OPENDIR: {
+	case IOQ_OPENDIR:
+		if (ops & IOQ_RING_OPENAT) {
 			sqe = io_uring_get_sqe(ring);
 			struct ioq_opendir *args = &ent->opendir;
 			int flags = O_RDONLY | O_CLOEXEC | O_DIRECTORY;
 			io_uring_prep_openat(sqe, args->dfd, args->path, flags, 0);
-			return sqe;
 		}
+		return sqe;
 
-		case IOQ_CLOSEDIR:
+	case IOQ_CLOSEDIR:
 #if BFS_USE_UNWRAPDIR
+		if (ops & IOQ_RING_CLOSE) {
 			sqe = io_uring_get_sqe(ring);
 			io_uring_prep_close(sqe, bfs_unwrapdir(ent->closedir.dir));
+		}
 #endif
-			return sqe;
+		return sqe;
 
-		case IOQ_STAT: {
+	case IOQ_STAT:
 #if BFS_USE_STATX
+		if (ops & IOQ_RING_STATX) {
 			sqe = io_uring_get_sqe(ring);
 			struct ioq_stat *args = &ent->stat;
 			int flags = bfs_statx_flags(args->flags);
 			unsigned int mask = STATX_BASIC_STATS | STATX_BTIME;
 			io_uring_prep_statx(sqe, args->dfd, args->path, flags, mask, args->xbuf);
-#endif
-			return sqe;
 		}
+#endif
+		return sqe;
 	}
 
 	bfs_bug("Unknown ioq_op %d", (int)ent->op);
@@ -617,7 +640,7 @@ static void ioq_prep_sqe(struct ioq_ring_state *state, struct ioq_ent *ent) {
 		return;
 	}
 
-	struct io_uring_sqe *sqe = ioq_dispatch_async(state->ring, ent);
+	struct io_uring_sqe *sqe = ioq_dispatch_async(state, ent);
 	if (sqe) {
 		io_uring_sqe_set_data(sqe, ent);
 		++state->prepped;
@@ -743,6 +766,7 @@ static void ioq_ring_work(struct ioq_thread *thread) {
 	struct ioq_ring_state state = {
 		.ioq = thread->parent,
 		.ring = &thread->ring,
+		.ops = thread->ring_ops,
 	};
 
 	while (ioq_ring_prep(&state)) {
@@ -824,14 +848,39 @@ static int ioq_ring_init(struct ioq *ioq, struct ioq_thread *thread) {
 		return -1;
 	}
 
-	if (!prev) {
-		// Limit the number of io_uring workers
-		unsigned int values[] = {
-			ioq->nthreads, // [IO_WQ_BOUND]
-			0,             // [IO_WQ_UNBOUND]
-		};
-		io_uring_register_iowq_max_workers(&thread->ring, values);
+	if (prev) {
+		// Initial setup already complete
+		return 0;
+	}
+
+	// Check for supported operations
+	struct io_uring_probe *probe = io_uring_get_probe_ring(&thread->ring);
+	if (probe) {
+		if (io_uring_opcode_supported(probe, IORING_OP_OPENAT)) {
+			thread->ring_ops |= IOQ_RING_OPENAT;
+		}
+		if (io_uring_opcode_supported(probe, IORING_OP_CLOSE)) {
+			thread->ring_ops |= IOQ_RING_CLOSE;
+		}
+#if BFS_USE_STATX
+		if (io_uring_opcode_supported(probe, IORING_OP_STATX)) {
+			thread->ring_ops |= IOQ_RING_STATX;
+		}
+#endif
+		io_uring_free_probe(probe);
 	}
+	if (!thread->ring_ops) {
+		io_uring_queue_exit(&thread->ring);
+		thread->ring_err = ENOTSUP;
+		return -1;
+	}
+
+	// Limit the number of io_uring workers
+	unsigned int values[] = {
+		ioq->nthreads, // [IO_WQ_BOUND]
+		0,             // [IO_WQ_UNBOUND]
+	};
+	io_uring_register_iowq_max_workers(&thread->ring, values);
 #endif
 
 	return 0;
-- 
cgit v1.2.3


From f64f76b55400b71e8576ed7e4a377eb5ef9576aa Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 6 Mar 2024 18:42:32 -0500
Subject: ioq: Copy ring_ops from the previous thread

Otherwise threads 2-N won't use io_uring at all!  Oops.

Fixes: 8bc72d6c ("ioq: Probe for supported io_uring operations")
---
 src/ioq.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index f71ee6e..00c3b86 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -850,6 +850,7 @@ static int ioq_ring_init(struct ioq *ioq, struct ioq_thread *thread) {
 
 	if (prev) {
 		// Initial setup already complete
+		thread->ring_ops = prev->ring_ops;
 		return 0;
 	}
 
@@ -916,12 +917,14 @@ static void ioq_thread_join(struct ioq_thread *thread) {
 }
 
 struct ioq *ioq_create(size_t depth, size_t nthreads) {
-	struct ioq *ioq = ZALLOC_FLEX(struct ioq, threads, nthreads);
+	struct ioq *ioq = ALLOC_FLEX(struct ioq, threads, nthreads);
 	if (!ioq) {
 		goto fail;
 	}
 
 	ioq->depth = depth;
+	ioq->size = 0;
+	ioq->cancel = false;
 
 	ARENA_INIT(&ioq->ents, struct ioq_ent);
 
-- 
cgit v1.2.3


From 2c3ef3a06ee1f951f6d68be6d0d3f6a1822b05b7 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Mon, 11 Mar 2024 09:51:03 -0400
Subject: Re-run include-what-you-use

---
 src/alloc.h    | 1 +
 src/bfstd.c    | 6 +++---
 src/bftw.c     | 1 +
 src/color.h    | 1 -
 src/ctx.c      | 1 +
 src/ctx.h      | 2 ++
 src/diag.c     | 2 +-
 src/dir.h      | 4 ++--
 src/dstring.c  | 2 ++
 src/eval.c     | 4 ++--
 src/expr.c     | 4 ++--
 src/expr.h     | 1 -
 src/ioq.c      | 5 +++--
 src/main.c     | 1 +
 src/opt.c      | 3 ++-
 src/parse.c    | 3 +--
 src/printf.c   | 3 ++-
 src/trie.c     | 2 --
 src/trie.h     | 1 -
 src/xspawn.c   | 1 -
 src/xtime.c    | 1 -
 tests/alloc.c  | 1 +
 tests/bfstd.c  | 3 ---
 tests/bit.c    | 2 +-
 tests/ioq.c    | 2 ++
 tests/main.c   | 3 ---
 tests/xtime.c  | 4 ++--
 tests/xtouch.c | 1 +
 28 files changed, 33 insertions(+), 32 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/alloc.h b/src/alloc.h
index 60dd738..ae055bc 100644
--- a/src/alloc.h
+++ b/src/alloc.h
@@ -10,6 +10,7 @@
 
 #include "config.h"
 #include <errno.h>
+#include <stddef.h>
 #include <stdlib.h>
 
 /** Check if a size is properly aligned. */
diff --git a/src/bfstd.c b/src/bfstd.c
index ce4aa49..c6c2e7f 100644
--- a/src/bfstd.c
+++ b/src/bfstd.c
@@ -2,18 +2,19 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "bfstd.h"
-#include "bit.h"
 #include "config.h"
 #include "diag.h"
 #include "sanity.h"
 #include "thread.h"
 #include "xregex.h"
-#include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <langinfo.h>
+#include <limits.h>
 #include <locale.h>
 #include <nl_types.h>
+#include <pthread.h>
+#include <stddef.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -24,7 +25,6 @@
 #include <sys/wait.h>
 #include <unistd.h>
 #include <wchar.h>
-#include <wctype.h>
 
 #if BFS_USE_SYS_SYSMACROS_H
 #  include <sys/sysmacros.h>
diff --git a/src/bftw.c b/src/bftw.c
index 6f52299..50b8b02 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -35,6 +35,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h>
+#include <sys/types.h>
 
 /** Initialize a bftw_stat cache. */
 static void bftw_stat_init(struct bftw_stat *bufs, struct bfs_stat *stat_buf, struct bfs_stat *lstat_buf) {
diff --git a/src/color.h b/src/color.h
index 85633a4..e3e7973 100644
--- a/src/color.h
+++ b/src/color.h
@@ -10,7 +10,6 @@
 
 #include "config.h"
 #include "dstring.h"
-#include <stdarg.h>
 #include <stdio.h>
 
 /**
diff --git a/src/ctx.c b/src/ctx.c
index 6c84f75..f5b28c7 100644
--- a/src/ctx.c
+++ b/src/ctx.c
@@ -6,6 +6,7 @@
 #include "color.h"
 #include "diag.h"
 #include "expr.h"
+#include "list.h"
 #include "mtab.h"
 #include "pwcache.h"
 #include "stat.h"
diff --git a/src/ctx.h b/src/ctx.h
index aa91f2c..e14db21 100644
--- a/src/ctx.h
+++ b/src/ctx.h
@@ -18,6 +18,8 @@
 #include <sys/resource.h>
 #include <time.h>
 
+struct CFILE;
+
 /**
  * The execution context for bfs.
  */
diff --git a/src/diag.c b/src/diag.c
index 656fa89..cb27b92 100644
--- a/src/diag.c
+++ b/src/diag.c
@@ -11,8 +11,8 @@
 #include "expr.h"
 #include <errno.h>
 #include <stdarg.h>
+#include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
 
 /** bfs_diagf() implementation. */
 attr(printf(2, 0))
diff --git a/src/dir.h b/src/dir.h
index b11d454..18d907e 100644
--- a/src/dir.h
+++ b/src/dir.h
@@ -8,8 +8,6 @@
 #ifndef BFS_DIR_H
 #define BFS_DIR_H
 
-#include "alloc.h"
-#include "config.h"
 #include <sys/types.h>
 
 /**
@@ -78,6 +76,8 @@ struct bfs_dirent {
  */
 struct bfs_dir *bfs_allocdir(void);
 
+struct arena;
+
 /**
  * Initialize an arena for directories.
  *
diff --git a/src/dstring.c b/src/dstring.c
index bc18308..10b0fad 100644
--- a/src/dstring.c
+++ b/src/dstring.c
@@ -7,6 +7,8 @@
 #include "config.h"
 #include "diag.h"
 #include <stdarg.h>
+#include <stddef.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
diff --git a/src/eval.c b/src/eval.c
index 1711001..9e55964 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -25,7 +25,6 @@
 #include "stat.h"
 #include "trie.h"
 #include "xregex.h"
-#include "xtime.h"
 #include <errno.h>
 #include <fcntl.h>
 #include <fnmatch.h>
@@ -36,8 +35,9 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <strings.h>
 #include <sys/resource.h>
-#include <sys/stat.h>
+#include <sys/types.h>
 #include <time.h>
 #include <unistd.h>
 #include <wchar.h>
diff --git a/src/expr.c b/src/expr.c
index 3e0033f..5784220 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -4,12 +4,12 @@
 #include "expr.h"
 #include "alloc.h"
 #include "ctx.h"
+#include "diag.h"
 #include "eval.h"
 #include "exec.h"
+#include "list.h"
 #include "printf.h"
 #include "xregex.h"
-#include <stdio.h>
-#include <stdlib.h>
 #include <string.h>
 
 struct bfs_expr *bfs_expr_new(struct bfs_ctx *ctx, bfs_eval_fn *eval_fn, size_t argc, char **argv) {
diff --git a/src/expr.h b/src/expr.h
index 957b04a..75cb5fd 100644
--- a/src/expr.h
+++ b/src/expr.h
@@ -12,7 +12,6 @@
 #include "config.h"
 #include "eval.h"
 #include "stat.h"
-#include <stddef.h>
 #include <sys/types.h>
 #include <time.h>
 
diff --git a/src/ioq.c b/src/ioq.c
index 00c3b86..37eed7d 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -126,13 +126,14 @@
 #include "config.h"
 #include "diag.h"
 #include "dir.h"
-#include "sanity.h"
 #include "stat.h"
 #include "thread.h"
-#include <assert.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <pthread.h>
+#include <stdint.h>
 #include <stdlib.h>
+#include <sys/stat.h>
 
 #if BFS_USE_LIBURING
 #  include <liburing.h>
diff --git a/src/main.c b/src/main.c
index 16a2576..e120f03 100644
--- a/src/main.c
+++ b/src/main.c
@@ -48,6 +48,7 @@
 #include "bfstd.h"
 #include "config.h"
 #include "ctx.h"
+#include "diag.h"
 #include "eval.h"
 #include "parse.h"
 #include <errno.h>
diff --git a/src/opt.c b/src/opt.c
index 74145ac..76965de 100644
--- a/src/opt.c
+++ b/src/opt.c
@@ -26,11 +26,13 @@
  */
 
 #include "opt.h"
+#include "bftw.h"
 #include "bit.h"
 #include "color.h"
 #include "config.h"
 #include "ctx.h"
 #include "diag.h"
+#include "dir.h"
 #include "eval.h"
 #include "exec.h"
 #include "expr.h"
@@ -40,7 +42,6 @@
 #include <limits.h>
 #include <stdarg.h>
 #include <stdio.h>
-#include <string.h>
 #include <unistd.h>
 
 static char *fake_and_arg = "-and";
diff --git a/src/parse.c b/src/parse.c
index b26a50f..3b7386d 100644
--- a/src/parse.c
+++ b/src/parse.c
@@ -42,8 +42,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h>
-#include <sys/time.h>
-#include <sys/wait.h>
+#include <sys/types.h>
 #include <time.h>
 #include <unistd.h>
 
diff --git a/src/printf.c b/src/printf.c
index 34ed606..487f039 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: 0BSD
 
 #include "printf.h"
+#include "alloc.h"
 #include "bfstd.h"
 #include "bftw.h"
 #include "color.h"
@@ -14,10 +15,10 @@
 #include "mtab.h"
 #include "pwcache.h"
 #include "stat.h"
-#include "xtime.h"
 #include <errno.h>
 #include <grp.h>
 #include <pwd.h>
+#include <stdarg.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/src/trie.c b/src/trie.c
index bd5300d..1ffb23a 100644
--- a/src/trie.c
+++ b/src/trie.c
@@ -87,9 +87,7 @@
 #include "config.h"
 #include "diag.h"
 #include "list.h"
-#include <limits.h>
 #include <stdint.h>
-#include <stdlib.h>
 #include <string.h>
 
 bfs_static_assert(CHAR_WIDTH == 8);
diff --git a/src/trie.h b/src/trie.h
index 02088f1..4288d76 100644
--- a/src/trie.h
+++ b/src/trie.h
@@ -5,7 +5,6 @@
 #define BFS_TRIE_H
 
 #include "alloc.h"
-#include "config.h"
 #include "list.h"
 #include <stddef.h>
 #include <stdint.h>
diff --git a/src/xspawn.c b/src/xspawn.c
index 8d6108b..6a94d3d 100644
--- a/src/xspawn.c
+++ b/src/xspawn.c
@@ -12,7 +12,6 @@
 #include <string.h>
 #include <sys/resource.h>
 #include <sys/types.h>
-#include <sys/wait.h>
 #include <unistd.h>
 
 #if BFS_USE_PATHS_H
diff --git a/src/xtime.c b/src/xtime.c
index 05f0e1a..5b259ab 100644
--- a/src/xtime.c
+++ b/src/xtime.c
@@ -7,7 +7,6 @@
 #include "diag.h"
 #include <errno.h>
 #include <limits.h>
-#include <stdlib.h>
 #include <sys/time.h>
 #include <time.h>
 #include <unistd.h>
diff --git a/tests/alloc.c b/tests/alloc.c
index 4ce23d4..9f08111 100644
--- a/tests/alloc.c
+++ b/tests/alloc.c
@@ -3,6 +3,7 @@
 
 #include "tests.h"
 #include "../src/alloc.h"
+#include "../src/config.h"
 #include "../src/diag.h"
 #include <errno.h>
 #include <stdlib.h>
diff --git a/tests/bfstd.c b/tests/bfstd.c
index 0ded5de..26abdb6 100644
--- a/tests/bfstd.c
+++ b/tests/bfstd.c
@@ -7,9 +7,6 @@
 #include "../src/diag.h"
 #include <errno.h>
 #include <langinfo.h>
-#include <locale.h>
-#include <stdint.h>
-#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
diff --git a/tests/bit.c b/tests/bit.c
index b944748..3d66ce3 100644
--- a/tests/bit.c
+++ b/tests/bit.c
@@ -3,10 +3,10 @@
 
 #include "tests.h"
 #include "../src/bit.h"
+#include "../src/config.h"
 #include "../src/diag.h"
 #include <limits.h>
 #include <stdint.h>
-#include <stdlib.h>
 
 bfs_static_assert(UMAX_WIDTH(0x1) == 1);
 bfs_static_assert(UMAX_WIDTH(0x3) == 2);
diff --git a/tests/ioq.c b/tests/ioq.c
index 56e1886..1ce8f75 100644
--- a/tests/ioq.c
+++ b/tests/ioq.c
@@ -4,10 +4,12 @@
 #include "tests.h"
 #include "../src/ioq.h"
 #include "../src/bfstd.h"
+#include "../src/config.h"
 #include "../src/diag.h"
 #include "../src/dir.h"
 #include <errno.h>
 #include <fcntl.h>
+#include <stdlib.h>
 
 /**
  * Test for blocking within ioq_slot_push().
diff --git a/tests/main.c b/tests/main.c
index 38438b2..8849e8c 100644
--- a/tests/main.c
+++ b/tests/main.c
@@ -6,11 +6,8 @@
  */
 
 #include "tests.h"
-#include "../src/bfstd.h"
 #include "../src/color.h"
 #include "../src/config.h"
-#include "../src/diag.h"
-#include <errno.h>
 #include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/tests/xtime.c b/tests/xtime.c
index 3f1fec2..c8dc00b 100644
--- a/tests/xtime.c
+++ b/tests/xtime.c
@@ -5,10 +5,10 @@
 #include "../src/xtime.h"
 #include "../src/bfstd.h"
 #include "../src/config.h"
+#include "../src/diag.h"
 #include <errno.h>
+#include <limits.h>
 #include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
 #include <time.h>
 
 static bool tm_equal(const struct tm *tma, const struct tm *tmb) {
diff --git a/tests/xtouch.c b/tests/xtouch.c
index 8c5c5f3..fad272f 100644
--- a/tests/xtouch.c
+++ b/tests/xtouch.c
@@ -11,6 +11,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h>
+#include <sys/types.h>
 #include <time.h>
 #include <unistd.h>
 
-- 
cgit v1.2.3


From 745fd4be765407f7c56d61c281c28e2468ba25b5 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 20 Mar 2024 16:41:24 -0400
Subject: ioq: Fix some allocation failure paths

---
 src/ioq.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 37eed7d..b936681 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -1064,7 +1064,7 @@ void ioq_free(struct ioq *ioq, struct ioq_ent *ent) {
 	--ioq->size;
 
 #if BFS_USE_LIBURING && BFS_USE_STATX
-	if (ent->op == IOQ_STAT) {
+	if (ent->op == IOQ_STAT && ent->stat.xbuf) {
 		arena_free(&ioq->xbufs, ent->stat.xbuf);
 	}
 #endif
@@ -1083,7 +1083,9 @@ void ioq_destroy(struct ioq *ioq) {
 		return;
 	}
 
-	ioq_cancel(ioq);
+	if (ioq->nthreads > 0) {
+		ioq_cancel(ioq);
+	}
 
 	for (size_t i = 0; i < ioq->nthreads; ++i) {
 		ioq_thread_join(&ioq->threads[i]);
-- 
cgit v1.2.3


From c66379749f423413913b406609dfe9311ba6e555 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Thu, 18 Apr 2024 14:53:56 -0400
Subject: Rename config.h to prelude.h

---
 src/alloc.c    |   2 +-
 src/alloc.h    |   2 +-
 src/bar.c      |   2 +-
 src/bfstd.c    |   2 +-
 src/bfstd.h    |   2 +-
 src/bftw.c     |   2 +-
 src/bit.h      |   2 +-
 src/color.c    |   2 +-
 src/color.h    |   2 +-
 src/config.h   | 377 ---------------------------------------------------------
 src/ctx.h      |   2 +-
 src/diag.c     |   2 +-
 src/diag.h     |   2 +-
 src/dir.c      |   2 +-
 src/dstring.c  |   2 +-
 src/dstring.h  |   2 +-
 src/eval.c     |   2 +-
 src/eval.h     |   2 +-
 src/exec.c     |   2 +-
 src/expr.h     |   2 +-
 src/fsade.c    |   2 +-
 src/fsade.h    |   2 +-
 src/ioq.c      |   2 +-
 src/ioq.h      |   2 +-
 src/main.c     |   4 +-
 src/mtab.c     |   2 +-
 src/mtab.h     |   2 +-
 src/opt.c      |   2 +-
 src/parse.c    |   2 +-
 src/prelude.h  | 377 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/printf.c   |   2 +-
 src/pwcache.c  |   2 +-
 src/sanity.h   |   2 +-
 src/stat.c     |   2 +-
 src/stat.h     |   2 +-
 src/thread.c   |   2 +-
 src/thread.h   |   2 +-
 src/trie.c     |   2 +-
 src/xregex.c   |   2 +-
 src/xspawn.c   |   2 +-
 src/xspawn.h   |   2 +-
 src/xtime.c    |   2 +-
 tests/alloc.c  |   2 +-
 tests/bfstd.c  |   2 +-
 tests/bit.c    |   2 +-
 tests/ioq.c    |   2 +-
 tests/main.c   |   2 +-
 tests/tests.h  |   2 +-
 tests/trie.c   |   2 +-
 tests/xspawn.c |   2 +-
 tests/xtime.c  |   2 +-
 tests/xtouch.c |   2 +-
 52 files changed, 428 insertions(+), 428 deletions(-)
 delete mode 100644 src/config.h
 create mode 100644 src/prelude.h

(limited to 'src/ioq.c')

diff --git a/src/alloc.c b/src/alloc.c
index b65d0c5..ec8608f 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -1,9 +1,9 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "alloc.h"
 #include "bit.h"
-#include "config.h"
 #include "diag.h"
 #include "sanity.h"
 #include <errno.h>
diff --git a/src/alloc.h b/src/alloc.h
index ae055bc..095134a 100644
--- a/src/alloc.h
+++ b/src/alloc.h
@@ -8,7 +8,7 @@
 #ifndef BFS_ALLOC_H
 #define BFS_ALLOC_H
 
-#include "config.h"
+#include "prelude.h"
 #include <errno.h>
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/src/bar.c b/src/bar.c
index 8ab4112..184d9a0 100644
--- a/src/bar.c
+++ b/src/bar.c
@@ -1,11 +1,11 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "bar.h"
 #include "atomic.h"
 #include "bfstd.h"
 #include "bit.h"
-#include "config.h"
 #include "dstring.h"
 #include <errno.h>
 #include <fcntl.h>
diff --git a/src/bfstd.c b/src/bfstd.c
index 2499f00..e1b4804 100644
--- a/src/bfstd.c
+++ b/src/bfstd.c
@@ -1,9 +1,9 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "bfstd.h"
 #include "bit.h"
-#include "config.h"
 #include "diag.h"
 #include "sanity.h"
 #include "thread.h"
diff --git a/src/bfstd.h b/src/bfstd.h
index fc22971..42f5d5b 100644
--- a/src/bfstd.h
+++ b/src/bfstd.h
@@ -8,7 +8,7 @@
 #ifndef BFS_BFSTD_H
 #define BFS_BFSTD_H
 
-#include "config.h"
+#include "prelude.h"
 #include "sanity.h"
 #include <stddef.h>
 
diff --git a/src/bftw.c b/src/bftw.c
index 6130c44..c4d3c17 100644
--- a/src/bftw.c
+++ b/src/bftw.c
@@ -18,10 +18,10 @@
  *   various helper functions to take fewer parameters.
  */
 
+#include "prelude.h"
 #include "bftw.h"
 #include "alloc.h"
 #include "bfstd.h"
-#include "config.h"
 #include "diag.h"
 #include "dir.h"
 #include "dstring.h"
diff --git a/src/bit.h b/src/bit.h
index 69df21e..17cfbcf 100644
--- a/src/bit.h
+++ b/src/bit.h
@@ -8,7 +8,7 @@
 #ifndef BFS_BIT_H
 #define BFS_BIT_H
 
-#include "config.h"
+#include "prelude.h"
 #include <limits.h>
 #include <stdint.h>
 
diff --git a/src/color.c b/src/color.c
index 8c32a68..f004bf2 100644
--- a/src/color.c
+++ b/src/color.c
@@ -1,11 +1,11 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "color.h"
 #include "alloc.h"
 #include "bfstd.h"
 #include "bftw.h"
-#include "config.h"
 #include "diag.h"
 #include "dir.h"
 #include "dstring.h"
diff --git a/src/color.h b/src/color.h
index e3e7973..3278cd6 100644
--- a/src/color.h
+++ b/src/color.h
@@ -8,7 +8,7 @@
 #ifndef BFS_COLOR_H
 #define BFS_COLOR_H
 
-#include "config.h"
+#include "prelude.h"
 #include "dstring.h"
 #include <stdio.h>
 
diff --git a/src/config.h b/src/config.h
deleted file mode 100644
index 2eff1fc..0000000
--- a/src/config.h
+++ /dev/null
@@ -1,377 +0,0 @@
-// Copyright © Tavian Barnes <tavianator@tavianator.com>
-// SPDX-License-Identifier: 0BSD
-
-/**
- * Configuration and feature/platform detection.
- */
-
-#ifndef BFS_CONFIG_H
-#define BFS_CONFIG_H
-
-// Possible __STDC_VERSION__ values
-
-#define C95 199409L
-#define C99 199901L
-#define C11 201112L
-#define C17 201710L
-#define C23 202311L
-
-#include <stddef.h>
-
-#if __STDC_VERSION__ < C23
-#  include <stdalign.h>
-#  include <stdbool.h>
-#  include <stdnoreturn.h>
-#endif
-
-// bfs packaging configuration
-
-#ifndef BFS_COMMAND
-#  define BFS_COMMAND "bfs"
-#endif
-#ifndef BFS_HOMEPAGE
-#  define BFS_HOMEPAGE "https://tavianator.com/projects/bfs.html"
-#endif
-
-// This is a symbol instead of a literal so we don't have to rebuild everything
-// when the version number changes
-extern const char bfs_version[];
-
-// Check for system headers
-
-#ifdef __has_include
-
-#if __has_include(<mntent.h>)
-#  define BFS_HAS_MNTENT_H true
-#endif
-#if __has_include(<paths.h>)
-#  define BFS_HAS_PATHS_H true
-#endif
-#if __has_include(<sys/acl.h>)
-#  define BFS_HAS_SYS_ACL_H true
-#endif
-#if __has_include(<sys/capability.h>)
-#  define BFS_HAS_SYS_CAPABILITY_H true
-#endif
-#if __has_include(<sys/extattr.h>)
-#  define BFS_HAS_SYS_EXTATTR_H true
-#endif
-#if __has_include(<sys/mkdev.h>)
-#  define BFS_HAS_SYS_MKDEV_H true
-#endif
-#if __has_include(<sys/param.h>)
-#  define BFS_HAS_SYS_PARAM_H true
-#endif
-#if __has_include(<sys/sysmacros.h>)
-#  define BFS_HAS_SYS_SYSMACROS_H true
-#endif
-#if __has_include(<sys/xattr.h>)
-#  define BFS_HAS_SYS_XATTR_H true
-#endif
-#if __has_include(<threads.h>)
-#  define BFS_HAS_THREADS_H true
-#endif
-#if __has_include(<util.h>)
-#  define BFS_HAS_UTIL_H true
-#endif
-
-#else // !__has_include
-
-#define BFS_HAS_MNTENT_H __GLIBC__
-#define BFS_HAS_PATHS_H true
-#define BFS_HAS_SYS_ACL_H true
-#define BFS_HAS_SYS_CAPABILITY_H __linux__
-#define BFS_HAS_SYS_EXTATTR_H __FreeBSD__
-#define BFS_HAS_SYS_MKDEV_H false
-#define BFS_HAS_SYS_PARAM_H true
-#define BFS_HAS_SYS_SYSMACROS_H __GLIBC__
-#define BFS_HAS_SYS_XATTR_H __linux__
-#define BFS_HAS_THREADS_H (!__STDC_NO_THREADS__)
-#define BFS_HAS_UTIL_H __NetBSD__
-
-#endif // !__has_include
-
-#ifndef BFS_USE_MNTENT_H
-#  define BFS_USE_MNTENT_H BFS_HAS_MNTENT_H
-#endif
-#ifndef BFS_USE_PATHS_H
-#  define BFS_USE_PATHS_H BFS_HAS_PATHS_H
-#endif
-#ifndef BFS_USE_SYS_ACL_H
-#  define BFS_USE_SYS_ACL_H (BFS_HAS_SYS_ACL_H && !__illumos__ && (!__linux__ || BFS_USE_LIBACL))
-#endif
-#ifndef BFS_USE_SYS_CAPABILITY_H
-#  define BFS_USE_SYS_CAPABILITY_H (BFS_HAS_SYS_CAPABILITY_H && !__FreeBSD__ && (!__linux__ || BFS_USE_LIBCAP))
-#endif
-#ifndef BFS_USE_SYS_EXTATTR_H
-#  define BFS_USE_SYS_EXTATTR_H (BFS_HAS_SYS_EXTATTR_H && !__DragonFly__)
-#endif
-#ifndef BFS_USE_SYS_MKDEV_H
-#  define BFS_USE_SYS_MKDEV_H BFS_HAS_SYS_MKDEV_H
-#endif
-#ifndef BFS_USE_SYS_PARAM_H
-#  define BFS_USE_SYS_PARAM_H BFS_HAS_SYS_PARAM_H
-#endif
-#ifndef BFS_USE_SYS_SYSMACROS_H
-#  define BFS_USE_SYS_SYSMACROS_H BFS_HAS_SYS_SYSMACROS_H
-#endif
-#ifndef BFS_USE_SYS_XATTR_H
-#  define BFS_USE_SYS_XATTR_H BFS_HAS_SYS_XATTR_H
-#endif
-#ifndef BFS_USE_THREADS_H
-#  define BFS_USE_THREADS_H BFS_HAS_THREADS_H
-#endif
-#ifndef BFS_USE_UTIL_H
-#  define BFS_USE_UTIL_H BFS_HAS_UTIL_H
-#endif
-
-// Stub out feature detection on old/incompatible compilers
-
-#ifndef __has_feature
-#  define __has_feature(feat) false
-#endif
-
-#ifndef __has_c_attribute
-#  define __has_c_attribute(attr) false
-#endif
-
-#ifndef __has_attribute
-#  define __has_attribute(attr) false
-#endif
-
-// Platform detection
-
-// Get the definition of BSD if available
-#if BFS_USE_SYS_PARAM_H
-#  include <sys/param.h>
-#endif
-
-#ifndef __GLIBC_PREREQ
-#  define __GLIBC_PREREQ(maj, min) false
-#endif
-
-#ifndef __NetBSD_Prereq__
-#  define __NetBSD_Prereq__(maj, min, patch) false
-#endif
-
-// Fundamental utilities
-
-/**
- * Get the length of an array.
- */
-#define countof(array) (sizeof(array) / sizeof(0[array]))
-
-/**
- * False sharing/destructive interference/largest cache line size.
- */
-#ifdef __GCC_DESTRUCTIVE_SIZE
-#  define FALSE_SHARING_SIZE __GCC_DESTRUCTIVE_SIZE
-#else
-#  define FALSE_SHARING_SIZE 64
-#endif
-
-/**
- * True sharing/constructive interference/smallest cache line size.
- */
-#ifdef __GCC_CONSTRUCTIVE_SIZE
-#  define TRUE_SHARING_SIZE __GCC_CONSTRUCTIVE_SIZE
-#else
-#  define TRUE_SHARING_SIZE 64
-#endif
-
-/**
- * Alignment specifier that avoids false sharing.
- */
-#define cache_align alignas(FALSE_SHARING_SIZE)
-
-#if __COSMOPOLITAN__
-typedef long double max_align_t;
-#endif
-
-// Wrappers for attributes
-
-/**
- * Silence warnings about switch/case fall-throughs.
- */
-#if __has_attribute(fallthrough)
-#  define fallthru __attribute__((fallthrough))
-#else
-#  define fallthru ((void)0)
-#endif
-
-/**
- * Silence warnings about unused declarations.
- */
-#if __has_attribute(unused)
-#  define attr_maybe_unused __attribute__((unused))
-#else
-#  define attr_maybe_unused
-#endif
-
-/**
- * Warn if a value is unused.
- */
-#if __has_attribute(warn_unused_result)
-#  define attr_nodiscard __attribute__((warn_unused_result))
-#else
-#  define attr_nodiscard
-#endif
-
-/**
- * Hint to avoid inlining a function.
- */
-#if __has_attribute(noinline)
-#  define attr_noinline __attribute__((noinline))
-#else
-#  define attr_noinline
-#endif
-
-/**
- * Hint that a function is unlikely to be called.
- */
-#if __has_attribute(cold)
-#  define attr_cold attr_noinline __attribute__((cold))
-#else
-#  define attr_cold attr_noinline
-#endif
-
-/**
- * Adds compiler warnings for bad printf()-style function calls, if supported.
- */
-#if __has_attribute(format)
-#  define attr_printf(fmt, args) __attribute__((format(printf, fmt, args)))
-#else
-#  define attr_printf(fmt, args)
-#endif
-
-/**
- * Annotates allocator-like functions.
- */
-#if __has_attribute(malloc)
-#  if __GNUC__ >= 11 && !__OPTIMIZE__ // malloc(deallocator) disables inlining on GCC
-#    define attr_malloc(...) attr_nodiscard __attribute__((malloc(__VA_ARGS__)))
-#  else
-#    define attr_malloc(...) attr_nodiscard __attribute__((malloc))
-#  endif
-#else
-#  define attr_malloc(...) attr_nodiscard
-#endif
-
-/**
- * Specifies that a function returns allocations with a given alignment.
- */
-#if __has_attribute(alloc_align)
-#  define attr_alloc_align(param) __attribute__((alloc_align(param)))
-#else
-#  define attr_alloc_align(param)
-#endif
-
-/**
- * Specifies that a function returns allocations with a given size.
- */
-#if __has_attribute(alloc_size)
-#  define attr_alloc_size(...) __attribute__((alloc_size(__VA_ARGS__)))
-#else
-#  define attr_alloc_size(...)
-#endif
-
-/**
- * Shorthand for attr_alloc_align() and attr_alloc_size().
- */
-#define attr_aligned_alloc(align, ...) \
-	attr_alloc_align(align) \
-	attr_alloc_size(__VA_ARGS__)
-
-/**
- * Check if function multiversioning via GNU indirect functions (ifunc) is supported.
- */
-#ifndef BFS_USE_TARGET_CLONES
-#  if __has_attribute(target_clones) && (__GLIBC__ || __FreeBSD__)
-#    define BFS_USE_TARGET_CLONES true
-#  endif
-#endif
-
-/**
- * Apply the target_clones attribute, if available.
- */
-#if BFS_USE_TARGET_CLONES
-#  define attr_target_clones(...) __attribute__((target_clones(__VA_ARGS__)))
-#else
-#  define attr_target_clones(...)
-#endif
-
-/**
- * Shorthand for multiple attributes at once. attr(a, b(c), d) is equivalent to
- *
- *     attr_a
- *     attr_b(c)
- *     attr_d
- */
-#define attr(...) \
-	attr__(attr_##__VA_ARGS__, none, none, none, none, none, none, none, none, none, )
-
-/**
- * attr() helper.  For exposition, pretend we support only 2 args, instead of 9.
- * There are a few cases:
- *
- *     attr()
- *         => attr__(attr_, none, none)
- *         => attr_                =>
- *            attr_none            =>
- *            attr_too_many_none() =>
- *
- *     attr(a)
- *         => attr__(attr_a, none, none)
- *         => attr_a               => __attribute__((a))
- *            attr_none            =>
- *            attr_too_many_none() =>
- *
- *     attr(a, b(c))
- *         => attr__(attr_a, b(c), none, none)
- *         => attr_a                   => __attribute__((a))
- *            attr_b(c)                => __attribute__((b(c)))
- *            attr_too_many_none(none) =>
- *
- *     attr(a, b(c), d)
- *         => attr__(attr_a, b(c), d, none, none)
- *         => attr_a                      => __attribute__((a))
- *            attr_b(c)                   => __attribute__((b(c)))
- *            attr_too_many_d(none, none) => error
- *
- * Some attribute names are the same as standard library functions, e.g. printf.
- * Standard libraries are permitted to define these functions as macros, like
- *
- *     #define printf(...) __builtin_printf(__VA_ARGS__)
- *
- * The token paste in
- *
- *     #define attr(...) attr__(attr_##__VA_ARGS__, none, none)
- *
- * is necessary to prevent macro expansion before evaluating attr__().
- * Otherwise, we could get
- *
- *     attr(printf(1, 2))
- *         => attr__(__builtin_printf(1, 2), none, none)
- *         => attr____builtin_printf(1, 2)
- *         => error
- */
-#define attr__(a1, a2, a3, a4, a5, a6, a7, a8, a9, none, ...) \
-	a1 \
-	attr_##a2 \
-	attr_##a3 \
-	attr_##a4 \
-	attr_##a5 \
-	attr_##a6 \
-	attr_##a7 \
-	attr_##a8 \
-	attr_##a9 \
-	attr_too_many_##none(__VA_ARGS__)
-
-// Ignore `attr_none` from expanding 1-9 argument attr(a1, a2, ...)
-#define attr_none
-// Ignore `attr_` from expanding 0-argument attr()
-#define attr_
-// Only trigger an error on more than 9 arguments
-#define attr_too_many_none(...)
-
-#endif // BFS_CONFIG_H
diff --git a/src/ctx.h b/src/ctx.h
index e14db21..fc3020c 100644
--- a/src/ctx.h
+++ b/src/ctx.h
@@ -8,9 +8,9 @@
 #ifndef BFS_CTX_H
 #define BFS_CTX_H
 
+#include "prelude.h"
 #include "alloc.h"
 #include "bftw.h"
-#include "config.h"
 #include "diag.h"
 #include "expr.h"
 #include "trie.h"
diff --git a/src/diag.c b/src/diag.c
index cb27b92..deb6f26 100644
--- a/src/diag.c
+++ b/src/diag.c
@@ -1,11 +1,11 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "diag.h"
 #include "alloc.h"
 #include "bfstd.h"
 #include "color.h"
-#include "config.h"
 #include "ctx.h"
 #include "dstring.h"
 #include "expr.h"
diff --git a/src/diag.h b/src/diag.h
index 4054c48..2b13609 100644
--- a/src/diag.h
+++ b/src/diag.h
@@ -8,7 +8,7 @@
 #ifndef BFS_DIAG_H
 #define BFS_DIAG_H
 
-#include "config.h"
+#include "prelude.h"
 #include <stdarg.h>
 
 /**
diff --git a/src/dir.c b/src/dir.c
index 98518f2..53c9be3 100644
--- a/src/dir.c
+++ b/src/dir.c
@@ -1,10 +1,10 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "dir.h"
 #include "alloc.h"
 #include "bfstd.h"
-#include "config.h"
 #include "diag.h"
 #include "sanity.h"
 #include "trie.h"
diff --git a/src/dstring.c b/src/dstring.c
index 10b0fad..913dda8 100644
--- a/src/dstring.c
+++ b/src/dstring.c
@@ -1,10 +1,10 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "dstring.h"
 #include "alloc.h"
 #include "bit.h"
-#include "config.h"
 #include "diag.h"
 #include <stdarg.h>
 #include <stddef.h>
diff --git a/src/dstring.h b/src/dstring.h
index 6006199..9ea7eb9 100644
--- a/src/dstring.h
+++ b/src/dstring.h
@@ -8,8 +8,8 @@
 #ifndef BFS_DSTRING_H
 #define BFS_DSTRING_H
 
+#include "prelude.h"
 #include "bfstd.h"
-#include "config.h"
 #include <stdarg.h>
 #include <stddef.h>
 
diff --git a/src/eval.c b/src/eval.c
index d0112c2..b103912 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -5,12 +5,12 @@
  * Implementation of all the primary expressions.
  */
 
+#include "prelude.h"
 #include "eval.h"
 #include "bar.h"
 #include "bfstd.h"
 #include "bftw.h"
 #include "color.h"
-#include "config.h"
 #include "ctx.h"
 #include "diag.h"
 #include "dir.h"
diff --git a/src/eval.h b/src/eval.h
index ae43628..4dd7996 100644
--- a/src/eval.h
+++ b/src/eval.h
@@ -9,7 +9,7 @@
 #ifndef BFS_EVAL_H
 #define BFS_EVAL_H
 
-#include "config.h"
+#include "prelude.h"
 
 struct bfs_ctx;
 struct bfs_expr;
diff --git a/src/exec.c b/src/exec.c
index 60bfd28..e782d49 100644
--- a/src/exec.c
+++ b/src/exec.c
@@ -1,12 +1,12 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "exec.h"
 #include "alloc.h"
 #include "bfstd.h"
 #include "bftw.h"
 #include "color.h"
-#include "config.h"
 #include "ctx.h"
 #include "diag.h"
 #include "dstring.h"
diff --git a/src/expr.h b/src/expr.h
index 75cb5fd..7bcace7 100644
--- a/src/expr.h
+++ b/src/expr.h
@@ -8,8 +8,8 @@
 #ifndef BFS_EXPR_H
 #define BFS_EXPR_H
 
+#include "prelude.h"
 #include "color.h"
-#include "config.h"
 #include "eval.h"
 #include "stat.h"
 #include <sys/types.h>
diff --git a/src/fsade.c b/src/fsade.c
index 0810c7f..34a4d57 100644
--- a/src/fsade.c
+++ b/src/fsade.c
@@ -1,11 +1,11 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "fsade.h"
 #include "atomic.h"
 #include "bfstd.h"
 #include "bftw.h"
-#include "config.h"
 #include "dir.h"
 #include "dstring.h"
 #include "sanity.h"
diff --git a/src/fsade.h b/src/fsade.h
index 1f1dbfc..6300852 100644
--- a/src/fsade.h
+++ b/src/fsade.h
@@ -9,7 +9,7 @@
 #ifndef BFS_FSADE_H
 #define BFS_FSADE_H
 
-#include "config.h"
+#include "prelude.h"
 
 #define BFS_CAN_CHECK_ACL BFS_USE_SYS_ACL_H
 
diff --git a/src/ioq.c b/src/ioq.c
index b936681..189bdac 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -118,12 +118,12 @@
  * [1]: https://arxiv.org/abs/2201.02179
  */
 
+#include "prelude.h"
 #include "ioq.h"
 #include "alloc.h"
 #include "atomic.h"
 #include "bfstd.h"
 #include "bit.h"
-#include "config.h"
 #include "diag.h"
 #include "dir.h"
 #include "stat.h"
diff --git a/src/ioq.h b/src/ioq.h
index 818eea6..d8e1573 100644
--- a/src/ioq.h
+++ b/src/ioq.h
@@ -8,7 +8,7 @@
 #ifndef BFS_IOQ_H
 #define BFS_IOQ_H
 
-#include "config.h"
+#include "prelude.h"
 #include "dir.h"
 #include "stat.h"
 #include <stddef.h>
diff --git a/src/main.c b/src/main.c
index e120f03..9d8b206 100644
--- a/src/main.c
+++ b/src/main.c
@@ -26,7 +26,7 @@
  *     - bit.h         (bit manipulation)
  *     - bfstd.[ch]    (standard library wrappers/polyfills)
  *     - color.[ch]    (for pretty terminal colors)
- *     - config.h      (configuration and feature/platform detection)
+ *     - prelude.h     (configuration and feature/platform detection)
  *     - diag.[ch]     (formats diagnostic messages)
  *     - dir.[ch]      (a directory API facade)
  *     - dstring.[ch]  (a dynamic string library)
@@ -45,8 +45,8 @@
  *     - xtime.[ch]    (date/time handling utilities)
  */
 
+#include "prelude.h"
 #include "bfstd.h"
-#include "config.h"
 #include "ctx.h"
 #include "diag.h"
 #include "eval.h"
diff --git a/src/mtab.c b/src/mtab.c
index 86ae151..7905d14 100644
--- a/src/mtab.c
+++ b/src/mtab.c
@@ -1,10 +1,10 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "mtab.h"
 #include "alloc.h"
 #include "bfstd.h"
-#include "config.h"
 #include "stat.h"
 #include "trie.h"
 #include <errno.h>
diff --git a/src/mtab.h b/src/mtab.h
index d99d78f..67290c2 100644
--- a/src/mtab.h
+++ b/src/mtab.h
@@ -8,7 +8,7 @@
 #ifndef BFS_MTAB_H
 #define BFS_MTAB_H
 
-#include "config.h"
+#include "prelude.h"
 
 struct bfs_stat;
 
diff --git a/src/opt.c b/src/opt.c
index b74b4e1..ffc795b 100644
--- a/src/opt.c
+++ b/src/opt.c
@@ -25,11 +25,11 @@
  * effects are reachable at all, skipping the traversal if not.
  */
 
+#include "prelude.h"
 #include "opt.h"
 #include "bftw.h"
 #include "bit.h"
 #include "color.h"
-#include "config.h"
 #include "ctx.h"
 #include "diag.h"
 #include "dir.h"
diff --git a/src/parse.c b/src/parse.c
index a3e32fe..c2ae58f 100644
--- a/src/parse.c
+++ b/src/parse.c
@@ -8,12 +8,12 @@
  * flags like always-true options, and skipping over paths wherever they appear.
  */
 
+#include "prelude.h"
 #include "parse.h"
 #include "alloc.h"
 #include "bfstd.h"
 #include "bftw.h"
 #include "color.h"
-#include "config.h"
 #include "ctx.h"
 #include "diag.h"
 #include "dir.h"
diff --git a/src/prelude.h b/src/prelude.h
new file mode 100644
index 0000000..c3a0752
--- /dev/null
+++ b/src/prelude.h
@@ -0,0 +1,377 @@
+// Copyright © Tavian Barnes <tavianator@tavianator.com>
+// SPDX-License-Identifier: 0BSD
+
+/**
+ * Configuration and feature/platform detection.
+ */
+
+#ifndef BFS_PRELUDE_H
+#define BFS_PRELUDE_H
+
+// Possible __STDC_VERSION__ values
+
+#define C95 199409L
+#define C99 199901L
+#define C11 201112L
+#define C17 201710L
+#define C23 202311L
+
+#include <stddef.h>
+
+#if __STDC_VERSION__ < C23
+#  include <stdalign.h>
+#  include <stdbool.h>
+#  include <stdnoreturn.h>
+#endif
+
+// bfs packaging configuration
+
+#ifndef BFS_COMMAND
+#  define BFS_COMMAND "bfs"
+#endif
+#ifndef BFS_HOMEPAGE
+#  define BFS_HOMEPAGE "https://tavianator.com/projects/bfs.html"
+#endif
+
+// This is a symbol instead of a literal so we don't have to rebuild everything
+// when the version number changes
+extern const char bfs_version[];
+
+// Check for system headers
+
+#ifdef __has_include
+
+#if __has_include(<mntent.h>)
+#  define BFS_HAS_MNTENT_H true
+#endif
+#if __has_include(<paths.h>)
+#  define BFS_HAS_PATHS_H true
+#endif
+#if __has_include(<sys/acl.h>)
+#  define BFS_HAS_SYS_ACL_H true
+#endif
+#if __has_include(<sys/capability.h>)
+#  define BFS_HAS_SYS_CAPABILITY_H true
+#endif
+#if __has_include(<sys/extattr.h>)
+#  define BFS_HAS_SYS_EXTATTR_H true
+#endif
+#if __has_include(<sys/mkdev.h>)
+#  define BFS_HAS_SYS_MKDEV_H true
+#endif
+#if __has_include(<sys/param.h>)
+#  define BFS_HAS_SYS_PARAM_H true
+#endif
+#if __has_include(<sys/sysmacros.h>)
+#  define BFS_HAS_SYS_SYSMACROS_H true
+#endif
+#if __has_include(<sys/xattr.h>)
+#  define BFS_HAS_SYS_XATTR_H true
+#endif
+#if __has_include(<threads.h>)
+#  define BFS_HAS_THREADS_H true
+#endif
+#if __has_include(<util.h>)
+#  define BFS_HAS_UTIL_H true
+#endif
+
+#else // !__has_include
+
+#define BFS_HAS_MNTENT_H __GLIBC__
+#define BFS_HAS_PATHS_H true
+#define BFS_HAS_SYS_ACL_H true
+#define BFS_HAS_SYS_CAPABILITY_H __linux__
+#define BFS_HAS_SYS_EXTATTR_H __FreeBSD__
+#define BFS_HAS_SYS_MKDEV_H false
+#define BFS_HAS_SYS_PARAM_H true
+#define BFS_HAS_SYS_SYSMACROS_H __GLIBC__
+#define BFS_HAS_SYS_XATTR_H __linux__
+#define BFS_HAS_THREADS_H (!__STDC_NO_THREADS__)
+#define BFS_HAS_UTIL_H __NetBSD__
+
+#endif // !__has_include
+
+#ifndef BFS_USE_MNTENT_H
+#  define BFS_USE_MNTENT_H BFS_HAS_MNTENT_H
+#endif
+#ifndef BFS_USE_PATHS_H
+#  define BFS_USE_PATHS_H BFS_HAS_PATHS_H
+#endif
+#ifndef BFS_USE_SYS_ACL_H
+#  define BFS_USE_SYS_ACL_H (BFS_HAS_SYS_ACL_H && !__illumos__ && (!__linux__ || BFS_USE_LIBACL))
+#endif
+#ifndef BFS_USE_SYS_CAPABILITY_H
+#  define BFS_USE_SYS_CAPABILITY_H (BFS_HAS_SYS_CAPABILITY_H && !__FreeBSD__ && (!__linux__ || BFS_USE_LIBCAP))
+#endif
+#ifndef BFS_USE_SYS_EXTATTR_H
+#  define BFS_USE_SYS_EXTATTR_H (BFS_HAS_SYS_EXTATTR_H && !__DragonFly__)
+#endif
+#ifndef BFS_USE_SYS_MKDEV_H
+#  define BFS_USE_SYS_MKDEV_H BFS_HAS_SYS_MKDEV_H
+#endif
+#ifndef BFS_USE_SYS_PARAM_H
+#  define BFS_USE_SYS_PARAM_H BFS_HAS_SYS_PARAM_H
+#endif
+#ifndef BFS_USE_SYS_SYSMACROS_H
+#  define BFS_USE_SYS_SYSMACROS_H BFS_HAS_SYS_SYSMACROS_H
+#endif
+#ifndef BFS_USE_SYS_XATTR_H
+#  define BFS_USE_SYS_XATTR_H BFS_HAS_SYS_XATTR_H
+#endif
+#ifndef BFS_USE_THREADS_H
+#  define BFS_USE_THREADS_H BFS_HAS_THREADS_H
+#endif
+#ifndef BFS_USE_UTIL_H
+#  define BFS_USE_UTIL_H BFS_HAS_UTIL_H
+#endif
+
+// Stub out feature detection on old/incompatible compilers
+
+#ifndef __has_feature
+#  define __has_feature(feat) false
+#endif
+
+#ifndef __has_c_attribute
+#  define __has_c_attribute(attr) false
+#endif
+
+#ifndef __has_attribute
+#  define __has_attribute(attr) false
+#endif
+
+// Platform detection
+
+// Get the definition of BSD if available
+#if BFS_USE_SYS_PARAM_H
+#  include <sys/param.h>
+#endif
+
+#ifndef __GLIBC_PREREQ
+#  define __GLIBC_PREREQ(maj, min) false
+#endif
+
+#ifndef __NetBSD_Prereq__
+#  define __NetBSD_Prereq__(maj, min, patch) false
+#endif
+
+// Fundamental utilities
+
+/**
+ * Get the length of an array.
+ */
+#define countof(array) (sizeof(array) / sizeof(0[array]))
+
+/**
+ * False sharing/destructive interference/largest cache line size.
+ */
+#ifdef __GCC_DESTRUCTIVE_SIZE
+#  define FALSE_SHARING_SIZE __GCC_DESTRUCTIVE_SIZE
+#else
+#  define FALSE_SHARING_SIZE 64
+#endif
+
+/**
+ * True sharing/constructive interference/smallest cache line size.
+ */
+#ifdef __GCC_CONSTRUCTIVE_SIZE
+#  define TRUE_SHARING_SIZE __GCC_CONSTRUCTIVE_SIZE
+#else
+#  define TRUE_SHARING_SIZE 64
+#endif
+
+/**
+ * Alignment specifier that avoids false sharing.
+ */
+#define cache_align alignas(FALSE_SHARING_SIZE)
+
+#if __COSMOPOLITAN__
+typedef long double max_align_t;
+#endif
+
+// Wrappers for attributes
+
+/**
+ * Silence warnings about switch/case fall-throughs.
+ */
+#if __has_attribute(fallthrough)
+#  define fallthru __attribute__((fallthrough))
+#else
+#  define fallthru ((void)0)
+#endif
+
+/**
+ * Silence warnings about unused declarations.
+ */
+#if __has_attribute(unused)
+#  define attr_maybe_unused __attribute__((unused))
+#else
+#  define attr_maybe_unused
+#endif
+
+/**
+ * Warn if a value is unused.
+ */
+#if __has_attribute(warn_unused_result)
+#  define attr_nodiscard __attribute__((warn_unused_result))
+#else
+#  define attr_nodiscard
+#endif
+
+/**
+ * Hint to avoid inlining a function.
+ */
+#if __has_attribute(noinline)
+#  define attr_noinline __attribute__((noinline))
+#else
+#  define attr_noinline
+#endif
+
+/**
+ * Hint that a function is unlikely to be called.
+ */
+#if __has_attribute(cold)
+#  define attr_cold attr_noinline __attribute__((cold))
+#else
+#  define attr_cold attr_noinline
+#endif
+
+/**
+ * Adds compiler warnings for bad printf()-style function calls, if supported.
+ */
+#if __has_attribute(format)
+#  define attr_printf(fmt, args) __attribute__((format(printf, fmt, args)))
+#else
+#  define attr_printf(fmt, args)
+#endif
+
+/**
+ * Annotates allocator-like functions.
+ */
+#if __has_attribute(malloc)
+#  if __GNUC__ >= 11 && !__OPTIMIZE__ // malloc(deallocator) disables inlining on GCC
+#    define attr_malloc(...) attr_nodiscard __attribute__((malloc(__VA_ARGS__)))
+#  else
+#    define attr_malloc(...) attr_nodiscard __attribute__((malloc))
+#  endif
+#else
+#  define attr_malloc(...) attr_nodiscard
+#endif
+
+/**
+ * Specifies that a function returns allocations with a given alignment.
+ */
+#if __has_attribute(alloc_align)
+#  define attr_alloc_align(param) __attribute__((alloc_align(param)))
+#else
+#  define attr_alloc_align(param)
+#endif
+
+/**
+ * Specifies that a function returns allocations with a given size.
+ */
+#if __has_attribute(alloc_size)
+#  define attr_alloc_size(...) __attribute__((alloc_size(__VA_ARGS__)))
+#else
+#  define attr_alloc_size(...)
+#endif
+
+/**
+ * Shorthand for attr_alloc_align() and attr_alloc_size().
+ */
+#define attr_aligned_alloc(align, ...) \
+	attr_alloc_align(align) \
+	attr_alloc_size(__VA_ARGS__)
+
+/**
+ * Check if function multiversioning via GNU indirect functions (ifunc) is supported.
+ */
+#ifndef BFS_USE_TARGET_CLONES
+#  if __has_attribute(target_clones) && (__GLIBC__ || __FreeBSD__)
+#    define BFS_USE_TARGET_CLONES true
+#  endif
+#endif
+
+/**
+ * Apply the target_clones attribute, if available.
+ */
+#if BFS_USE_TARGET_CLONES
+#  define attr_target_clones(...) __attribute__((target_clones(__VA_ARGS__)))
+#else
+#  define attr_target_clones(...)
+#endif
+
+/**
+ * Shorthand for multiple attributes at once. attr(a, b(c), d) is equivalent to
+ *
+ *     attr_a
+ *     attr_b(c)
+ *     attr_d
+ */
+#define attr(...) \
+	attr__(attr_##__VA_ARGS__, none, none, none, none, none, none, none, none, none, )
+
+/**
+ * attr() helper.  For exposition, pretend we support only 2 args, instead of 9.
+ * There are a few cases:
+ *
+ *     attr()
+ *         => attr__(attr_, none, none)
+ *         => attr_                =>
+ *            attr_none            =>
+ *            attr_too_many_none() =>
+ *
+ *     attr(a)
+ *         => attr__(attr_a, none, none)
+ *         => attr_a               => __attribute__((a))
+ *            attr_none            =>
+ *            attr_too_many_none() =>
+ *
+ *     attr(a, b(c))
+ *         => attr__(attr_a, b(c), none, none)
+ *         => attr_a                   => __attribute__((a))
+ *            attr_b(c)                => __attribute__((b(c)))
+ *            attr_too_many_none(none) =>
+ *
+ *     attr(a, b(c), d)
+ *         => attr__(attr_a, b(c), d, none, none)
+ *         => attr_a                      => __attribute__((a))
+ *            attr_b(c)                   => __attribute__((b(c)))
+ *            attr_too_many_d(none, none) => error
+ *
+ * Some attribute names are the same as standard library functions, e.g. printf.
+ * Standard libraries are permitted to define these functions as macros, like
+ *
+ *     #define printf(...) __builtin_printf(__VA_ARGS__)
+ *
+ * The token paste in
+ *
+ *     #define attr(...) attr__(attr_##__VA_ARGS__, none, none)
+ *
+ * is necessary to prevent macro expansion before evaluating attr__().
+ * Otherwise, we could get
+ *
+ *     attr(printf(1, 2))
+ *         => attr__(__builtin_printf(1, 2), none, none)
+ *         => attr____builtin_printf(1, 2)
+ *         => error
+ */
+#define attr__(a1, a2, a3, a4, a5, a6, a7, a8, a9, none, ...) \
+	a1 \
+	attr_##a2 \
+	attr_##a3 \
+	attr_##a4 \
+	attr_##a5 \
+	attr_##a6 \
+	attr_##a7 \
+	attr_##a8 \
+	attr_##a9 \
+	attr_too_many_##none(__VA_ARGS__)
+
+// Ignore `attr_none` from expanding 1-9 argument attr(a1, a2, ...)
+#define attr_none
+// Ignore `attr_` from expanding 0-argument attr()
+#define attr_
+// Only trigger an error on more than 9 arguments
+#define attr_too_many_none(...)
+
+#endif // BFS_PRELUDE_H
diff --git a/src/printf.c b/src/printf.c
index 3b8269e..4df399b 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -1,12 +1,12 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "printf.h"
 #include "alloc.h"
 #include "bfstd.h"
 #include "bftw.h"
 #include "color.h"
-#include "config.h"
 #include "ctx.h"
 #include "diag.h"
 #include "dir.h"
diff --git a/src/pwcache.c b/src/pwcache.c
index 79437d8..af8c237 100644
--- a/src/pwcache.c
+++ b/src/pwcache.c
@@ -1,9 +1,9 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "pwcache.h"
 #include "alloc.h"
-#include "config.h"
 #include "trie.h"
 #include <errno.h>
 #include <grp.h>
diff --git a/src/sanity.h b/src/sanity.h
index 423e6ff..e168b8f 100644
--- a/src/sanity.h
+++ b/src/sanity.h
@@ -8,7 +8,7 @@
 #ifndef BFS_SANITY_H
 #define BFS_SANITY_H
 
-#include "config.h"
+#include "prelude.h"
 #include <stddef.h>
 
 #if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
diff --git a/src/stat.c b/src/stat.c
index 2f2743b..eca5bab 100644
--- a/src/stat.c
+++ b/src/stat.c
@@ -1,10 +1,10 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "stat.h"
 #include "atomic.h"
 #include "bfstd.h"
-#include "config.h"
 #include "diag.h"
 #include "sanity.h"
 #include <errno.h>
diff --git a/src/stat.h b/src/stat.h
index 856a2ca..1fdd263 100644
--- a/src/stat.h
+++ b/src/stat.h
@@ -12,7 +12,7 @@
 #ifndef BFS_STAT_H
 #define BFS_STAT_H
 
-#include "config.h"
+#include "prelude.h"
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <time.h>
diff --git a/src/thread.c b/src/thread.c
index 200d8c3..3793896 100644
--- a/src/thread.c
+++ b/src/thread.c
@@ -1,9 +1,9 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "thread.h"
 #include "bfstd.h"
-#include "config.h"
 #include "diag.h"
 #include <errno.h>
 #include <pthread.h>
diff --git a/src/thread.h b/src/thread.h
index 8174fe4..db11bd8 100644
--- a/src/thread.h
+++ b/src/thread.h
@@ -8,7 +8,7 @@
 #ifndef BFS_THREAD_H
 #define BFS_THREAD_H
 
-#include "config.h"
+#include "prelude.h"
 #include <pthread.h>
 
 #if __STDC_VERSION__ < C23 && !defined(thread_local)
diff --git a/src/trie.c b/src/trie.c
index f275064..808953e 100644
--- a/src/trie.c
+++ b/src/trie.c
@@ -81,10 +81,10 @@
  * and insert intermediate singleton "jump" nodes when necessary.
  */
 
+#include "prelude.h"
 #include "trie.h"
 #include "alloc.h"
 #include "bit.h"
-#include "config.h"
 #include "diag.h"
 #include "list.h"
 #include <stdint.h>
diff --git a/src/xregex.c b/src/xregex.c
index 3df27f0..c2711bc 100644
--- a/src/xregex.c
+++ b/src/xregex.c
@@ -1,10 +1,10 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "xregex.h"
 #include "alloc.h"
 #include "bfstd.h"
-#include "config.h"
 #include "diag.h"
 #include "sanity.h"
 #include "thread.h"
diff --git a/src/xspawn.c b/src/xspawn.c
index 347625d..113d7ec 100644
--- a/src/xspawn.c
+++ b/src/xspawn.c
@@ -1,10 +1,10 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "xspawn.h"
 #include "alloc.h"
 #include "bfstd.h"
-#include "config.h"
 #include "list.h"
 #include <errno.h>
 #include <fcntl.h>
diff --git a/src/xspawn.h b/src/xspawn.h
index a20cbd0..6a8f54a 100644
--- a/src/xspawn.h
+++ b/src/xspawn.h
@@ -8,7 +8,7 @@
 #ifndef BFS_XSPAWN_H
 #define BFS_XSPAWN_H
 
-#include "config.h"
+#include "prelude.h"
 #include <sys/resource.h>
 #include <sys/types.h>
 #include <unistd.h>
diff --git a/src/xtime.c b/src/xtime.c
index bcf6dd3..91ed915 100644
--- a/src/xtime.c
+++ b/src/xtime.c
@@ -1,9 +1,9 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "xtime.h"
 #include "bfstd.h"
-#include "config.h"
 #include "diag.h"
 #include <errno.h>
 #include <limits.h>
diff --git a/tests/alloc.c b/tests/alloc.c
index 54b84ba..6c0defd 100644
--- a/tests/alloc.c
+++ b/tests/alloc.c
@@ -1,9 +1,9 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "tests.h"
 #include "alloc.h"
-#include "config.h"
 #include "diag.h"
 #include <errno.h>
 #include <stdlib.h>
diff --git a/tests/bfstd.c b/tests/bfstd.c
index 5e408ca..07b68b0 100644
--- a/tests/bfstd.c
+++ b/tests/bfstd.c
@@ -1,9 +1,9 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "tests.h"
 #include "bfstd.h"
-#include "config.h"
 #include "diag.h"
 #include <errno.h>
 #include <langinfo.h>
diff --git a/tests/bit.c b/tests/bit.c
index b444e50..674d1b2 100644
--- a/tests/bit.c
+++ b/tests/bit.c
@@ -1,9 +1,9 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "tests.h"
 #include "bit.h"
-#include "config.h"
 #include "diag.h"
 #include <limits.h>
 #include <stdint.h>
diff --git a/tests/ioq.c b/tests/ioq.c
index a69f2bf..ef5ee3b 100644
--- a/tests/ioq.c
+++ b/tests/ioq.c
@@ -1,10 +1,10 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "tests.h"
 #include "ioq.h"
 #include "bfstd.h"
-#include "config.h"
 #include "diag.h"
 #include "dir.h"
 #include <errno.h>
diff --git a/tests/main.c b/tests/main.c
index 281c417..429772b 100644
--- a/tests/main.c
+++ b/tests/main.c
@@ -5,10 +5,10 @@
  * Entry point for unit tests.
  */
 
+#include "prelude.h"
 #include "tests.h"
 #include "bfstd.h"
 #include "color.h"
-#include "config.h"
 #include <errno.h>
 #include <locale.h>
 #include <stdio.h>
diff --git a/tests/tests.h b/tests/tests.h
index d61ffd7..9078938 100644
--- a/tests/tests.h
+++ b/tests/tests.h
@@ -8,7 +8,7 @@
 #ifndef BFS_TESTS_H
 #define BFS_TESTS_H
 
-#include "config.h"
+#include "prelude.h"
 #include "diag.h"
 
 /** Unit test function type. */
diff --git a/tests/trie.c b/tests/trie.c
index 2a6eb48..4667322 100644
--- a/tests/trie.c
+++ b/tests/trie.c
@@ -1,9 +1,9 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "tests.h"
 #include "trie.h"
-#include "config.h"
 #include "diag.h"
 #include <stdlib.h>
 #include <string.h>
diff --git a/tests/xspawn.c b/tests/xspawn.c
index fd8362e..7362aa5 100644
--- a/tests/xspawn.c
+++ b/tests/xspawn.c
@@ -1,10 +1,10 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "tests.h"
 #include "alloc.h"
 #include "bfstd.h"
-#include "config.h"
 #include "dstring.h"
 #include "xspawn.h"
 #include <stdlib.h>
diff --git a/tests/xtime.c b/tests/xtime.c
index fd7aa0f..a7c63d2 100644
--- a/tests/xtime.c
+++ b/tests/xtime.c
@@ -1,10 +1,10 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "tests.h"
 #include "xtime.h"
 #include "bfstd.h"
-#include "config.h"
 #include "diag.h"
 #include <errno.h>
 #include <limits.h>
diff --git a/tests/xtouch.c b/tests/xtouch.c
index b1daec7..82d749d 100644
--- a/tests/xtouch.c
+++ b/tests/xtouch.c
@@ -1,8 +1,8 @@
 // Copyright © Tavian Barnes <tavianator@tavianator.com>
 // SPDX-License-Identifier: 0BSD
 
+#include "prelude.h"
 #include "bfstd.h"
-#include "config.h"
 #include "sanity.h"
 #include "xtime.h"
 #include <errno.h>
-- 
cgit v1.2.3


From 19189e3f6f18b13e4f9c947c99062e658e98827d Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@tavianator.com>
Date: Wed, 24 Apr 2024 14:45:12 -0400
Subject: ioq: Fix uninitialized values in ioq_create() cleanup path

I switched from ZALLOC_FLEX() to ALLOC_FLEX() in hopes that msan would
catch uninitialized values in ioq_thread_create(), but in doing so,
forgot to initialize all fields before the first goto fail.

Fixes: f64f76b ("ioq: Copy ring_ops from the previous thread")
---
 src/ioq.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

(limited to 'src/ioq.c')

diff --git a/src/ioq.c b/src/ioq.c
index 189bdac..43a1b35 100644
--- a/src/ioq.c
+++ b/src/ioq.c
@@ -918,17 +918,14 @@ static void ioq_thread_join(struct ioq_thread *thread) {
 }
 
 struct ioq *ioq_create(size_t depth, size_t nthreads) {
-	struct ioq *ioq = ALLOC_FLEX(struct ioq, threads, nthreads);
+	struct ioq *ioq = ZALLOC_FLEX(struct ioq, threads, nthreads);
 	if (!ioq) {
 		goto fail;
 	}
 
 	ioq->depth = depth;
-	ioq->size = 0;
-	ioq->cancel = false;
 
 	ARENA_INIT(&ioq->ents, struct ioq_ent);
-
 #if BFS_USE_LIBURING && BFS_USE_STATX
 	ARENA_INIT(&ioq->xbufs, struct statx);
 #endif
-- 
cgit v1.2.3