From 3a22b45974ddd1230da0dfa21f886c3401bee020 Mon Sep 17 00:00:00 2001 From: Russell Nelson Date: Fri, 12 Jul 2019 23:19:51 -0600 Subject: [PATCH] Apply Russell Nelson's big-todo.103.patch. ---Note to readers: After you apply this patch, your queue will no longer be readable in its current state. So, either install this into /var/qmail2 and run both qmails until the queue is emptied on the old one, or else accept that your current queue will have to be destroyed and ``rm -rf /var/qmail/queue; make setup''.--- From: Bruce Guenter To: Russell Nelson Subject: Missing piece of your big-todo patch Date: Tue, 3 Aug 1999 08:12:11 -0600 Greetings. While testing the performance of a big todo directory, I observed that qmail-qstat miscounted the number of messages in the todo directory. I've appended the necessary patch to yours. -- Bruce Guenter http://em.ca/~bruceg/ Another issue that can come into play when email comes into your server extremely quickly is that if several thousand email messages accumulate in the todo queue, an inefficient filesystem that has trouble with large directories may restrict the speed of todo processing significantly. The best way to address this is to use a better filesystem, but that's not always possible, depending on your situation. Of course, qmail already worked around this kind of problem in the rest of the queue by using a hash system to reduce the number of message files per directory in the queue. For whatever reason, this workaround wasn't applied to the todo queue. Russ Nelson wrote a patch to make qmail use that hashing system in the todo portion of the queue as well. This patch is referred to as the "big-todo" patch. There's no reason to apply this patch unless you really really need it, because getting a better filesystem (or enabling the right features on the one you're already using) is the best option; this patch can require that your queue be rebuilt, and so applying it to a running system can be a bit of a pain. --- hier.c | 2 ++ qmail-clean.c | 14 +++++++++----- qmail-qstat.sh | 4 ++-- qmail-queue.c | 4 ++-- qmail-send.c | 38 ++++++++++++++++---------------------- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/hier.c b/hier.c index a3ef141..e532d0f 100644 --- a/hier.c +++ b/hier.c @@ -55,6 +55,8 @@ void hier() d(auto_qmail,"queue/bounce",auto_uids,auto_gidq,0700); dsplit("queue/mess",auto_uidq,0750); + dsplit("queue/todo",auto_uidq,0750); + dsplit("queue/intd",auto_uidq,0700); dsplit("queue/info",auto_uids,0700); dsplit("queue/local",auto_uids,0700); dsplit("queue/remote",auto_uids,0700); diff --git a/qmail-clean.c b/qmail-clean.c index 7539007..4926f86 100644 --- a/qmail-clean.c +++ b/qmail-clean.c @@ -73,22 +73,26 @@ void main() if (line.len < 7) { respond("x"); continue; } if (line.len > 100) { respond("x"); continue; } if (line.s[line.len - 1]) { respond("x"); continue; } /* impossible */ - for (i = 5;i < line.len - 1;++i) + for (i = line.len - 2;i > 4;--i) + { + if (line.s[i] == '/') break; if ((unsigned char) (line.s[i] - '0') > 9) { respond("x"); continue; } - if (!scan_ulong(line.s + 5,&id)) { respond("x"); continue; } + } + if (line.s[i] == '/') + if (!scan_ulong(line.s + i + 1,&id)) { respond("x"); continue; } if (byte_equal(line.s,5,"foop/")) { #define U(prefix,flag) fmtqfn(fnbuf,prefix,id,flag); \ if (unlink(fnbuf) == -1) if (errno != error_noent) { respond("!"); continue; } - U("intd/",0) + U("intd/",1) U("mess/",1) respond("+"); } else if (byte_equal(line.s,4,"todo/")) { - U("intd/",0) - U("todo/",0) + U("intd/",1) + U("todo/",1) respond("+"); } else diff --git a/qmail-qstat.sh b/qmail-qstat.sh index 26a6d3f..243f29c 100644 --- a/qmail-qstat.sh +++ b/qmail-qstat.sh @@ -1,7 +1,7 @@ cd QMAIL messdirs=`echo queue/mess/* | wc -w` messfiles=`find queue/mess/* -print | wc -w` -tododirs=`echo queue/todo | wc -w` -todofiles=`find queue/todo -print | wc -w` +tododirs=`echo queue/todo/* | wc -w` +todofiles=`find queue/todo/* -print | wc -w` echo messages in queue: `expr $messfiles - $messdirs` echo messages in queue but not yet preprocessed: `expr $todofiles - $tododirs` diff --git a/qmail-queue.c b/qmail-queue.c index 24c0467..2b5744b 100644 --- a/qmail-queue.c +++ b/qmail-queue.c @@ -190,8 +190,8 @@ void main() messnum = pidst.st_ino; messfn = fnnum("mess/",1); - todofn = fnnum("todo/",0); - intdfn = fnnum("intd/",0); + todofn = fnnum("todo/",1); + intdfn = fnnum("intd/",1); if (link(pidfn,messfn) == -1) die(64); if (unlink(pidfn) == -1) die(63); diff --git a/qmail-send.c b/qmail-send.c index f41bfae..3b01a95 100644 --- a/qmail-send.c +++ b/qmail-send.c @@ -96,7 +96,7 @@ void fnmake_init() } void fnmake_info(id) unsigned long id; { fn.len = fmtqfn(fn.s,"info/",id,1); } -void fnmake_todo(id) unsigned long id; { fn.len = fmtqfn(fn.s,"todo/",id,0); } +void fnmake_todo(id) unsigned long id; { fn.len = fmtqfn(fn.s,"todo/",id,1); } void fnmake_mess(id) unsigned long id; { fn.len = fmtqfn(fn.s,"mess/",id,1); } void fnmake_foop(id) unsigned long id; { fn.len = fmtqfn(fn.s,"foop/",id,0); } void fnmake_split(id) unsigned long id; { fn.len = fmtqfn(fn.s,"",id,1); } @@ -1213,7 +1213,8 @@ void pass_do() /* this file is too long ---------------------------------------------- TODO */ datetime_sec nexttodorun; -DIR *tododir; /* if 0, have to opendir again */ +int flagtododir = 0; /* if 0, have to readsubdir_init again */ +readsubdir todosubdir; stralloc todoline = {0}; char todobuf[SUBSTDIO_INSIZE]; char todobufinfo[512]; @@ -1221,7 +1222,7 @@ char todobufchan[CHANNELS][1024]; void todo_init() { - tododir = 0; + flagtododir = 0; nexttodorun = now(); trigger_set(); } @@ -1233,7 +1234,7 @@ datetime_sec *wakeup; { if (flagexitasap) return; trigger_selprep(nfds,rfds); - if (tododir) *wakeup = 0; + if (flagtododir) *wakeup = 0; if (*wakeup > nexttodorun) *wakeup = nexttodorun; } @@ -1250,8 +1251,7 @@ fd_set *rfds; char ch; int match; unsigned long id; - unsigned int len; - direntry *dent; + int z; int c; unsigned long uid; unsigned long pid; @@ -1262,32 +1262,26 @@ fd_set *rfds; if (flagexitasap) return; - if (!tododir) + if (!flagtododir) { if (!trigger_pulled(rfds)) if (recent < nexttodorun) return; trigger_set(); - tododir = opendir("todo"); - if (!tododir) - { - pausedir("todo"); - return; - } + readsubdir_init(&todosubdir, "todo", pausedir); + flagtododir = 1; nexttodorun = recent + SLEEP_TODO; } - dent = readdir(tododir); - if (!dent) + switch(readsubdir_next(&todosubdir, &id)) { - closedir(tododir); - tododir = 0; - return; + case 1: + break; + case 0: + flagtododir = 0; + default: + return; } - if (str_equal(dent->d_name,".")) return; - if (str_equal(dent->d_name,"..")) return; - len = scan_ulong(dent->d_name,&id); - if (!len || dent->d_name[len]) return; fnmake_todo(id);