Journal: which services are filling up the allowed journal-size?

Question

On a Debian machine, by default the systemd-journal is caring about the logs.

It's configuration limits the max disk space it can use:

$ cat /etc/systemd/journald.conf
SystemMaxUse=100M
SystemKeepFree=200M

Now its directory is filled up to around 100M:

$ du -hs /var/log/journal
109M    /var/log/journal

Inside the journal directory, there is only one single directory with many binary files inside.

Is there a way to know the distribution of this 109M space by services?

I.e.

nginx: 15M
httpd: 20M
...

Currently I didn't found any way to check this information.

Answer 1

You can do that, but only by going through all entries in your journal manually, using the sd-journal C API. It's not complicated, and copying together an example, some of it from bits and pieces from the sd_journal_next manpage and replacing _get_data with _enumerate_data:

I put the source code here. **This is a bit quick&dirty, the sd_* documentation is really not that good, so here's to hoping things are correct:

/* SPDX-License-Identifier: GPLv3 */
/* Copyright 2023 Marcus Müller
 * Somewhat based on man sd_journal_get_data (MIT-0), but most of the original work by
 * Marcus Müller
 */
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <systemd/sd-journal.h>
const char* KEY = "_SYSTEMD_UNIT=";
const char* ALTKEY = "_TRANSPORT=";
int main(int argc, char* argv[])
{
    int r;
    size_t KEYLEN = strlen(KEY);
    size_t ALTKEYLEN = strlen(ALTKEY);
    sd_journal* j;
    r = sd_journal_open(&j, SD_JOURNAL_LOCAL_ONLY);
    if (r < 0) {
        errno = -r;
        fprintf(stderr, "Failed to open journal: %m\n");
        return 1;
    }
    /* Print header */
    puts("unit name;length\n");
SD_JOURNAL_FOREACH(j)
{
    const char* data_ptr;
    size_t field_length;
    size_t total_length = 0;
    char* unitname = NULL;
    size_t unitname_len = 0;
    int unit_found = 0;

    while (0 &lt; sd_journal_enumerate_data(j, (const void**)&amp;data_ptr, &amp;field_length)) {
        /* We just break the counting if we encounter a broken entry. */
        total_length += field_length;
        if (unit_found == 0) {
            if (field_length &gt; KEYLEN &amp;&amp; strncmp(data_ptr, KEY, KEYLEN) == 0) {
                unit_found = 1;
                unitname_len = field_length - KEYLEN;
                unitname = realloc(unitname, unitname_len);
                strncpy(unitname, data_ptr + KEYLEN, unitname_len);
            } else if (field_length &gt; ALTKEYLEN &amp;&amp; (unitname_len == 0) &amp;&amp;
                       strncmp(data_ptr, ALTKEY, ALTKEYLEN) == 0) {
                unitname_len = field_length - ALTKEYLEN;
                unitname = realloc(unitname, unitname_len);
                strncpy(unitname, data_ptr + ALTKEYLEN, unitname_len);
            }
        }
    }

    if (unitname_len == 0) {
        printf(&quot;UNKNOWN;%zu\n&quot;, total_length);
    } else {
        printf(&quot;%*s;%zu\n&quot;, (int)unitname_len, unitname, total_length);
        free(unitname);
    }
}
sd_journal_close(j);
return 0;

}

save as journalsizes.c, compile with

c99 -o journalsizes $(pkg-config --cflags --libs libsystemd) journalsizes.c

and run by ./journalsizes to get a semicolon-separated list of unit names ; log entry lengths in bytes.

You can use that, for example, by putting into a text file called accumulate_logsizes.sql the following:

.mode csv
.separator ;
.import "|./journalsizes" journal
SELECT
  "unit name",
  SUM("length")/(1024*1024.0) AS megabyte
FROM
  journal
GROUP BY
  "unit name"
ORDER BY
  megabyte
DESC LIMIT 10;

and running that through sqlite3:

sqlite3 < accumulate_logsizes.sql