From 57850536d501376feb46fe908418937376efe85f Mon Sep 17 00:00:00 2001 From: Alex Gartrell Date: Mon, 26 Feb 2018 21:56:35 -0800 Subject: [PATCH 1/2] journal: provide compress_threshold_bytes parameter Previously the compression threshold was hardcoded to 512, which meant that smaller values wouldn't be compressed. This left some storage savings on the table, so instead, we make that number tunable. --- src/journal-remote/journal-remote-write.c | 2 +- src/journal-remote/journal-remote.c | 4 +- src/journal/journal-file.c | 30 +++++-- src/journal/journal-file.h | 5 +- src/journal/journald-server.c | 6 +- src/journal/sd-journal.c | 2 +- src/journal/test-journal-flush.c | 2 +- src/journal/test-journal-interleaving.c | 8 +- src/journal/test-journal-stream.c | 6 +- src/journal/test-journal-verify.c | 6 +- src/journal/test-journal.c | 102 ++++++++++++++++++++-- 11 files changed, 141 insertions(+), 32 deletions(-) diff --git a/src/journal-remote/journal-remote-write.c b/src/journal-remote/journal-remote-write.c index c5c6abbd5c..2836ba8b48 100644 --- a/src/journal-remote/journal-remote-write.c +++ b/src/journal-remote/journal-remote-write.c @@ -22,7 +22,7 @@ #include "journal-remote.h" static int do_rotate(JournalFile **f, bool compress, bool seal) { - int r = journal_file_rotate(f, compress, seal, NULL); + int r = journal_file_rotate(f, compress, (uint64_t) -1, seal, NULL); if (r < 0) { if (*f) log_error_errno(r, "Failed to rotate %s: %m", (*f)->path); diff --git a/src/journal-remote/journal-remote.c b/src/journal-remote/journal-remote.c index 428725223d..ff46023920 100644 --- a/src/journal-remote/journal-remote.c +++ b/src/journal-remote/journal-remote.c @@ -185,7 +185,7 @@ static int open_output(Writer *w, const char* host) { r = journal_file_open_reliably(output, O_RDWR|O_CREAT, 0640, - arg_compress, arg_seal, + arg_compress, (uint64_t) -1, arg_seal, &w->metrics, w->mmap, NULL, NULL, &w->journal); @@ -729,7 +729,7 @@ static int setup_microhttpd_server(RemoteServer *s, } r = sd_event_add_time(s->events, &d->timer_event, - CLOCK_MONOTONIC, UINT64_MAX, 0, + CLOCK_MONOTONIC, (uint64_t) -1, 0, null_timer_event_handler, d); if (r < 0) { log_error_errno(r, "Failed to add timer_event: %m"); diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c index 5643c0578d..5fc61b16e0 100644 --- a/src/journal/journal-file.c +++ b/src/journal/journal-file.c @@ -51,7 +51,8 @@ #define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem)) #define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem)) -#define COMPRESSION_SIZE_THRESHOLD (512ULL) +#define DEFAULT_COMPRESS_THRESHOLD (512ULL) +#define MIN_COMPRESS_THRESHOLD (8ULL) /* This is the minimum journal file size */ #define JOURNAL_FILE_SIZE_MIN (512ULL*1024ULL) /* 512 KiB */ @@ -1552,7 +1553,7 @@ static int journal_file_append_data( o->data.hash = htole64(hash); #if HAVE_XZ || HAVE_LZ4 - if (JOURNAL_FILE_COMPRESS(f) && size >= COMPRESSION_SIZE_THRESHOLD) { + if (JOURNAL_FILE_COMPRESS(f) && size >= f->compress_threshold_bytes) { size_t rsize = 0; compression = compress_blob(data, size, o->data.payload, size - 1, &rsize); @@ -3211,6 +3212,7 @@ int journal_file_open( int flags, mode_t mode, bool compress, + uint64_t compress_threshold_bytes, bool seal, JournalMetrics *metrics, MMapCache *mmap_cache, @@ -3222,6 +3224,7 @@ int journal_file_open( JournalFile *f; void *h; int r; + char bytes[FORMAT_BYTES_MAX]; assert(ret); assert(fd >= 0 || fname); @@ -3247,10 +3250,20 @@ int journal_file_open( #elif HAVE_XZ f->compress_xz = compress; #endif + + if (compress_threshold_bytes == (uint64_t) -1) + f->compress_threshold_bytes = DEFAULT_COMPRESS_THRESHOLD; + else + f->compress_threshold_bytes = MAX(MIN_COMPRESS_THRESHOLD, compress_threshold_bytes); + #if HAVE_GCRYPT f->seal = seal; #endif + log_debug("Journal effective settings seal=%s compress=%s compress_threshold_bytes=%s", + yes_no(f->seal), yes_no(JOURNAL_FILE_COMPRESS(f)), + format_bytes(bytes, sizeof(bytes), f->compress_threshold_bytes)); + if (mmap_cache) f->mmap = mmap_cache_ref(mmap_cache); else { @@ -3435,7 +3448,7 @@ fail: return r; } -int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes) { +int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes) { _cleanup_free_ char *p = NULL; size_t l; JournalFile *old_file, *new_file = NULL; @@ -3489,7 +3502,9 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred * we archive them */ old_file->defrag_on_close = true; - r = journal_file_open(-1, old_file->path, old_file->flags, old_file->mode, compress, seal, NULL, old_file->mmap, deferred_closes, old_file, &new_file); + r = journal_file_open(-1, old_file->path, old_file->flags, old_file->mode, compress, + compress_threshold_bytes, seal, NULL, old_file->mmap, deferred_closes, + old_file, &new_file); if (deferred_closes && set_put(deferred_closes, old_file) >= 0) @@ -3506,6 +3521,7 @@ int journal_file_open_reliably( int flags, mode_t mode, bool compress, + uint64_t compress_threshold_bytes, bool seal, JournalMetrics *metrics, MMapCache *mmap_cache, @@ -3517,7 +3533,8 @@ int journal_file_open_reliably( size_t l; _cleanup_free_ char *p = NULL; - r = journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret); + r = journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache, + deferred_closes, template, ret); if (!IN_SET(r, -EBADMSG, /* Corrupted */ -ENODATA, /* Truncated */ @@ -3559,7 +3576,8 @@ int journal_file_open_reliably( log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname); - return journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret); + return journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache, + deferred_closes, template, ret); } int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) { diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h index 67abf8da49..04d35f5813 100644 --- a/src/journal/journal-file.h +++ b/src/journal/journal-file.h @@ -124,6 +124,7 @@ typedef struct JournalFile { unsigned last_seen_generation; + uint64_t compress_threshold_bytes; #if HAVE_XZ || HAVE_LZ4 void *compress_buffer; size_t compress_buffer_size; @@ -153,6 +154,7 @@ int journal_file_open( int flags, mode_t mode, bool compress, + uint64_t compress_threshold_bytes, bool seal, JournalMetrics *metrics, MMapCache *mmap_cache, @@ -169,6 +171,7 @@ int journal_file_open_reliably( int flags, mode_t mode, bool compress, + uint64_t compress_threshold_bytes, bool seal, JournalMetrics *metrics, MMapCache *mmap_cache, @@ -246,7 +249,7 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6 void journal_file_dump(JournalFile *f); void journal_file_print_header(JournalFile *f); -int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes); +int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes); void journal_file_post_change(JournalFile *f); int journal_file_enable_post_change_timer(JournalFile *f, sd_event *e, usec_t t); diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 5cd58e8a77..570450d321 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -280,9 +280,9 @@ static int open_journal( assert(ret); if (reliably) - r = journal_file_open_reliably(fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f); + r = journal_file_open_reliably(fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f); else - r = journal_file_open(-1, fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f); + r = journal_file_open(-1, fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f); if (r < 0) return r; @@ -463,7 +463,7 @@ static int do_rotate( if (!*f) return -EINVAL; - r = journal_file_rotate(f, s->compress, seal, s->deferred_closes); + r = journal_file_rotate(f, s->compress, (uint64_t) -1, seal, s->deferred_closes); if (r < 0) { if (*f) return log_error_errno(r, "Failed to rotate %s: %m", (*f)->path); diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c index 11dbd83f2d..79ffd9cb3f 100644 --- a/src/journal/sd-journal.c +++ b/src/journal/sd-journal.c @@ -1337,7 +1337,7 @@ static int add_any_file( goto finish; } - r = journal_file_open(fd, path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, NULL, &f); + r = journal_file_open(fd, path, O_RDONLY, 0, false, 0, false, NULL, j->mmap, NULL, NULL, &f); if (r < 0) { log_debug_errno(r, "Failed to open journal file %s: %m", path); goto finish; diff --git a/src/journal/test-journal-flush.c b/src/journal/test-journal-flush.c index f9b7b75ef1..e302fd61f1 100644 --- a/src/journal/test-journal-flush.c +++ b/src/journal/test-journal-flush.c @@ -39,7 +39,7 @@ int main(int argc, char *argv[]) { assert_se(mkdtemp(dn)); fn = strappend(dn, "/test.journal"); - r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, false, NULL, NULL, NULL, NULL, &new_journal); + r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, 0, false, NULL, NULL, NULL, NULL, &new_journal); assert_se(r >= 0); r = sd_journal_open(&j, 0); diff --git a/src/journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c index d87bdbdd32..f8b8d1e6d8 100644 --- a/src/journal/test-journal-interleaving.c +++ b/src/journal/test-journal-interleaving.c @@ -52,7 +52,7 @@ _noreturn_ static void log_assert_errno(const char *text, int error, const char static JournalFile *test_open(const char *name) { JournalFile *f; - assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, false, NULL, NULL, NULL, NULL, &f)); + assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f)); return f; } @@ -217,7 +217,7 @@ static void test_sequence_numbers(void) { assert_se(chdir(t) >= 0); assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0644, - true, false, NULL, NULL, NULL, NULL, &one) == 0); + true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0); append_number(one, 1, &seqnum); printf("seqnum=%"PRIu64"\n", seqnum); @@ -234,7 +234,7 @@ static void test_sequence_numbers(void) { memcpy(&seqnum_id, &one->header->seqnum_id, sizeof(sd_id128_t)); assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0644, - true, false, NULL, NULL, NULL, one, &two) == 0); + true, (uint64_t) -1, false, NULL, NULL, NULL, one, &two) == 0); assert_se(two->header->state == STATE_ONLINE); assert_se(!sd_id128_equal(two->header->file_id, one->header->file_id)); @@ -265,7 +265,7 @@ static void test_sequence_numbers(void) { seqnum = 0; assert_se(journal_file_open(-1, "two.journal", O_RDWR, 0, - true, false, NULL, NULL, NULL, NULL, &two) == 0); + true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0); assert_se(sd_id128_equal(two->header->seqnum_id, seqnum_id)); diff --git a/src/journal/test-journal-stream.c b/src/journal/test-journal-stream.c index 73ed6e5dcb..279301c67d 100644 --- a/src/journal/test-journal-stream.c +++ b/src/journal/test-journal-stream.c @@ -93,9 +93,9 @@ int main(int argc, char *argv[]) { assert_se(mkdtemp(t)); assert_se(chdir(t) >= 0); - assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &one) == 0); - assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &two) == 0); - assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &three) == 0); + assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0); + assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0); + assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &three) == 0); for (i = 0; i < N_ENTRIES; i++) { char *p, *q; diff --git a/src/journal/test-journal-verify.c b/src/journal/test-journal-verify.c index fbb75e43e3..89389579f3 100644 --- a/src/journal/test-journal-verify.c +++ b/src/journal/test-journal-verify.c @@ -56,7 +56,7 @@ static int raw_verify(const char *fn, const char *verification_key) { JournalFile *f; int r; - r = journal_file_open(-1, fn, O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f); + r = journal_file_open(-1, fn, O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f); if (r < 0) return r; @@ -89,7 +89,7 @@ int main(int argc, char *argv[]) { log_info("Generating..."); - assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0); + assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0); for (n = 0; n < N_ENTRIES; n++) { struct iovec iovec; @@ -112,7 +112,7 @@ int main(int argc, char *argv[]) { log_info("Verifying..."); - assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0); + assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0); /* journal_file_print_header(f); */ journal_file_dump(f); diff --git a/src/journal/test-journal.c b/src/journal/test-journal.c index 517c9102a6..3b9917b4a8 100644 --- a/src/journal/test-journal.c +++ b/src/journal/test-journal.c @@ -43,7 +43,7 @@ static void test_non_empty(void) { assert_se(mkdtemp(t)); assert_se(chdir(t) >= 0); - assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f) == 0); + assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f) == 0); dual_timestamp_get(&ts); @@ -105,8 +105,8 @@ static void test_non_empty(void) { assert_se(journal_file_move_to_entry_by_seqnum(f, 10, DIRECTION_DOWN, &o, NULL) == 0); - journal_file_rotate(&f, true, true, NULL); - journal_file_rotate(&f, true, true, NULL); + journal_file_rotate(&f, true, (uint64_t) -1, true, NULL); + journal_file_rotate(&f, true, (uint64_t) -1, true, NULL); (void) journal_file_close(f); @@ -132,13 +132,13 @@ static void test_empty(void) { assert_se(mkdtemp(t)); assert_se(chdir(t) >= 0); - assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, false, NULL, NULL, NULL, NULL, &f1) == 0); + assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f1) == 0); - assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &f2) == 0); + assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f2) == 0); - assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, true, NULL, NULL, NULL, NULL, &f3) == 0); + assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f3) == 0); - assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f4) == 0); + assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f4) == 0); journal_file_print_header(f1); puts(""); @@ -165,6 +165,91 @@ static void test_empty(void) { (void) journal_file_close(f4); } +#if HAVE_XZ || HAVE_LZ4 +static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) { + dual_timestamp ts; + JournalFile *f; + struct iovec iovec; + Object *o; + uint64_t p; + char t[] = "/tmp/journal-XXXXXX"; + char data[2048] = {0}; + bool is_compressed; + int r; + + assert_se(data_size <= sizeof(data)); + + log_set_max_level(LOG_DEBUG); + + assert_se(mkdtemp(t)); + assert_se(chdir(t) >= 0); + + assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0); + + dual_timestamp_get(&ts); + + iovec.iov_base = (void*) data; + iovec.iov_len = data_size; + assert_se(journal_file_append_entry(f, &ts, &iovec, 1, NULL, NULL, NULL) == 0); + +#if HAVE_GCRYPT + journal_file_append_tag(f); +#endif + journal_file_dump(f); + + /* We have to partially reimplement some of the dump logic, because the normal next_entry does the + * decompression for us. */ + p = le64toh(f->header->header_size); + while (true) { + r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o); + assert_se(r == 0); + if (o->object.type == OBJECT_DATA) + break; + + assert_se(p < le64toh(f->header->tail_object_offset)); + p = p + ALIGN64(le64toh(o->object.size)); + } + + is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0; + + (void) journal_file_close(f); + + log_info("Done..."); + + if (arg_keep) + log_info("Not removing %s", t); + else { + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); + + assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); + } + + puts("------------------------------------------------------------"); + + return is_compressed; +} + +static void test_min_compress_size(void) { + /* Note that XZ will actually fail to compress anything under 80 bytes, so you have to choose the limits + * carefully */ + + /* DEFAULT_MIN_COMPRESS_SIZE is 512 */ + assert_se(!check_compressed((uint64_t) -1, 255)); + assert_se(check_compressed((uint64_t) -1, 513)); + + /* compress everything */ + assert_se(check_compressed(0, 96)); + assert_se(check_compressed(8, 96)); + + /* Ensure we don't try to compress less than 8 bytes */ + assert_se(!check_compressed(0, 7)); + + /* check boundary conditions */ + assert_se(check_compressed(256, 256)); + assert_se(!check_compressed(256, 255)); +} +#endif + int main(int argc, char *argv[]) { arg_keep = argc > 1; @@ -174,6 +259,9 @@ int main(int argc, char *argv[]) { test_non_empty(); test_empty(); +#if HAVE_XZ || HAVE_LZ4 + test_min_compress_size(); +#endif return 0; } From 1b7cf0e5879ec04cb372766385228e8fb75e5500 Mon Sep 17 00:00:00 2001 From: Alex Gartrell Date: Tue, 27 Feb 2018 09:37:23 -0800 Subject: [PATCH 2/2] journal: make the compression threshold tunable Allow a user to set a number of bytes as Compress to use as the compression threshold. --- man/journald.conf.xml | 11 +++-- src/journal/journald-gperf.gperf | 2 +- src/journal/journald-server.c | 46 ++++++++++++++++++-- src/journal/journald-server.h | 8 +++- src/journal/test-journal-config.c | 71 +++++++++++++++++++++++++++++++ src/test/meson.build | 7 +++ 6 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 src/journal/test-journal-config.c diff --git a/man/journald.conf.xml b/man/journald.conf.xml index 844228e324..aaef49efd5 100644 --- a/man/journald.conf.xml +++ b/man/journald.conf.xml @@ -107,10 +107,13 @@ Compress= - Takes a boolean value. If enabled (the - default), data objects that shall be stored in the journal and - are larger than a certain threshold are compressed before they - are written to the file system. + Can take a boolean value. If enabled (the + default), data objects that shall be stored in the journal + and are larger than the default threshold of 512 bytes are + compressed before they are written to the file system. It + can also be set to a number of bytes to specify the + compression threshold directly. Suffixes like K, M, and G + can be used to specify larger units. diff --git a/src/journal/journald-gperf.gperf b/src/journal/journald-gperf.gperf index e8c1f2e2d5..1adcb50b6d 100644 --- a/src/journal/journald-gperf.gperf +++ b/src/journal/journald-gperf.gperf @@ -19,7 +19,7 @@ struct ConfigPerfItem; %includes %% Journal.Storage, config_parse_storage, 0, offsetof(Server, storage) -Journal.Compress, config_parse_bool, 0, offsetof(Server, compress) +Journal.Compress, config_parse_compress, 0, offsetof(Server, compress) Journal.Seal, config_parse_bool, 0, offsetof(Server, seal) Journal.ReadKMsg, config_parse_bool, 0, offsetof(Server, read_kmsg) Journal.SyncIntervalSec, config_parse_sec, 0, offsetof(Server, sync_interval_usec) diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 570450d321..a5f1315327 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -280,9 +280,12 @@ static int open_journal( assert(ret); if (reliably) - r = journal_file_open_reliably(fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f); + r = journal_file_open_reliably(fname, flags, 0640, s->compress.enabled, s->compress.threshold_bytes, + seal, metrics, s->mmap, s->deferred_closes, NULL, &f); else - r = journal_file_open(-1, fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f); + r = journal_file_open(-1, fname, flags, 0640, s->compress.enabled, s->compress.threshold_bytes, seal, + metrics, s->mmap, s->deferred_closes, NULL, &f); + if (r < 0) return r; @@ -463,7 +466,7 @@ static int do_rotate( if (!*f) return -EINVAL; - r = journal_file_rotate(f, s->compress, (uint64_t) -1, seal, s->deferred_closes); + r = journal_file_rotate(f, s->compress.enabled, s->compress.threshold_bytes, seal, s->deferred_closes); if (r < 0) { if (*f) return log_error_errno(r, "Failed to rotate %s: %m", (*f)->path); @@ -1695,7 +1698,8 @@ int server_init(Server *s) { zero(*s); s->syslog_fd = s->native_fd = s->stdout_fd = s->dev_kmsg_fd = s->audit_fd = s->hostname_fd = s->notify_fd = -1; - s->compress = true; + s->compress.enabled = true; + s->compress.threshold_bytes = (uint64_t) -1; s->seal = true; s->read_kmsg = true; @@ -2038,3 +2042,37 @@ int config_parse_line_max( return 0; } + +int config_parse_compress(const char* unit, + const char *filename, + unsigned line, + const char *section, + unsigned section_line, + const char *lvalue, + int ltype, + const char *rvalue, + void *data, + void *userdata) { + JournalCompressOptions* compress = data; + int r; + + if (streq(rvalue, "1")) { + log_syntax(unit, LOG_WARNING, filename, line, 0, + "Compress= ambiguously specified as 1, enabling compression with default threshold"); + compress->enabled = true; + } else if (streq(rvalue, "0")) { + log_syntax(unit, LOG_WARNING, filename, line, 0, + "Compress= ambiguously specified as 0, disabling compression"); + compress->enabled = false; + } else if ((r = parse_boolean(rvalue)) >= 0) + compress->enabled = r; + else if (parse_size(rvalue, 1024, &compress->threshold_bytes) == 0) + compress->enabled = true; + else if (isempty(rvalue)) { + compress->enabled = true; + compress->threshold_bytes = (uint64_t) -1; + } else + log_syntax(unit, LOG_ERR, filename, line, r, "Failed to parse Compress= value, ignoring: %s", rvalue); + + return 0; +} diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h index bf4ba6897a..505652723d 100644 --- a/src/journal/journald-server.h +++ b/src/journal/journald-server.h @@ -52,6 +52,11 @@ typedef enum SplitMode { _SPLIT_INVALID = -1 } SplitMode; +typedef struct JournalCompressOptions { + bool enabled; + uint64_t threshold_bytes; +} JournalCompressOptions; + typedef struct JournalStorageSpace { usec_t timestamp; @@ -113,7 +118,7 @@ struct Server { JournalStorage runtime_storage; JournalStorage system_storage; - bool compress; + JournalCompressOptions compress; bool seal; bool read_kmsg; @@ -205,6 +210,7 @@ const struct ConfigPerfItem* journald_gperf_lookup(const char *key, GPERF_LEN_TY int config_parse_storage(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); int config_parse_line_max(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); +int config_parse_compress(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); const char *storage_to_string(Storage s) _const_; Storage storage_from_string(const char *s) _pure_; diff --git a/src/journal/test-journal-config.c b/src/journal/test-journal-config.c new file mode 100644 index 0000000000..948739486e --- /dev/null +++ b/src/journal/test-journal-config.c @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +/*** + This file is part of systemd. + + Copyright 2011 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include + +#include "journald-server.h" + +#define _COMPRESS_PARSE_CHECK(str, enab, thresh, varname) \ + do { \ + JournalCompressOptions varname = {true, 111}; \ + config_parse_compress("", "", 0, "", 0, "", 0, str, \ + &varname, NULL); \ + assert_se((enab) == varname.enabled); \ + if (varname.enabled) \ + assert_se((thresh) == varname.threshold_bytes); \ + } while (0) + +#define COMPRESS_PARSE_CHECK(str, enabled, threshold) \ + _COMPRESS_PARSE_CHECK(str, enabled, threshold, conf##__COUNTER__) + +static void test_config_compress(void) { + COMPRESS_PARSE_CHECK("yes", true, 111); + COMPRESS_PARSE_CHECK("no", false, 111); + COMPRESS_PARSE_CHECK("y", true, 111); + COMPRESS_PARSE_CHECK("n", false, 111); + COMPRESS_PARSE_CHECK("true", true, 111); + COMPRESS_PARSE_CHECK("false", false, 111); + COMPRESS_PARSE_CHECK("t", true, 111); + COMPRESS_PARSE_CHECK("f", false, 111); + COMPRESS_PARSE_CHECK("on", true, 111); + COMPRESS_PARSE_CHECK("off", false, 111); + + /* Weird size/bool overlapping case. We preserve backward compatibility instead of assuming these are byte + * counts. */ + COMPRESS_PARSE_CHECK("1", true, 111); + COMPRESS_PARSE_CHECK("0", false, 111); + + /* IEC sizing */ + COMPRESS_PARSE_CHECK("1B", true, 1); + COMPRESS_PARSE_CHECK("1K", true, 1024); + COMPRESS_PARSE_CHECK("1M", true, 1024 * 1024); + COMPRESS_PARSE_CHECK("1G", true, 1024 * 1024 * 1024); + + /* Invalid Case */ + COMPRESS_PARSE_CHECK("-1", true, 111); + COMPRESS_PARSE_CHECK("blah blah", true, 111); + COMPRESS_PARSE_CHECK("", true, (uint64_t)-1); +} + +int main(int argc, char *argv[]) { + test_config_compress(); + + return 0; +} diff --git a/src/test/meson.build b/src/test/meson.build index e45618ef88..205a09d62a 100644 --- a/src/test/meson.build +++ b/src/test/meson.build @@ -725,6 +725,13 @@ tests += [ libxz, liblz4]], + [['src/journal/test-journal-config.c'], + [libjournal_core, + libshared], + [libxz, + liblz4, + libselinux]], + [['src/journal/test-journal-verify.c'], [libjournal_core, libshared],