From ee62e49f329410b13f6ecbf513bab041c4f6dcb4 Mon Sep 17 00:00:00 2001 From: sschindler Date: Wed, 7 Oct 2015 15:33:50 +0200 Subject: [PATCH 1/3] add converting of all data types to journal conform format * some fields in the journal were silently dropped by the api * to not lose information: additional converting * to not oversee the loss: additional logging --- c_src/journald_api.c | 5 ++++- src/journald_api.erl | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/c_src/journald_api.c b/c_src/journald_api.c index 7babc68..e84673a 100644 --- a/c_src/journald_api.c +++ b/c_src/journald_api.c @@ -121,7 +121,10 @@ static ERL_NIF_TERM nif_sendv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ iov[i].iov_len = item.size; i++; } - else len--; // skip bad argument + else { + sd_journal_print(LOG_ERR, "skipped a journal field"); + len--; // skip bad argument + } } while(enif_get_list_cell(env, tail, &buffer, &tail)); diff --git a/src/journald_api.erl b/src/journald_api.erl index b8bef73..816d229 100644 --- a/src/journald_api.erl +++ b/src/journald_api.erl @@ -86,11 +86,43 @@ list_conversion([_|T]) -> % skip bad argument list_conversion(T); list_conversion(_) -> []. +% convert all data types accordingly +% (see http://erlang.org/doc/reference_manual/data_types.html) +% ------------------------------------------------------------ +% Number to_list(V) when is_integer(V) -> integer_to_list(V); to_list(V) when is_float(V) -> float_to_list(V); +% Atom to_list(V) when is_atom(V) -> atom_to_binary(V, utf8); +% Binary +% -- will be sent through without conversion +% Fun +to_list(V) when is_function(V)-> io_lib:format("~p", [V]); +% Port Identifier +to_list(V) when is_port(V) -> io_lib:format("~p", [V]); +% Pid to_list(V) when is_pid(V) -> pid_to_list(V); -to_list(V) -> V. +% Tuple +to_list(V) when is_tuple(V) -> io_lib:format("~p", [V]); +% Map +to_list(V) when is_map(V) -> io_lib:format("~p", [V]); +% List +to_list(V) when is_list(V) -> list_to_string(V); +% String +% -- same as list +% Record +% -- same as tuple +% Boolean +% -- same as atoms +% anything else will given plainly to the journal api +to_list(V) -> V. + +list_to_string (V) -> + try io_lib:format('~s', [V]) + catch _:_ -> + io_lib:format('~p', [V]) + end. + load_nif() -> Dir = "priv", From 9768415a2b80e3e4e039752902d479caf6a11623 Mon Sep 17 00:00:00 2001 From: sschindler Date: Wed, 7 Oct 2015 15:34:47 +0200 Subject: [PATCH 2/3] cosmetic changes * purged emtpy spaces --- c_src/journald_api.c | 152 +++++++++++++++++++++---------------------- src/journald_api.erl | 36 +++++----- 2 files changed, 94 insertions(+), 94 deletions(-) diff --git a/c_src/journald_api.c b/c_src/journald_api.c index e84673a..ec2f8d1 100644 --- a/c_src/journald_api.c +++ b/c_src/journald_api.c @@ -1,4 +1,4 @@ -/* +/* Copyright 2010-2013, Travelping GmbH Permission is hereby granted, free of charge, to any person obtaining a @@ -35,11 +35,11 @@ #include //A pointer to the head of the journal will be stored in journal_container. -//journal_container_type is just the ErlNifResourceType for journal_container +//journal_container_type is just the ErlNifResourceType for journal_container //to allocate it later in "open" as this type. static ErlNifResourceType *journal_container_type = NULL; -typedef struct { +typedef struct { sd_journal *journal_pointer; ErlNifTid tid; ErlNifPid pid; @@ -90,7 +90,7 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) ErlNifResourceType *rt = enif_open_resource_type(env, NULL, "journal_container_type", dtor_jc, ERL_NIF_RT_CREATE, NULL); if (rt == NULL) return -1; journal_container_type = rt; - + atom_ok = enif_make_atom(env, "ok"); atom_error = enif_make_atom(env, "error"); atom_no_more = enif_make_atom(env, "no_more"); @@ -99,25 +99,25 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) return 0; } -/* Sendv - convert every parameter of the list to binary and copy that into a iovec and call sd_journal_sendv(2) +/* Sendv + convert every parameter of the list to binary and copy that into a iovec and call sd_journal_sendv(2) */ static ERL_NIF_TERM nif_sendv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ +{ unsigned int len=0, i=0; - struct iovec *iov = NULL; + struct iovec *iov = NULL; ErlNifBinary item; ERL_NIF_TERM buffer, tail; - - if (!enif_get_list_cell(env, argv[0], &buffer, &tail) + + if (!enif_get_list_cell(env, argv[0], &buffer, &tail) || !enif_get_list_length(env, argv[0], &len) || len==0) return enif_make_badarg(env); iov = alloca(len * sizeof(struct iovec)); do { - if (enif_inspect_iolist_as_binary(env, buffer, &item)) //binary conversion from list element - { - iov[i].iov_base = item.data; //copy process into iovec + if (enif_inspect_iolist_as_binary(env, buffer, &item)) //binary conversion from list element + { + iov[i].iov_base = item.data; //copy process into iovec iov[i].iov_len = item.size; i++; } @@ -128,21 +128,21 @@ static ERL_NIF_TERM nif_sendv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ } while(enif_get_list_cell(env, tail, &buffer, &tail)); - if (sd_journal_sendv(iov, len) == 0) + if (sd_journal_sendv(iov, len) == 0) return atom_ok; - - return atom_error; + + return atom_error; } /* convert the first parameter to binary and the secound and third to int and call sd_journal_stream_fd(3) */ -static ERL_NIF_TERM nif_stream_fd(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static ERL_NIF_TERM nif_stream_fd(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { int priority = -1, level_prefix = -1; int fd; ErlNifBinary syslog_id; - + if (!enif_inspect_binary(env, argv[0], &syslog_id) || !enif_get_int(env, argv[1], &priority) || !enif_get_int(env, argv[2], &level_prefix)) @@ -154,21 +154,21 @@ static ERL_NIF_TERM nif_stream_fd(ErlNifEnv* env, int argc, const ERL_NIF_TERM a ERL_NIF_TERM nif_fd = enif_make_int(env, fd); - return nif_fd; + return nif_fd; } static ERL_NIF_TERM nif_write_fd(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { int fd; ErlNifBinary msg; - + if (!enif_get_int(env, argv[0], (void*) &fd) || !enif_inspect_iolist_as_binary(env, argv[1], &msg)) return enif_make_badarg(env); - + if (write(fd, msg.data, msg.size) < 0) return atom_error; - + return atom_ok; } @@ -180,13 +180,13 @@ static ERL_NIF_TERM nif_close_fd(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar return enif_make_badarg(env); if (close(fd) < 0) return atom_error; - + return atom_ok; } /* - Opens the journal and stores a pointer in a journal_container. To share the pointer - to the journal with the calling Erlang module it is made a resource via + Opens the journal and stores a pointer in a journal_container. To share the pointer + to the journal with the calling Erlang module it is made a resource via enif_make_resource. */ static ERL_NIF_TERM nif_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]){ @@ -196,39 +196,39 @@ static ERL_NIF_TERM nif_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] jc = enif_alloc_resource(journal_container_type, sizeof(journal_container)); int r = sd_journal_open(&(jc->journal_pointer), SD_JOURNAL_LOCAL_ONLY); - if (r < 0) + if (r < 0) return return_error(env, r); jc->notifier_used=0; - + ERL_NIF_TERM nif_journal_pointer = enif_make_resource(env, jc); enif_release_resource(jc); - + return enif_make_tuple2(env, atom_ok, nif_journal_pointer); - + } static ERL_NIF_TERM nif_open_directory(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]){ journal_container *jc; ErlNifBinary dir; - + if (!enif_inspect_binary(env, argv[0], &dir)) return enif_make_badarg(env); jc = enif_alloc_resource(journal_container_type, sizeof(journal_container)); - + int r = sd_journal_open_directory(&(jc->journal_pointer), (const char*) dir.data, 0); if (r < 0) return return_error(env, r); - + jc->notifier_used=0; ERL_NIF_TERM nif_journal_pointer = enif_make_resource(env, jc); enif_release_resource(jc); - + return enif_make_tuple2(env, atom_ok, nif_journal_pointer); - + } @@ -242,7 +242,7 @@ static ERL_NIF_TERM nif_close(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ if (!enif_get_resource(env, argv[0], journal_container_type, (void **) &jc)) return enif_make_badarg(env); - + close_journal_container(jc); return atom_ok; @@ -258,13 +258,13 @@ static ERL_NIF_TERM nif_next(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] journal_container *jc; enif_get_resource(env, argv[0], journal_container_type, (void **) &jc); - - int r = sd_journal_next(jc->journal_pointer); - if (r < 0) + + int r = sd_journal_next(jc->journal_pointer); + if (r < 0) return return_error(env, r); - if (r == 0) + if (r == 0) return atom_no_more; - else + else return atom_ok; } @@ -273,19 +273,19 @@ static ERL_NIF_TERM nif_previous(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar journal_container *jc; enif_get_resource(env, argv[0], journal_container_type, (void **) &jc); - + int r = sd_journal_previous(jc->journal_pointer); - if (r < 0) + if (r < 0) return return_error(env, r); - if (r == 0) + if (r == 0) return atom_no_more; - else + else return atom_ok; } /* gets the entries of the current head position (according to given specifications - made by e.g. argv[1], add_match...). There are two arguments. First the journal + made by e.g. argv[1], add_match...). There are two arguments. First the journal pointer, second the entry-string which should be matched. */ static ERL_NIF_TERM nif_get_data(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]){ @@ -338,9 +338,9 @@ static ERL_NIF_TERM nif_flush_matches(ErlNifEnv* env, int argc, const ERL_NIF_TE if (!enif_get_resource(env, argv[0], journal_container_type, (void **) &jc)) return enif_make_badarg(env); - + sd_journal_flush_matches(jc->journal_pointer); - + return atom_ok; } @@ -351,7 +351,7 @@ static ERL_NIF_TERM nif_add_disjunction(ErlNifEnv* env, int argc, const ERL_NIF_ if (!enif_get_resource(env, argv[0], journal_container_type, (void **) &jc)) return enif_make_badarg(env); - + int r = sd_journal_add_disjunction(jc->journal_pointer); if (r < 0) return return_error(env, r); @@ -365,8 +365,8 @@ static ERL_NIF_TERM nif_add_conjunction(ErlNifEnv* env, int argc, const ERL_NIF_ if (!enif_get_resource(env, argv[0], journal_container_type, (void **) &jc)) return enif_make_badarg(env); - - int r = sd_journal_add_conjunction(jc->journal_pointer); + + int r = sd_journal_add_conjunction(jc->journal_pointer); if (r < 0) return return_error(env, r); @@ -379,8 +379,8 @@ static ERL_NIF_TERM nif_seek_head(ErlNifEnv* env, int argc, const ERL_NIF_TERM a if (!enif_get_resource(env, argv[0], journal_container_type, (void **) &jc)) return enif_make_badarg(env); - - int r = sd_journal_seek_head(jc->journal_pointer); + + int r = sd_journal_seek_head(jc->journal_pointer); if (r < 0) return return_error(env, r); @@ -393,8 +393,8 @@ static ERL_NIF_TERM nif_seek_tail(ErlNifEnv* env, int argc, const ERL_NIF_TERM a if (!enif_get_resource(env, argv[0], journal_container_type, (void **) &jc)) return enif_make_badarg(env); - - int r = sd_journal_seek_tail(jc->journal_pointer); + + int r = sd_journal_seek_tail(jc->journal_pointer); if (r < 0) return return_error(env, r); @@ -408,9 +408,9 @@ static ERL_NIF_TERM nif_get_cursor(ErlNifEnv* env, int argc, const ERL_NIF_TERM if (!enif_get_resource(env, argv[0], journal_container_type, (void **) &jc)) return enif_make_badarg(env); - + char *cursor; - int r = sd_journal_get_cursor(jc->journal_pointer, &cursor); + int r = sd_journal_get_cursor(jc->journal_pointer, &cursor); if (r < 0) return return_error(env, r); @@ -432,9 +432,9 @@ static ERL_NIF_TERM nif_test_cursor(ErlNifEnv* env, int argc, const ERL_NIF_TERM return enif_make_badarg(env); int r = sd_journal_test_cursor(jc->journal_pointer, (const char *) p.data); - if (r < 0) + if (r < 0) return return_error(env, r); - else if (r == 0) + else if (r == 0) return atom_no_match; else return atom_ok; } @@ -484,11 +484,11 @@ static ERL_NIF_TERM nif_enumerate_unique(ErlNifEnv* env, int argc, const ERL_NIF size_t l; r = sd_journal_enumerate_unique(jc->journal_pointer, (void *) &d, &l); - if ( r < 0) + if ( r < 0) return return_error(env, r); - else if (r == 0) + else if (r == 0) return atom_no_more; - + memcpy(enif_make_new_binary(env, l, &ret), d, l); return enif_make_tuple2(env, atom_ok, ret); @@ -501,7 +501,7 @@ static ERL_NIF_TERM nif_restart_unique(ErlNifEnv* env, int argc, const ERL_NIF_T if (!enif_get_resource(env, argv[0], journal_container_type, (void **) &jc)) return enif_make_badarg(env); - sd_journal_restart_unique(jc->journal_pointer); + sd_journal_restart_unique(jc->journal_pointer); return atom_ok; } @@ -519,11 +519,11 @@ static ERL_NIF_TERM nif_enumerate_data(ErlNifEnv* env, int argc, const ERL_NIF_T size_t l; r = sd_journal_enumerate_data(jc->journal_pointer, (void *) &d, &l); - if ( r < 0) + if ( r < 0) return return_error(env, r); - else if (r == 0) + else if (r == 0) return atom_no_more; - + memcpy(enif_make_new_binary(env, l, &ret), d, l); return enif_make_tuple2(env, atom_ok, ret); @@ -535,8 +535,8 @@ static ERL_NIF_TERM nif_restart_data(ErlNifEnv* env, int argc, const ERL_NIF_TER if (!enif_get_resource(env, argv[0], journal_container_type, (void **) &jc)) return enif_make_badarg(env); - - sd_journal_restart_data(jc->journal_pointer); + + sd_journal_restart_data(jc->journal_pointer); return atom_ok; } @@ -546,12 +546,12 @@ static void * notifier_run(void *arg){ journal_container *jc = (journal_container *) arg; ErlNifEnv *t_env; int changes; - + changes = sd_journal_wait(jc->journal_pointer, (uint64_t) 1000); while(1){ if ( jc->notifier_flag == 1 ){ - jc->notifier_flag = 0; + jc->notifier_flag = 0; enif_thread_exit(NULL); } @@ -597,7 +597,7 @@ static ERL_NIF_TERM nif_open_notifier (ErlNifEnv* env, int argc, const ERL_NIF_T jc->pid = pid; jc->notifier_flag = 0; - + int r = enif_thread_create("notifier_worker", &jc->tid, notifier_run, (void *) jc, NULL); if (r != 0) return return_error(env, r); @@ -634,10 +634,10 @@ static ERL_NIF_TERM nif_get_realtime_usec (ErlNifEnv *env, int argc, const ERL_N return enif_make_badarg(env); r = sd_journal_get_realtime_usec(jc->journal_pointer, &usec); - if (r < 0 ) + if (r < 0 ) return return_error(env, r); - return enif_make_tuple2(env, atom_ok, enif_make_uint64(env, usec)); + return enif_make_tuple2(env, atom_ok, enif_make_uint64(env, usec)); } static ERL_NIF_TERM nif_seek_realtime_usec (ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]){ @@ -645,20 +645,20 @@ static ERL_NIF_TERM nif_seek_realtime_usec (ErlNifEnv *env, int argc, const ERL_ journal_container *jc; uint64_t usec; int r; - + if (!enif_get_resource(env, argv[0], journal_container_type, (void **) &jc) || !enif_get_uint64(env, argv[1], &usec)) return enif_make_badarg(env); r = sd_journal_seek_realtime_usec(jc->journal_pointer, usec); - if (r < 0) + if (r < 0) return return_error(env, r); - + return atom_ok; } static ErlNifFunc nif_funcs[] = -{ +{ {"sendv_nif", 1, nif_sendv}, {"stream_fd", 3, nif_stream_fd}, {"write_fd", 2, nif_write_fd}, diff --git a/src/journald_api.erl b/src/journald_api.erl index 816d229..aebaf75 100644 --- a/src/journald_api.erl +++ b/src/journald_api.erl @@ -25,9 +25,9 @@ %% External API -export([sendv/1, stream_fd/3, write_fd/2, close_fd/1, - open/0, open_directory/1, close/1, next/1, - previous/1, get_data/2, add_match/2, - add_disjunction/1, add_conjunction/1, flush_matches/1, seek_head/1, seek_tail/1, + open/0, open_directory/1, close/1, next/1, + previous/1, get_data/2, add_match/2, + add_disjunction/1, add_conjunction/1, flush_matches/1, seek_head/1, seek_tail/1, get_cursor/1, test_cursor/2, seek_cursor/2, query_unique/2, enumerate_unique/1, restart_unique/1, open_notifier/2, enumerate_data/1, restart_data/1, close_notifier/1, @@ -54,16 +54,16 @@ write_fd(_Fd, _Msg) -> ?nif_stub. close_fd(_Fd) -> ?nif_stub. open() -> ?nif_stub. open_directory(_Arg) -> ?nif_stub. -close(_Arg) -> ?nif_stub. +close(_Arg) -> ?nif_stub. next(_Arg) -> ?nif_stub. -previous(_Arg) -> ?nif_stub. -get_data(_Arg1, _Arg2) -> ?nif_stub. -add_match(_Arg1, _Arg2) -> ?nif_stub. -add_disjunction(_Arg) -> ?nif_stub. -add_conjunction(_Arg) -> ?nif_stub. -flush_matches(_Arg) -> ?nif_stub. -seek_head(_Arg) -> ?nif_stub. -seek_tail(_Arg) -> ?nif_stub. +previous(_Arg) -> ?nif_stub. +get_data(_Arg1, _Arg2) -> ?nif_stub. +add_match(_Arg1, _Arg2) -> ?nif_stub. +add_disjunction(_Arg) -> ?nif_stub. +add_conjunction(_Arg) -> ?nif_stub. +flush_matches(_Arg) -> ?nif_stub. +seek_head(_Arg) -> ?nif_stub. +seek_tail(_Arg) -> ?nif_stub. get_cursor(_Arg) -> ?nif_stub. test_cursor(_Arg1, _Arg2) -> ?nif_stub. seek_cursor(_Arg1, _Arg2) -> ?nif_stub. @@ -81,10 +81,10 @@ seek_realtime_usec(_Journal, _Usec) -> ?nif_stub. %% -- helpers list_conversion([]) -> []; list_conversion([{E,V}|T]) -> - [[E, $=, to_list(V)] | list_conversion(T)]; + [[E, $=, to_list(V)] | list_conversion(T)]; list_conversion([_|T]) -> % skip bad argument list_conversion(T); -list_conversion(_) -> []. +list_conversion(_) -> []. % convert all data types accordingly % (see http://erlang.org/doc/reference_manual/data_types.html) @@ -126,9 +126,9 @@ list_to_string (V) -> load_nif() -> Dir = "priv", - PrivDir = case code:priv_dir(ejournald) of % check existence of priv folder - {error, _} -> Dir; + PrivDir = case code:priv_dir(ejournald) of % check existence of priv folder + {error, _} -> Dir; X -> X end, - Lib = filename:join(PrivDir, "journald_api"), % create priv path so journald_api.so - erlang:load_nif(Lib, 0). % load NIF + Lib = filename:join(PrivDir, "journald_api"), % create priv path so journald_api.so + erlang:load_nif(Lib, 0). % load NIF From 3da3a027b07baa9adaaba62da75ae91bfceaa370 Mon Sep 17 00:00:00 2001 From: sschindler Date: Wed, 7 Oct 2015 17:46:07 +0200 Subject: [PATCH 3/3] add testing against all field types to common test suite --- test/ejournald_SUITE.erl | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/test/ejournald_SUITE.erl b/test/ejournald_SUITE.erl index 7e4e8a0..fab22ce 100644 --- a/test/ejournald_SUITE.erl +++ b/test/ejournald_SUITE.erl @@ -28,7 +28,7 @@ -define(CONTROL, {control, [], [start_stop_io, start_stop_reader]}). -define(READ, {read, [], [read_last_3_logs, read_last_3_messages, read_since, read_since_until, notify]}). --define(WRITE, {write, [], [io_put_chars, io_format, io_write, io_fwrite]}). +-define(WRITE, {write, [], [io_put_chars, io_format, io_write, io_fwrite, io_fieldtypes]}). %% ---------------------------------------------------------------------------------------------------- %% -- testcases CONTROL @@ -42,7 +42,7 @@ start_stop_reader(_Config) -> %% ---------------------------------------------------------------------------------------------------- %% -- testcases WRITING -io_put_chars(_Config) -> +io_put_chars(_Config) -> ok = io:put_chars(?IO_SERVER, "put_chars1"), ok = io:put_chars(?IO_SERVER, " put_chars2"), ok = io:put_chars(?IO_SERVER, " written by io:put_char()"), @@ -62,6 +62,24 @@ io_fwrite(_Config) -> ok = io:fwrite(?IO_SERVER, "|~10.5c|~-10.5c|~5c|~n", [$a, $b, $c]), ok = io:nl(?IO_SERVER). +io_fieldtypes(_Config) -> + Fun = fun (X) -> X+1 end, + Atom = testatom, + Pid = spawn(m, f, []), + Tup = {adam,24,{july,29}}, + Map = #{name=>adam,age=>24,date=>{july,29}}, + List = [a,2,{c,4}], + Str = "teststring", + journald_api:sendv([{"MESSAGE", "testmessage"}, + {"TEST_ATOM", Atom}, + {"TEST_FUNCTION", Fun}, + {"TEST_PID", Pid}, + {"TEST_TUPLE", Tup}, + {"TEST_MAP", Map}, + {"TEST_LIST", List}, + {"TEST_STRING", Str} + ]). + %% ---------------------------------------------------------------------------------------------------- %% -- testcases READING read_last_3_logs(_Config) -> @@ -108,7 +126,7 @@ all() -> [{group, control}, {group, write}, {group, read}]. init_per_suite(Config) -> application:start(ejournald), Config. - + end_per_suite(_Config) -> application:stop(ejournald). @@ -121,7 +139,7 @@ mod(0,_Y) -> 0. receive_flush(0) -> ok; receive_flush(N) -> receive - {_Timestamp, _Priority, Log} -> + {_Timestamp, _Priority, Log} -> ct:log(binary_to_list(Log)), receive_flush(N-1); journal_invalidate -> @@ -129,4 +147,4 @@ receive_flush(N) -> after 500 -> erlang:error(timeout) - end. \ No newline at end of file + end.