[openrtm-commit:02392] r2925 - in trunk/OpenRTM-aist/src/ext: . logger logger/fluentbit_stream
openrtm ＠ openrtm.org
openrtm ＠ openrtm.org
2017年 2月 7日 (火) 04:43:55 JST
Author: n-ando
Date: 2017-02-07 04:43:54 +0900 (Tue, 07 Feb 2017)
New Revision: 2925
Added:
trunk/OpenRTM-aist/src/ext/logger/
trunk/OpenRTM-aist/src/ext/logger/Makefile.am
trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/
trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/FluentBit.cpp
trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/FluentBit.h
trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/Makefile.am
trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/fluentbit.conf
Log:
[incompat,new func] fluent-bit logger stream has been added. refs #3567
Added: trunk/OpenRTM-aist/src/ext/logger/Makefile.am
===================================================================
--- trunk/OpenRTM-aist/src/ext/logger/Makefile.am (rev 0)
+++ trunk/OpenRTM-aist/src/ext/logger/Makefile.am 2017-02-06 19:43:54 UTC (rev 2925)
@@ -0,0 +1,10 @@
+## -*- Makefile -*-
+##---------------------------------------------------------------------------
+## Makefile.am for external libraries
+##
+## $Id$
+##---------------------------------------------------------------------------
+
+AUTOMAKE_OPTIONS = 1.4
+
+SUBDIRS = fluentbit_stream
Added: trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/FluentBit.cpp
===================================================================
--- trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/FluentBit.cpp (rev 0)
+++ trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/FluentBit.cpp 2017-02-06 19:43:54 UTC (rev 2925)
@@ -0,0 +1,177 @@
+// -*- C++ -*-
+/*!
+ * @file LogstreamBase.h
+ * @brief Logger stream buffer base class
+ * @date $Date$
+ * @author Noriaki Ando <n-ando at aist.go.jp>
+ *
+ * Copyright (C) 2017
+ * Noriaki Ando
+ * National Institute of
+ * Advanced Industrial Science and Technology (AIST), Japan
+ * All rights reserved.
+ *
+ * $Id$
+ *
+ */
+#include <algorithm>
+
+#include <rtm/LogstreamBase.h>
+#include <coil/stringutil.h>
+#include "FluentBit.h"
+
+namespace RTC
+{
+ // Static variables initialization
+ flb_ctx_t* FluentBitStream::s_flbContext = NULL;
+ int FluentBitStream::s_instance = 0;
+
+ //============================================================
+ // FluentBitStream class
+ FluentBitStream::FluentBitStream()
+ : m_pos(0)
+ {
+ for (size_t i(0); i < BUFFER_LEN; ++i) { m_buf[i] = '\0'; }
+ if (s_flbContext == NULL)
+ {
+ s_flbContext = flb_create();
+ }
+ }
+
+ FluentBitStream::~FluentBitStream()
+ {
+ --s_instance;
+ if (s_instance == 0)
+ {
+ flb_stop(s_flbContext);
+ flb_destroy(s_flbContext);
+ }
+ }
+
+ bool FluentBitStream::init(const coil::Properties& prop)
+ {
+ flb_stop(s_flbContext);
+
+ // Default lib-input setting
+ FlbHandler flbhandler = flb_input(s_flbContext, (char*)"lib", NULL);
+ flb_input_set(s_flbContext, flbhandler, "tag", prop.getName(), NULL);
+ m_flbIn.push_back(flbhandler);
+
+ const std::vector<coil::Properties*>& leaf(prop.getLeaf());
+
+ for (size_t i(0); i < leaf.size(); ++i)
+ {
+ std::string key(leaf[i]->getName());
+ if (key.find("output") != std::string::npos)
+ {
+ createOutputStream(*leaf[i]);
+ }
+ else if (key.find("input") != std::string::npos)
+ {
+ createInputStream(*leaf[i]);
+ }
+ }
+ // Start the background worker
+ flb_start(s_flbContext);
+ return true;
+ }
+
+ bool FluentBitStream::createOutputStream(const coil::Properties& prop)
+ {
+ std::string plugin = prop["plugin"];
+ FlbHandler flbout = flb_output(s_flbContext,
+ (char*)plugin.c_str(), NULL);
+ m_flbOut.push_back(flbout);
+ const std::vector<Properties*>& leaf = prop.getLeaf();
+ for (size_t i(0); i < leaf.size(); ++i)
+ {
+ flb_output_set(s_flbContext, flbout,
+ leaf[i]->getName(), leaf[i]->getValue(), NULL);
+ }
+ return true;
+ }
+
+ bool FluentBitStream::createInputStream(const coil::Properties& prop)
+ {
+ std::string plugin = prop["plugin"];
+ FlbHandler flbin = flb_input(s_flbContext,
+ (char*)plugin.c_str(), NULL);
+ m_flbIn.push_back(flbin);
+ const std::vector<Properties*>& leaf = prop.getLeaf();
+ for (size_t i(0); i < leaf.size(); ++i)
+ {
+ flb_input_set(s_flbContext, flbin,
+ leaf[i]->getName(), leaf[i]->getValue(), NULL);
+ }
+ return true;
+ }
+
+ std::streamsize FluentBitStream::pushLogger()
+ {
+ char tmp[BUFFER_LEN];
+ std::streamsize n(0);
+ for (size_t i(0); i < m_flbIn.size(); ++i)
+ {
+ n = snprintf(tmp, sizeof(tmp) - 1,
+ "[%lu, {\"msg\": \"%s\"}]",
+ time(NULL), m_buf);
+ flb_lib_push(s_flbContext, m_flbIn[i], tmp, n);
+ }
+ return n;
+ }
+
+ std::streamsize FluentBitStream::xsputn(const char_type* str,
+ std::streamsize insize)
+ {
+ std::streamsize outsize(insize);
+ for (std::streamsize i(0); i < insize; ++i, ++m_pos)
+ {
+ m_buf[m_pos] = str[i];
+ if (str[i] == '\n' || str[i] == '\r' || m_pos == (BUFFER_LEN - 1))
+ {
+ m_buf[m_pos] = '\0';
+ outsize = pushLogger();
+ m_pos = 0;
+ }
+ }
+ return outsize;
+ }
+ // end of FluentBitStream class
+ //============================================================
+
+ //============================================================
+ // FluentBit class
+ FluentBit::FluentBit()
+ {
+ }
+
+ FluentBit::~FluentBit()
+ {
+ }
+
+ bool FluentBit::init(const coil::Properties& prop)
+ {
+ return m_fbstream.init(prop);
+ }
+
+ StreambufType* FluentBit::getStreamBuffer()
+ {
+ return &m_fbstream;
+ }
+ // end of FluentBit class
+ //============================================================
+};
+
+extern "C"
+{
+ void FluentBitInit()
+ {
+ ::RTC::LogstreamFactory::
+ instance().addFactory("fluentd",
+ ::coil::Creator< ::RTC::LogstreamBase,
+ ::RTC::FluentBit>,
+ ::coil::Destructor< ::RTC::LogstreamBase,
+ ::RTC::FluentBit>);
+ }
+};
+
Property changes on: trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/FluentBit.cpp
___________________________________________________________________
Added: svn:executable
+ *
Added: trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/FluentBit.h
===================================================================
--- trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/FluentBit.h (rev 0)
+++ trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/FluentBit.h 2017-02-06 19:43:54 UTC (rev 2925)
@@ -0,0 +1,302 @@
+// -*- C++ -*-
+/*!
+ * @file FluentBit.h
+ * @brief File logger stream class
+ * @date $Date$
+ * @author Noriaki Ando <n-ando at aist.go.jp>
+ *
+ * Copyright (C) 2017
+ * Noriaki Ando
+ * National Institute of
+ * Advanced Industrial Science and Technology (AIST), Japan
+ * All rights reserved.
+ *
+ * $Id$
+ *
+ */
+
+#ifndef RTC_LOGGER_FLUENTBIT_H
+#define RTC_LOGGER_FLUENTBIT_H
+
+#include <string>
+#include <fstream>
+#include <iostream>
+#include <fluent-bit.h>
+
+#include <coil/stringutil.h>
+#include <rtm/LogstreamBase.h>
+
+#ifndef LINE_MAX
+#define LINE_MAX 1024
+#endif
+
+#ifndef BUFFER_LEN
+#define BUFFER_LEN LINE_MAX
+#endif
+
+namespace RTC
+{
+ /*!
+ * @if jp
+ * @class FluentbitStream
+ *
+ * このクラスは ログ出力を fluent-bit へ送信するための logstream_buf
+ * クラスである。FluentBit クラスにてインスタンス化され、
+ * SystemLogger の logstream に追加される。
+ *
+ * @else
+ * @class FluentbitStream
+ *
+ * This class is a logstream_buf class to send log output to
+ * fluent-bit. This is instantiate in the FluentBit class and it is
+ * added to SystemLogger's logstreams.
+ *
+ * @endif
+ */
+ class FluentBitStream
+ : public StreambufType
+ {
+ public:
+ FluentBitStream();
+
+ virtual ~FluentBitStream();
+
+ bool init(const coil::Properties& prop);
+
+ bool createOutputStream(const coil::Properties& prop);
+
+ bool createInputStream(const coil::Properties& prop);
+
+ protected:
+ std::streamsize pushLogger();
+
+ virtual std::streamsize xsputn(const char_type* str,
+ std::streamsize insize);
+
+ private:
+ char m_buf[BUFFER_LEN];
+ size_t m_pos;
+
+ typedef int FlbHandler;
+ FlbHandler m_fibInLib;
+
+ std::vector<FlbHandler> m_flbIn;
+ std::vector<FlbHandler> m_flbOut;
+
+ // Static variables
+ /*!
+ * @brief Fluent-bit context
+ */
+ static flb_ctx_t* s_flbContext;
+ /*!
+ * @brief Fluent-bit instance count
+ */
+ static int s_instance;
+ };
+
+ /*!
+ * @if jp
+ * @class FluentBit
+ *
+ * このクラスは ログ出力を fluent-bit へ送信するためのログストリーム
+ * 用プラグインクラスである。
+ *
+ * fluent-bit はログ収集・分配ミドルウェア fluentd のC言語実装である。
+ * fluent-bit/fluentd は様々なプロトコルでログの受信、フィルタリング、
+ * 送信を行うことができる。このクラスは、ログストリームのプラグインを
+ * 構成する FluentBit クラスの std::stream_buff クラスのサブクラスで
+ * あり、実際の FluentBit へのログの出力部分を担うクラスである。
+ *
+ * デフォルトでは、OpenRTMのログ出力を入力 (input) として取り、
+ * rtc.conf に設定された出力 (output) に対してログを送出することがで
+ * きる。input も fluent-bit で利用できるプラグインを rtc.conf から有
+ * 効にすることができ、他の fluentd/fluent-bit からのログ出力を受信し
+ * たり、CPUやメモリ使用量などをログ入力として取得することも可能であ
+ * る。実質的に、コマンドラインプログラムの fluent-bit とほぼ同じこと
+ * が実現可能になっている。
+ *
+ * オプションは、基本的には fluent-bit の key-value 型のプロパティを
+ * rtc.conf で指定することですべてのプラグインを利用できるが、以下に、
+ * 代表的なプラグインとそのオプションを示す。
+ *
+ * * Available Output plugins
+ * - reference: http://fluentbit.io/documentation/0.8/output/index.html
+ *
+ * ** forward: fluentd forwarding
+ * ______________________________________________________________________
+ * | key | Description | Default |
+ * ----------------------------------------------------------------------
+ * | host | Target host where Fluent-Bit or Fluentd are | 127.0.0.1 |
+ * | | listening for Forward messages. | |
+ * ----------------------------------------------------------------------
+ * | port | TCP port of the target service. | 24224 |
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ * Example:
+ * logger.logstream.fluentd.output0.plugin: forward
+ * logger.logstream.fluentd.output0.tag: <tagname>
+ * logger.logstream.fluentd.output0.host: <fluentd_hostname>
+ * logger.logstream.fluentd.output0.port: <fluentd_port>
+ *
+ * ** es: Elasticsearch
+ * ______________________________________________________________________
+ * | key | Description | Default |
+ * ----------------------------------------------------------------------
+ * | host | IP address or hostname of the target | 127.0.0.1 |
+ * | | Elasticsearch instance. | |
+ * ----------------------------------------------------------------------
+ * | port | TCP port of the target Elasticsearch | 9200 |
+ * | | instance. | |
+ * ----------------------------------------------------------------------
+ * | index | Elastic index. | fluentbit |
+ * ----------------------------------------------------------------------
+ * | type | Elastic type. | test |
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * Example:
+ * logger.logstream.fluentd.output0.plugin: es
+ * logger.logstream.fluentd.output0.tag: <tagname>
+ * logger.logstream.fluentd.output0.host: <es_hostname>
+ * logger.logstream.fluentd.output0.port: <es_port>
+ * logger.logstream.fluentd.output0.index: <es_index>
+ * logger.logstream.fluentd.output0.type: <es_type>
+ *
+ * ** http: HTTP POST request in MessagePack format
+ * ______________________________________________________________________
+ * | key | Description | Default |
+ * ----------------------------------------------------------------------
+ * | Host | IP address or hostname of the target HTTP | 127.0.0.1 |
+ * | | Server. | |
+ * ----------------------------------------------------------------------
+ * | Port | TCP port of the target HTTP Server. | 80 |
+ * ----------------------------------------------------------------------
+ * | Proxy | Specify an HTTP Proxy. The expected format | |
+ * | | of this value is http://host:port. | |
+ * | | Note that https is not supported yet. | |
+ * ----------------------------------------------------------------------
+ * | URI | Specify an optional HTTP URI for the target | / |
+ * | | web server, e.g: /something | |
+ * ----------------------------------------------------------------------
+ * | Format | Specify the data format to be used in the | msgpack |
+ * | | HTTP request body, by default it uses | |
+ * | | msgpack, optionally it can be set to json. | |
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * Example:
+ * logger.logstream.fluentd.output0.plugin: http
+ * logger.logstream.fluentd.output0.tag: <tagname>
+ * logger.logstream.fluentd.output0.host: 127.0.0.1
+ * logger.logstream.fluentd.output0.port: 80
+ * logger.logstream.fluentd.output0.proxy:
+ * logger.logstream.fluentd.output0.uri: /openrtm/
+ * logger.logstream.fluentd.output0.format: msgpack
+ *
+ * ** nats: NATS output plugin
+ * ______________________________________________________________________
+ * | key | Description | Default |
+ * ----------------------------------------------------------------------
+ * | host | IP address or hostname of the NATS Server. | 127.0.0.1 |
+ * ----------------------------------------------------------------------
+ * | port | TCP port of the target NATS Server. | 4222 |
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * Example:
+ * logger.logstream.fluentd.output0.plugin: nats
+ * logger.logstream.fluentd.output0.tag: <tagname>
+ * logger.logstream.fluentd.output0.host: <nats_host>
+ * logger.logstream.fluentd.output0.port: <nats_port>
+ *
+ * * stdout: Standard Output plugin
+ *
+ * Example:
+ * logger.logstream.fluentd.output0.plugin: stdin
+ *
+ * @else
+ * @class FluentbitStream
+ *
+ * @endif
+ */
+ class FluentBit
+ : public LogstreamBase
+ {
+ public:
+ /*!
+ * @if jp
+ *
+ * @brief コンストラクタ
+ *
+ * @else
+ *
+ * @brief Constructor
+ *
+ * @endif
+ */
+ FluentBit();
+
+ /*!
+ * @if jp
+ *
+ * @brief デストラクタ
+ *
+ * @else
+ *
+ * @brief Destructor
+ *
+ * @endif
+ */
+ virtual ~FluentBit(void);
+
+ /*!
+ * @if jp
+ * @brief 設定初期化
+ *
+ * Logstreamクラスの各種設定を行う。実装クラスでは、与えられた
+ * Propertiesから必要な情報を取得して各種設定を行う。
+ *
+ * @param prop 設定情報
+ *
+ * @else
+ *
+ * @brief Initializing configuration
+ *
+ * This operation would be called to configure in initialization.
+ * In the concrete class, configuration should be performed
+ * getting appropriate information from the given Properties data.
+ *
+ * @param prop Configuration information
+ *
+ * @endif
+ */
+ virtual bool init(const coil::Properties& prop);
+
+ /*!
+ * @if jp
+ * @brief basic_strembuf へのポインタを返す
+ *
+ * Loggerで使用する basic_streambuf へのポインタを返す。
+ *
+ * @return basic_streambuf (coil::LogStreambuf) へのポインタ
+ *
+ * @else
+ *
+ * @brief Returns a pointer to the basic_streambuf
+ *
+ * This operation would returns a pointer to the basic_streambuf
+ * or its subclass that is kept in this class.
+ *
+ * @return pointer to the basic_streambuf (coil::LogStreambuf)
+ *
+ * @endif
+ */
+ virtual StreambufType* getStreamBuffer();
+
+ protected:
+ FluentBitStream m_fbstream;
+ };
+}; // namespace RTC
+
+extern "C"
+{
+ void DLL_EXPORT FluentBitInit();
+};
+
+#endif // RTC_LOGGER_FLUENTBIT_H
Property changes on: trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/FluentBit.h
___________________________________________________________________
Added: svn:executable
+ *
Added: trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/Makefile.am
===================================================================
--- trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/Makefile.am (rev 0)
+++ trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/Makefile.am 2017-02-06 19:43:54 UTC (rev 2925)
@@ -0,0 +1,62 @@
+## -*- Makefile -*-
+##---------------------------------------------------------------------------
+## Makefile.am for external libraries
+##
+## $Id$
+##---------------------------------------------------------------------------
+
+AUTOMAKE_OPTIONS = 1.4
+
+AM_CPPFLAGS= \
+ -I$(top_srcdir)/src/ext/logger/fluentbit_stream/fluent-bit/include \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/lib/coil/include \
+ -I$(top_srcdir)/src/lib/rtm/idl
+
+AM_LDFLAGS= \
+ -L$(top_builddir)/src/ext/logger/fluentbit_stream/fluent-bit/build/lib \
+ -L$(top_builddir) \
+ -L$(top_builddir)/src/lib/rtm \
+ -L$(top_builddir)/src/lib/rtm/idl
+
+FluentBit.cpp FluentBit.h: fluent-bit.h libfluent-bit.so
+fluent-bit.h: fluent-bit-clone
+libfluent-bit.so: fluent-bit-clone fluent-bit-build
+
+fluent-bit-clone:
+ @if test -d fluent-bit ; then \
+ echo "fluent-bit pulling"; \
+ (cd fluent-bit ; git pull); \
+ else \
+ rm -rf fluent-bit; \
+ echo "fluent-bit cloneing"; \
+ git clone --depth 1 https://github.com/fluent/fluent-bit.git; \
+ fi;
+
+fluent-bit-build: fluent-bit-clone
+ mkdir -p fluent-bit/build
+ ( cd fluent-bit/build ; cmake .. -DCMAKE_INSTALL_PREFIX=/usr ; make )
+
+fluent-bit-install: fluent-bit-build
+ ( cd fluent-bit/build; make install )
+
+install-data-local: fluent-bit-install
+
+#------------------------------------------------------------
+# Targets
+#------------------------------------------------------------
+# Targets directories
+moduledir = $(rtm_loggerdir)
+
+module_LTLIBRARIES = FluentBit.la
+
+FluentBit_la_SOURCES = FluentBit.cpp FluentBit.h
+FluentBit_la_LDFLAGS = -module -shared -lfluent-bit
+FluentBit_la_LIBADD = \
+ $(top_builddir)/src/lib/rtm/libRTC.la \
+ $(top_builddir)/src/lib/coil/lib/libcoil.la
+
+
+clean-local:
+ rm -rf *~ *.o *.so
+ rm -rf fluent-bit
Property changes on: trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/Makefile.am
___________________________________________________________________
Added: svn:executable
+ *
Added: trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/fluentbit.conf
===================================================================
--- trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/fluentbit.conf (rev 0)
+++ trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/fluentbit.conf 2017-02-06 19:43:54 UTC (rev 2925)
@@ -0,0 +1,24 @@
+# This is fluentbit logger plugin example in rtc.conf
+logger.enable: YES
+logger.log_level: PARANOID
+logger.file_name: rtc%p.log, stderr
+
+# fluentbit specific configurations
+logger.plugins: FluentBit.so
+
+# Output example (forward)
+logger.logstream.fluentd.output0.plugin: forward
+logger.logstream.fluentd.output0.tag: fluent_forward
+logger.logstream.fluentd.output0.match: *
+#logger.logstream.fluentd.output0.host: 127.0.0.1 (default)
+#logger.logstream.fluentd.output0.port: 24224 (default)
+
+# Output example (stdout)
+logger.logstream.fluentd.output1.plugin: stdout
+logger.logstream.fluentd.output1.tag: fluent_stdout
+logger.logstream.fluentd.output1.match: *
+
+# Input example (CPU)
+logger.logstream.fluentd.input0.plugin: cpu
+logger.logstream.fluentd.input0.tag: fluent_cpu
+
Property changes on: trunk/OpenRTM-aist/src/ext/logger/fluentbit_stream/fluentbit.conf
___________________________________________________________________
Added: svn:executable
+ *
More information about the openrtm-commit
mailing list