Flume 설치, 환경 설정 및 테스트

1. Flume 라이브러리 다운 후 압축 풀기

 wget http://mirror.apache-kr.org/flume/1.4.0/apache-flume-1.5.0-bin.tar.gz
   tar zxvf apache-flume-1.5.0.bin.tar.gz

2. 환경변수 설정

   vi .bash_profile
   export FLUME_HOME=/home/bigdata/apache-flume-1.4.0-bin
   export PATH=$PATH:$FLUME_HOME/bin
   :wq
   source .bash_profile

3. Flume Configuration

   cd $FLUME_HOME/conf
   cp flume-conf.properties.template flume.conf
   vi flume.conf

agent.sources = seqGenSrc
agent.channels = memoryChannel
agent.sinks = hdfsSink

# For each one of the sources, the type is defined
agent.sources.seqGenSrc.type = exec
agent.sources.seqGenSrc.command = tail -F /home/bigdata/hadoop-1.2.1/logs/hadoop-hadoop-namenode-localhost.localdomain.log
#가상분산환경에서 테스트용으로 잡은것.

# The channel can be defined as follows.
agent.sources.seqGenSrc.channels = memoryChannel

# Each sink's type must be defined
agent.sinks.hdfsSink.type = hdfs
agent.sinks.hdfsSink.hdfs.path = hdfs://localhost:9000/flume/data     #테스트용
agent.sinks.hdfsSink.rollInterval = 30
agent.sinks.hdfsSink.sink.batchSize = 100

#Specify the channel the sink should use
agent.sinks.hdfsSink.channel = memoryChannel

# Each channel's type is defined.
agent.channels.memoryChannel.type = memory

# Other config values specific to each type of channel(sink or source)
# can be defined as well
# In this case, it specifies the capacity of the memory channel
agent.channels.memoryChannel.capacity = 100000
agent.channels.memoryChannel.transactionCapacity = 10000

:wq

4. 동작 테스트
   cd ~
   flume-ng agent --conf-file ./apache-flume-1.4.0-bin/conf/flume.conf --name agent