【Mahout二】基于Mahout CBayes算法的20newsgroup的脚本分析

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#
# Downloads the 20newsgroups dataset, trains and tests a classifier.
#
# To run:  change into the mahout directory and type:
# examples/bin/classify-20newsgroups.sh

////支持--help以查看该脚本的功能，本脚本是对经典的20 News Groups运行CBayes分类算法
if [ "$1" = "--help" ] || [ "$1" = "--?" ]; then
  echo "This script runs SGD and Bayes classifiers over the classic 20 News Groups."
  exit
fi

SCRIPT_PATH=${0%/*}
if [ "$0" != "$SCRIPT_PATH" ] && [ "$SCRIPT_PATH" != "" ]; then
  cd $SCRIPT_PATH
fi
START_PATH=`pwd`

# Set commands for dfs
source ${START_PATH}/set-dfs-commands.sh

////Mahout MapReduce 作业的本地工作目录，
WORK_DIR=/tmp/mahout-work-${USER}

////本脚本支持的分类算法，这里选择cnaivebayes-MapReduce算法
algorithm=( cnaivebayes-MapReduce naivebayes-MapReduce cnaivebayes-Spark naivebayes-Spark sgd clean)

/////选择分类算法
if [ -n "$1" ]; then
  choice=$1
else
  echo "Please select a number to choose the corresponding task to run"
  echo "1. ${algorithm[0]}"
  echo "2. ${algorithm[1]}"
  echo "3. ${algorithm[2]}"
  echo "4. ${algorithm[3]}"
  echo "5. ${algorithm[4]}"
  echo "6. ${algorithm[5]}-- cleans up the work area in $WORK_DIR"
  read -p "Enter your choice : " choice
fi

echo "ok. You chose $choice and we'll use ${algorithm[$choice-1]}"

///alg中保存选择的算法
alg=${algorithm[$choice-1]}

# Spark specific check and work 
if [ "x$alg" == "xnaivebayes-Spark" -o "x$alg" == "xcnaivebayes-Spark" ]; then
  if [ "$MASTER" == "" ] ; then
    echo "Plese set your MASTER env variable to point to your Spark Master URL. exiting..."
    exit 1
  fi
  if [ "$MAHOUT_LOCAL" != "" ] ; then
    echo "Options 3 and 4 can not run in MAHOUT_LOCAL mode. exiting..."
    exit 1
  fi
fi

////如果不是clean操作
if [ "x$alg" != "xclean" ]; then

  echo "creating work directory at ${WORK_DIR}"
  ////创建工作目录
  mkdir -p ${WORK_DIR}
  ////下载样本文件
  if [ ! -e ${WORK_DIR}/20news-bayesinput ]; then
    if [ ! -e ${WORK_DIR}/20news-bydate ]; then
      if [ ! -f ${WORK_DIR}/20news-bydate.tar.gz ]; then
        echo "Downloading 20news-bydate"
        curl http://people.csail.mit.edu/jrennie/20Newsgroups/20news-bydate.tar.gz -o ${WORK_DIR}/20news-bydate.tar.gz
      fi
      mkdir -p ${WORK_DIR}/20news-bydate
      echo "Extracting..."
      cd ${WORK_DIR}/20news-bydate && tar xzf ../20news-bydate.tar.gz && cd .. && cd ..
    fi
  fi
fi
#echo $START_PATH
cd $START_PATH
cd ../..

set -e

////如果CBayes分类算法
if  ( [ "x$alg" == "xnaivebayes-MapReduce" ] ||  [ "x$alg" == "xcnaivebayes-MapReduce" ] || [ "x$alg" == "xnaivebayes-Spark"  ] || [ "x$alg" == "xcnaivebayes-Spark" ] ); then
  c=""

  if [ "x$alg" == "xcnaivebayes-MapReduce" -o "x$alg" == "xnaivebayes-Spark" ]; then
    c=" -c"
  fi

  set -x
  echo "Preparing 20newsgroups data"
  rm -rf ${WORK_DIR}/20news-all
  mkdir ${WORK_DIR}/20news-all
  cp -R ${WORK_DIR}/20news-bydate/*/* ${WORK_DIR}/20news-all

  if [ "$HADOOP_HOME" != "" ] && [ "$MAHOUT_LOCAL" == "" ] ; then
    echo "Copying 20newsgroups data to HDFS"
    set +e
    $DFSRM ${WORK_DIR}/20news-all
    $DFS -mkdir ${WORK_DIR}
    $DFS -mkdir ${WORK_DIR}/20news-all
    set -e
    if [ $HVERSION -eq "1" ] ; then
      echo "Copying 20newsgroups data to Hadoop 1 HDFS"
      $DFS -put ${WORK_DIR}/20news-all ${WORK_DIR}/20news-all
    elif [ $HVERSION -eq "2" ] ; then
      echo "Copying 20newsgroups data to Hadoop 2 HDFS"
      $DFS -put ${WORK_DIR}/20news-all ${WORK_DIR}/
    fi
  fi

////1. 对样本文件进行创建Sequence File
  echo "Creating sequence files from 20newsgroups data"
  ./bin/mahout seqdirectory \
    -i ${WORK_DIR}/20news-all \
    -o ${WORK_DIR}/20news-seq -ow

////2. 将Sequence File转换为tfidf向量文件，存放于20news-vectors目录下
  echo "Converting sequence files to vectors"
  ./bin/mahout seq2sparse \
    -i ${WORK_DIR}/20news-seq \
    -o ${WORK_DIR}/20news-vectors  -lnorm -nv  -wt tfidf

////3. 对生成的向量数据集（20news-vectors/tfidf-vectors）创建training向量文件和测试向量文件
  echo "Creating training and holdout set with a random 80-20 split of the generated vector dataset"
  ./bin/mahout split \
    -i ${WORK_DIR}/20news-vectors/tfidf-vectors \
    --trainingOutput ${WORK_DIR}/20news-train-vectors \
    --testOutput ${WORK_DIR}/20news-test-vectors  \
    --randomSelectionPct 40 --overwrite --sequenceFiles -xm sequential

    if [ "x$alg" == "xnaivebayes-MapReduce"  -o  "x$alg" == "xcnaivebayes-MapReduce" ]; then
      ////4. 训练Naive Bayes模型，输入为训练向量，输出为训练模型
      ////labelindex指的是分类名称，共20个，以Sequence File格式保存
      echo "Training Naive Bayes model"
      ./bin/mahout trainnb \
        -i ${WORK_DIR}/20news-train-vectors \
        -o ${WORK_DIR}/model \
        -li ${WORK_DIR}/labelindex \
        -ow $c
      
      ////5.1 基于训练向量文件进行测试
      echo "Self testing on training set"

      ./bin/mahout testnb \
        -i ${WORK_DIR}/20news-train-vectors\
        -m ${WORK_DIR}/model \
        -l ${WORK_DIR}/labelindex \
        -ow -o ${WORK_DIR}/20news-testing $c

      echo "Testing on holdout set"
      
       ////5.2 基于测试向量文件进行测试
      ./bin/mahout testnb \
        -i ${WORK_DIR}/20news-test-vectors\
        -m ${WORK_DIR}/model \
        -l ${WORK_DIR}/labelindex \
        -ow -o ${WORK_DIR}/20news-testing $c

    elif [ "x$alg" == "xnaivebayes-Spark" -o "x$alg" == "xcnaivebayes-Spark" ]; then

      echo "Training Naive Bayes model"
      ./bin/mahout spark-trainnb \
        -i ${WORK_DIR}/20news-train-vectors \
        -o ${WORK_DIR}/spark-model $c -ow -ma $MASTER

      echo "Self testing on training set"
      ./bin/mahout spark-testnb \
        -i ${WORK_DIR}/20news-train-vectors\
        -m ${WORK_DIR}/spark-model $c -ma $MASTER

      echo "Testing on holdout set"
      ./bin/mahout spark-testnb \
        -i ${WORK_DIR}/20news-test-vectors\
        -m ${WORK_DIR}/spark-model $c -ma $MASTER
        
    fi

////如果是线性回归算法
elif [ "x$alg" == "xsgd" ]; then
  if [ ! -e "/tmp/news-group.model" ]; then
    echo "Training on ${WORK_DIR}/20news-bydate/20news-bydate-train/"
    ./bin/mahout org.apache.mahout.classifier.sgd.TrainNewsGroups ${WORK_DIR}/20news-bydate/20news-bydate-train/
  fi
  echo "Testing on ${WORK_DIR}/20news-bydate/20news-bydate-test/ with model: /tmp/news-group.model"
  ./bin/mahout org.apache.mahout.classifier.sgd.TestNewsGroups --input ${WORK_DIR}/20news-bydate/20news-bydate-test/ --model /tmp/news-group.model

////如果是clean操作，则删除工作目录
elif [ "x$alg" == "xclean" ]; then
  rm -rf $WORK_DIR
  rm -rf /tmp/news-group.model
  $DFSRM $WORK_DIR
fi
# Remove the work directory
#

 

 

基于Mahout的CBayes算法执行流程如下：

1. 对样本文件进行创建Sequence File
2. 将Sequence File转换为tfidf向量文件，存放于20news-vectors目录下
3. 对生成的向量数据集（20news-vectors/tfidf-vectors）创建training向量文件和测试向量文件
4. 训练Naive Bayes模型，输入为训练向量，输出为训练模型 
5.1 基于训练向量文件进行测试
5.2 基于测试向量文件进行测试