EventFileChain.cc

Go to the documentation of this file.

#include <utl/ErrorLogger.h>
#include <utl/AugerException.h>
#include <utl/Branch.h>

#include <io/EventFileChain.h>
#include <io/EventFile.h>
#include <io/VEventFile.h>

#include <sstream>
#include <cstdio>
#include <boost/process.hpp>
#include <boost/format.hpp>


namespace io {

  std::map<std::string, std::string> EventFileChain::fgXMLDictionary;


  EventFileChain::EventFileChain(utl::Branch& branch) :
    fBranch(branch)
  {
    const std::string fileTypeName = fBranch.GetName();

    io::Type fileType = io::Type::eUnknownType;
    if (fileTypeName == "Offline")
      fileType = eOffline;
    else if (fileTypeName == "FDAS")
      fileType = eFDas;
    else if (fileTypeName == "CDAS")
      fileType = eCDas;
    else if (fileTypeName == "CORSIKA")
      fileType = eCorsika;
  #ifdef HAVE_AIRES
    else if (fileTypeName == "AIRES")
      fileType = eAires;
  #endif
    else if (fileTypeName == "IoAuger")
      fileType = eIoAuger;
    else if (fileTypeName == "CONEX")
      fileType = eCONEX;
    else if (fileTypeName == "CONEXRandom")
      fileType = eCONEXRandom;
    else if (fileTypeName == "SENECA")
      fileType = eSeneca;
    else if (fileTypeName == "Seneca")
      fileType = eSeneca;
    else if (fileTypeName == "Artificial" || fileTypeName == "ARTIFICIAL")
      fileType = eArtificial;
    else if (fileTypeName == "ZHAireS")
      fileType = eZHAireS;
    else if (fileTypeName == "REAS")
      fileType = eREAS;
    else if (fileTypeName == "REASH5")
      fileType = eREASH5;
    //else if (fileTypeName == "ReAires")
    //  fileType = eReAires;
    else if (fileTypeName == "EVA")
      fileType = eEVA;
    else if (fileTypeName == "RadioAERA")
      fileType = eRadioAERA;
    else if (fileTypeName == "RadioAERAroot")
      fileType = eRadioAERAroot;
    else if (fileTypeName == "SELFAS")
      fileType = eSELFAS;
    else {
      fileType = eUnknownType;
      std::stringstream err;
      err << " Unknown file type '" << fileTypeName << "'.";
      ERROR(err.str());
      throw utl::IOFailureException(err.str());
    }
    fType = fileType;

    const auto filelist =
      fBranch.GetChild("InputFilenames").Get<std::vector<std::string>>();
    SetFileList(filelist);
  }


  EventFileChain::EventFileChain(const std::string& filenames, const Type type) :
    fType(type)
  {
    std::istringstream iss(filenames);
    std::string name;
    while (iss >> name)
      fFileList.push_back(name);

    Parse();

    fFileIterator = fFileList.begin();
  }


  EventFileChain::EventFileChain(const std::vector<std::string>& filenames,
                                 const Type type) :
    fFileList(filenames),
    fFileIterator(fFileList.begin()),
    fType(type)
  {
    Parse();
  }


  EventFileChain::~EventFileChain()
  {
    delete fOpenFile;
  }


  void
  EventFileChain::SetFileList(const std::vector<std::string>& filenames)
  {
    delete fOpenFile;
    fOpenFile = nullptr;

    if (filenames.empty())
      throw utl::IOFailureException("EventFileChain::SetFileList called with empty filelist");

    fFileList = filenames;

    Parse();

    fFileIterator = fFileList.begin();
  }


  Status
  EventFileChain::Read(evt::Event& event)
  {
    for (;;) {

      if (fOpenFile)
        switch (const Status stat = fOpenFile->Read(event)) {
        case eFail:
          {
            std::ostringstream warn;
            warn << "file '"
                 << GetCurrentFilename() << "' is skipped due to IO problems";
            WARNING(warn);
          }
          // fall through
        case eEOF:
          delete fOpenFile;
          fOpenFile = nullptr;
          ++fFileIterator;
          break;
        default:
          return stat;
        }

      if (fFileIterator == fFileList.end())
        return eEOF;

      while (OpenCurrent() != eSuccess) {
        ++fFileIterator;
        if (fFileIterator == fFileList.end())
          return eEOF;
      }
    }
  }


  const std::string&
  EventFileChain::GetCurrentFilename()
    const
  {
    if (fFileIterator != fFileList.end())
      return *fFileIterator;
    else {
      static const std::string empty("");
      return empty;
    }
  }


  inline
  Status
  EventFileChain::OpenCurrent()
  {
    // Exception handling, to support empty or corrput data files
    try {

      delete fOpenFile;
      fOpenFile = new EventFile(*fFileIterator, eRead, fType, &fBranch);

    } catch (utl::IOFailureException& ex) {

      fOpenFile = nullptr;
      std::ostringstream err;
      err << "File '" << *fFileIterator
          << "' is not readable/corrupt: " << ex.GetMessage()
          << " Skipping...";

      if (fCatchExceptions) {
        WARNING(err);
        return eFail;
      } else {
        // re-throw
        ERROR(err);
        throw ex;
      }

    }

    return eSuccess;
  }


  void
  EventFileChain::Parse()
  {
    if (fgXMLDictionary.empty())
      InitXMLDictionary();

    // replace xml constructs
    for (auto& s : fFileList) {
      for (const auto& d : fgXMLDictionary) {
        const auto n = d.first.size();
        for (std::string::size_type begin = 0, pos = s.find(d.first, 0);
             pos != std::string::npos; pos = s.find(d.first, begin)) {
          s.replace(pos, n, d.second);
          begin = pos + n;
        }
      }
    }

    // find backticks or $( to execute
    for (auto sIt = fFileList.begin(); sIt != fFileList.end(); ) {

      const auto& line = *sIt;
      if (line.length() < 2 ||
          !(line[0] == '`' || (line[0] == '$' && line[1] == '('))) {
        // skip lines that do not start a shell command
        ++sIt;
        continue;
      }

      std::string command;

      if (line[0] == '`') {

        // `bash code`
        // we assume the shell code does not contain any backticks

        std::string assemble;
        while (sIt != fFileList.end() && (*sIt)[sIt->length()-1] != '`') {
          assemble += *sIt;
          assemble += ' ';
          sIt = fFileList.erase(sIt);
        }
        if (sIt == fFileList.end())
          throw utl::IOFailureException("unmatched closing backtick ` in file list");
        assemble += *sIt;
        sIt = fFileList.erase(sIt);
        // remove starting and ending backtick
        command = assemble.substr(1, assemble.length()-2);

        if (command.find_first_of("`") != std::string::npos)
          throw utl::IOFailureException("additional backticks in shell script are not supported");

      } else {

        // $(bash code)

        std::string assemble;
        int bracket = 0;
        while (sIt != fFileList.end()) {
          const auto& line = *sIt;
          for (const auto c : line)
            bracket += (c == '(') - (c == ')');
          if (!assemble.empty())
            assemble += ' ';
          assemble += line;
          sIt = fFileList.erase(sIt);
          if (!bracket)
            break;
        }
        if (assemble.back() != ')')
          throw utl::IOFailureException("'$(' shell command has to be terminated with ')'");

        // remove $( and )
        const int skipFront = 2;
        const int skipBack = 1;
        command = assemble.substr(skipFront, assemble.length()-(skipFront + skipBack));

      }

      INFO("executing command $(" + command + ")");

      namespace bp = boost::process;
      bp::ipstream is;
      // bp::child assumes first argument is the executable to run, we must call bash explicitly
      bp::child pipe(bp::search_path("sh"), std::vector<std::string>{ "-e", "-c", command }, bp::std_out > is);
      std::string word;
      while (is >> word) {
        sIt = fFileList.insert(sIt, word);
        ++sIt;
      }
      pipe.wait();
      if (pipe.exit_code())
        throw utl::IOFailureException(str(boost::format("file-list command `%1%` returned status %2%") % command % pipe.exit_code()));

    }

    if (fFileList.empty())
      WARNING("filelist is empty");

  }


  void
  EventFileChain::InitXMLDictionary()
  {
    if (!fgXMLDictionary.empty())
      return;

    fgXMLDictionary = {
      { "&amp;",  "&" },
      { "&quot;", "\"" },
      { "&apos;", "'" },
      { "&lt;",   "<" },
      { "&gt;",   ">" }
    };
  }

}