Ads_bDataBase.cc

Go to the documentation of this file.

/*
 * AdsbDataBase.cpp
 *
 *  Created on: Jul 23, 2015
 *      Author: leven
 */

#include "Ads_bDataBase.h"
#include "AirplaneUtil.h"
#include "FlightAndPositionPair.h"
#include "Airplane.h"
#include "Flight.h"

#include "fwk/CentralConfig.h"

#include "utl/LeapSeconds.h"
#include "utl/UTMPoint.h"
#include "utl/PhysicalConstants.h"
#include "utl/ErrorLogger.h"
#include "utl/Branch.h"

#include "det/Detector.h"

#include "boost/filesystem.hpp"
#include "boost/lexical_cast.hpp"

#include <iostream>
#include <fstream>
#include <sstream>
#include <limits>
#include <sys/time.h>



using namespace utl;
using namespace fwk;
using namespace std;
using namespace RdAirplane;
using namespace boost;
using namespace det;
namespace fs = filesystem;


Ads_bDataBase::Ads_bDataBase(const TimeInterval& pTimeShift, const AirplaneOffsetsMap& pAirplaneOffsets, const string& pRootTrajectoryFitFileName) :
  _coordinateSystem_(Detector::GetInstance().GetReferenceCoordinateSystem()),
  _timeShift_(pTimeShift),
  _airplaneOffsets_(pAirplaneOffsets),
  _rootTrajectoryFitFile_(pRootTrajectoryFitFileName.empty() ? NULL : new TFile(pRootTrajectoryFitFileName.c_str(), "RECREATE")){}

Ads_bDataBase::~Ads_bDataBase() {
  for(AirplaneMap::iterator iter = _airplanes_.begin();
      iter!=_airplanes_.end();
      ++iter) {
    delete iter->second;
  }
  if(_rootTrajectoryFitFile_) {
    _rootTrajectoryFitFile_->Close();
    _rootTrajectoryFitFile_->Delete();
  }
  delete _rootTrajectoryFitFile_;
}

Airplane* Ads_bDataBase::getAirplaneById(const std::string& pId) {
  AirplaneMap::iterator iter = _airplanes_.find(pId);
  if(iter==_airplanes_.end())
    return NULL;
  return iter->second;
}

Ads_bDataBase::ReadLogFileResults Ads_bDataBase::AddAirplanesFromLogFiles(const std::vector<std::string>& pAds_bLogFileNames) {
  ReadLogFileResults results;
  for(vector<string>::const_iterator iter = pAds_bLogFileNames.begin();iter!=pAds_bLogFileNames.end();++iter) {
    const string& fileName = *iter;
    results.push_back(_addAirplanesFromLogFile(fileName));
  }
  for(AirplaneMap::iterator iter=_airplanes_.begin(); iter!=_airplanes_.end(); ++iter) {
    iter->second->Init();
  }
  if(_rootTrajectoryFitFile_) {
    _rootTrajectoryFitFile_->Write();
    _rootTrajectoryFitFile_->Close();
  }
  return results;
}

Ads_bDataBase::ReadLogFileResult Ads_bDataBase::_addAirplanesFromLogFile(const string& pAds_bLogFileName) {
  timeval startTime;
  gettimeofday(&startTime, NULL);
  INFO("Reading File: " + pAds_bLogFileName + "...");
  if(_logFiles_.find(pAds_bLogFileName)!=_logFiles_.end())
    return ReadLogFileResult("Log file already loaded.", true, pAds_bLogFileName);
  _logFiles_.insert(pAds_bLogFileName);
  fs::path pathToFile = fs::path(pAds_bLogFileName);
  if(!fs::exists(pathToFile)) {
    return ReadLogFileResult("Could not open file.", false, pAds_bLogFileName);
  }
  ifstream file(pAds_bLogFileName.c_str());
  bool allGood = true;
    stringstream message;
  try {
    if(!file.is_open())
      return ReadLogFileResult("Could not open file.", false, pAds_bLogFileName);
    string line;
    unsigned int lineNumber=0;

    while(getline(file,line)) {
      ++lineNumber;
      string::size_type doubleWhiteSpacePosition=-1;

      while((doubleWhiteSpacePosition=line.find("  "))!=string::npos)
        line.replace(doubleWhiteSpacePosition, 2, " ");

      if(line.empty() || line==" ")
        continue;

      unsigned long capturedAtUnixTime=0;
      double latitude=0.0;
      double longitude=0.0;
      double altitude=0.0;
      string id;
      string ads_bMessageAsHex;
      stringstream lineAsStream(line);
      lineAsStream >> capturedAtUnixTime;
      lineAsStream >> ads_bMessageAsHex;
      lineAsStream.ignore(numeric_limits<streamsize>::max(),' ');
      lineAsStream.ignore(numeric_limits<streamsize>::max(),' ');
      lineAsStream.ignore(numeric_limits<streamsize>::max(),' ');
      lineAsStream.ignore(numeric_limits<streamsize>::max(),' ');
      lineAsStream.ignore(numeric_limits<streamsize>::max(),' ');
      lineAsStream.ignore(numeric_limits<streamsize>::max(),' ');
      lineAsStream.ignore(numeric_limits<streamsize>::max(),' ');
      lineAsStream.ignore(numeric_limits<streamsize>::max(),' ');
      lineAsStream.ignore(numeric_limits<streamsize>::max(),' ');
      lineAsStream >> id;
      lineAsStream >> latitude;
      lineAsStream >> longitude;
      lineAsStream >> altitude;

      ads_bMessageAsHex = ads_bMessageAsHex.substr(1, ads_bMessageAsHex.length()-2);
      // some sanity checks
      if( capturedAtUnixTime < 1388534400000 || // later than 1.1.14 00:00:00
          id.length()!=6 ||
          latitude<-40.0 || latitude>-30.0 ||
          longitude<-75.0 || longitude>-60.0 ||
          altitude < 4000.0 || altitude > 16000.0) {

        allGood = false;
        message << "Skipped bad data in Line " << lineNumber << endl;
        message << "\t time: " << Util::TimestampHumanReadAble(capturedAtUnixTime) << "(" << capturedAtUnixTime << ")" << endl;
        message << "\t airplane id: " << id << endl;
        message << "\t latitude: " << latitude << endl;
        message << "\t longitude: " << longitude << endl;
        message << "\t altitude: " << altitude << endl;
        continue;
      }
      // convert unix time to gps time and do from now on everything in the auger time format
      unsigned long unixMilliSeconds = capturedAtUnixTime%1000;
      time_t unixSeconds = (capturedAtUnixTime-unixMilliSeconds)/1000.0;
      unsigned long gpsSecond;
      LeapSeconds::GetInstance().ConvertUnixToGPS(unixSeconds, gpsSecond);
      TimeStamp capturedAtTime(gpsSecond, ((double)unixMilliSeconds)*pow(10.0,6));
      capturedAtTime+=getTimeShift();
      Airplane* airplane = _getOrCreateAndGetAirplaneById(id);
      airplane->AddAds_bEvent(capturedAtTime, UTMPoint(latitude*degree, longitude*degree, altitude*m, ReferenceEllipsoid::eWGS84), ads_bMessageAsHex);
    }
  } catch(...) {
    file.close();
    throw;
  }
  file.close();
  timeval endTime;
  gettimeofday(&endTime, NULL);
  double passedTime = endTime.tv_sec-startTime.tv_sec+(1e-6)*(endTime.tv_usec-startTime.tv_usec);
  INFO("(Finished in " + lexical_cast<string>(round(passedTime)) + " s)");
  return ReadLogFileResult(message.str(), allGood, pAds_bLogFileName);
}

Ads_bDataBase::FlightsAndCoordinatesList Ads_bDataBase::getFlights(const TimeStamp& pTime, const Point& pReconstructedEventPosition, const double& pMaxAngleDifferenceInRadian, CoordinateSystemPtr pCoreCoordinateSystem) {
  Ads_bDataBase::FlightsAndCoordinatesList results;

  for(AirplaneMap::iterator iter = _airplanes_.begin(); iter!=_airplanes_.end(); ++iter) {
    Airplane* airplane = iter->second;
    Flight* flight = airplane->getFlight(pTime);

    if(flight==NULL) // no flight during this time
      continue;

    Point* airplanePosition = flight->getPosition(pTime);

    if(airplanePosition==NULL) // no interpolation during this time
      continue;

    double eX=airplanePosition->GetX(pCoreCoordinateSystem);
    double eY=airplanePosition->GetY(pCoreCoordinateSystem);
    double eZ=airplanePosition->GetZ(pCoreCoordinateSystem);
    double pX=pReconstructedEventPosition.GetX(pCoreCoordinateSystem);
    double pY=pReconstructedEventPosition.GetY(pCoreCoordinateSystem);
    double pZ=pReconstructedEventPosition.GetZ(pCoreCoordinateSystem);
    double absE=sqrt(eX*eX+eY*eY+eZ*eZ);
    double absP=sqrt(pX*pX+pY*pY+pZ*pZ);
    double scalarProduct=eX*pX+eY*pY+eZ*pZ;
    double angle=acos(scalarProduct/absE/absP);

    if(angle>pMaxAngleDifferenceInRadian) // too big difference of reconstructed event origin and airplane position
      continue;

    // copy Point because the value pointed to changes
    results.push_back(FlightAndPositionPair(flight, Point(*airplanePosition), pReconstructedEventPosition, pTime));
  }
  results.sort(FlightAndPositionPair::comparator);
  return results;
}


Airplane* Ads_bDataBase::_getOrCreateAndGetAirplaneById(const std::string& pId) {
  Airplane* airplane = getAirplaneById(pId);
  if(airplane==NULL) {
    airplane = new Airplane(pId, this);
    _airplanes_.insert(make_pair(pId, airplane));
  }
  return airplane;
}

string Ads_bDataBase::getInformations() const {
  string info;
  for(AirplaneMap::const_iterator iter=_airplanes_.begin();iter!=_airplanes_.end();++iter) {
    info+=iter->second->getInformation()+"\n";
  }
  return info;
}

const AirplaneOffset* Ads_bDataBase::getAirplaneOffset(const Airplane& pAirplane, const TimeStamp& pTime) const {
  AirplaneOffsetsMap::const_iterator iterMap = _airplaneOffsets_.find(pAirplane.getId());
  if(iterMap==_airplaneOffsets_.end())
    return NULL;
  const AirplaneOffsetList& offsetList = iterMap->second;
  for(AirplaneOffsetList::const_iterator iter = offsetList.begin(); iter != offsetList.end(); ++iter) {
    double timeDifferenceInS = std::abs((iter->getWhen()-pTime).GetSecond());
    if(timeDifferenceInS < Airplane::LONGEST_TIMESSPAN_BETWEEN_TWO_ADS_B_EVENTS_DURING_ONE_FLIGHT_IN_SECONDS) {
      return &*iter;
    }
  }
  return NULL;
}

Ads_bDataBase::ReadLogFileResults Ads_bDataBase::InitSharedDataBase(const vector<string>& pAds_bLogFileNames, const TimeInterval& pTimeShift, const AirplaneOffsetsMap& pAirplaneOffsets, const string& pRootTrajecotryFitFileName) {
  if(_sharedDataBase==NULL) {
    _sharedDataBase=new Ads_bDataBase(pTimeShift, pAirplaneOffsets, pRootTrajecotryFitFileName);
  }
  return _sharedDataBase->AddAirplanesFromLogFiles(pAds_bLogFileNames);
}

Ads_bDataBase::ReadLogFileResults Ads_bDataBase::InitSharedDataBase(const std::string& pAds_bLogFileNames, const TimeInterval& pTimeShift, const AirplaneOffsetsMap& pAirplaneOffsets, const string& pRootTrajecotryFitFileName) {
  vector<string> fileNames;
  istringstream fileNamesStream(pAds_bLogFileNames);
  copy(istream_iterator<string>(fileNamesStream),
       istream_iterator<string>(),
       back_inserter(fileNames));
  return InitSharedDataBase(fileNames, pTimeShift, pAirplaneOffsets, pRootTrajecotryFitFileName);
}

Ads_bDataBase::ReadLogFileResults Ads_bDataBase::InitSharedDataBaseFromCentralConfig() {
  static bool initialized = false;
  if(initialized)
    return Ads_bDataBase::ReadLogFileResults();
  string ads_bLogFiles;
  string rootTrajecotryFitFileName;
  double timeShift;
  Branch ads_bTopBranch = CentralConfig::GetInstance()->GetTopBranch("RdAds_bDataBase");
  ads_bTopBranch.GetChild("Ads_bLogFiles").GetData(ads_bLogFiles);
  ads_bTopBranch.GetChild("TimeShift").GetData(timeShift);
  ads_bTopBranch.GetChild("RootTrajectoryFitOutputFile").GetData(rootTrajecotryFitFileName);
  Branch offsetBranch = ads_bTopBranch.GetChild("AirplaneOffsets");
  AirplaneOffsetsMap airplaneOffsetsMap;
  for (Branch offsetElement = offsetBranch.GetFirstChild(); offsetElement; offsetElement = offsetElement.GetNextSibling()) {
    string airplaneId;
    double unixTimeStamp;
    double altitudeOffset;
    double timeOffset;
    offsetElement.GetChild("AirplaneId").GetData(airplaneId);
    offsetElement.GetChild("UnixTimeStamp").GetData(unixTimeStamp);
    offsetElement.GetChild("AltitudeOffset").GetData(altitudeOffset);
    offsetElement.GetChild("TimeOffset").GetData(timeOffset);
    AirplaneOffsetsMap::iterator iter = airplaneOffsetsMap.find(airplaneId);
    AirplaneOffsetList* airplaneOffsets = NULL;
    if(iter == airplaneOffsetsMap.end()) {
      airplaneOffsets = &airplaneOffsetsMap.insert(make_pair(airplaneId, AirplaneOffsetList())).first->second;
    } else {
      airplaneOffsets = &iter->second;
    }
    unsigned long gpsSecond;
    LeapSeconds::GetInstance().ConvertUnixToGPS((time_t)unixTimeStamp/s, gpsSecond);
    airplaneOffsets->push_back(AirplaneOffset(TimeStamp(gpsSecond), TimeInterval(timeOffset), altitudeOffset));
  }
  initialized = true;
  return Ads_bDataBase::InitSharedDataBase(ads_bLogFiles, TimeInterval(timeShift), airplaneOffsetsMap, rootTrajecotryFitFileName);
}


Ads_bDataBase* Ads_bDataBase::getSharedDataBase() {
  if(!_sharedDataBase)
    InitSharedDataBaseFromCentralConfig();
  return _sharedDataBase;
}

Ads_bDataBase* Ads_bDataBase::_sharedDataBase=NULL;