Message-Id: <200301070806.h07862v6014405@inti.its.uow.edu.au>
Date: Tue, 7 Jan 2003 19:10:7 +1000
From: Y Chen <yc12 AT uow DOT edu DOT au>
To: "djgpp AT delorie DOT com" <djgpp AT delorie DOT com>
Subject: scan() in c++
X-mailer: FoxMail 3.11 Release [cn]
Mime-Version: 1.0
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Reply-To: djgpp AT delorie DOT com

Dear Sir,

I got the following errors(The file is at the bottom):

LSTM.cpp: In member function `virtual char TLSTM::LoadPar()':
LSTM.cpp:338: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[17])'
LSTM.cpp:343: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[17])'
LSTM.cpp:346: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[16])'
LSTM.cpp:347: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[21])'
LSTM.cpp:348: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[14])'
LSTM.cpp:353: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[33])'
LSTM.cpp:355: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[20])'
LSTM.cpp:356: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[15])'
LSTM.cpp:358: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[15])'
LSTM.cpp:360: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[15])'
LSTM.cpp:364: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[15])'
LSTM.cpp:366: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[16])'
LSTM.cpp:368: no matching function for call to `std::basic_fstream<char, 
   std::char_traits<char> >::scan(const char[16])'
LSTM.cpp: In member function `char TLSTM::WriteWeightFile(char*)':
LSTM.cpp:2019: using typedef-name `std::iostream' after `class'
LSTM.cpp: In member function `char TLSTM::LoadWeightFile(char*)':
LSTM.cpp:2025: using typedef-name `std::iostream' after `class'
LSTM.cpp: In member function `char TLSTM::DumpAll(char*)':
LSTM.cpp:2033: using typedef-name `std::iostream' after `class'
LSTM.cpp: In member function `void TLSTM::DisplayWeights()':
LSTM.cpp:2388: using typedef-name `std::iostream' after `class'

How to fix the problem about "no matching function..." and "using typedef-name `std::iostream' after `class' "? Thanks a lot for your help.

Regards,
Y Chen

 // LSTM.cpp
#include <stdlib.h>
#include <stdio.h>
#include <iostream.h>
#include <iomanip.h>
#include <new.h> // for memory error handling
//#include <ctype.h>
#include <unistd.h> // sleep 
//#include <termios.h> // struct termios
#include <fcntl.h>
#include <sys/param.h>
#include <math.h>
#include <errno.h>
#include <string.h>
#include <time.h>
#include <float.h> // To have max values for imposible inits.
#include "LSTM.h"
#include "Error.h"

//// TLSTM

void TLSTM::NewTs_d_Gate(Ts_d_Gate *&sd, unsigned int size) {
  sd = new Ts_d_Gate[size];
  for(int i=0;i<size;i++) {
    sd[i].s_d = 0;
    sd[i].alpha = AlphaBase;
    sd[i].h=0;
  }
}
void TLSTM::NewTs_d_Gate(Ts_d_Gate &sd) {
  Ts_d_Gate *psdTmp; NewTs_d_Gate(psdTmp,1);
  sd.s_d   = psdTmp[0].s_d;
  sd.alpha = psdTmp[0].alpha;
  sd.h     = psdTmp[0].h;
  delete psdTmp;
}

char TLSTM::Run() {
  if(Init()) return 1;
  for(Tri=0;Tri<NbTrials;Tri++) { // Tri loop.
    if(Tri>0) if(LoadPar()) return 1;
    InitNet();
    //WriteWeightFile(WEIGHTLOGFILE); //exit(0);
    //LoadWeightFile(WEIGHTLOGFILE);
    //sprintf(cBuf, "w.init.%d",Tri); LoadWeightFile(cBuf);
    //LoadWeightFile(cBuf);
    //WriteWeightFile(cBuf);
    //LoadWeightFile("w.init");
    //if(LoadWeightFile("W")) exit(1); //ddd
    //LoadWeightFile("w.hand");
    //WriteWeightFile(WEIGHTLOGFILE);
    // Run and Train the net.
    //Test(); exit(0); //ddd
    //GetOnlineProblemStatistics(); exit(0); //ddd
    do { // Epo loop.
      Epo++;
#ifdef DO_ONLINE_PAT_PRODUCTION
      // This Seq will be overwritten, we init the Nb variables here.
      ONLINE_PAT_FUNCNAME(sTrainData, true);
#else
      if(Generate_Pat && Generate_Pat_Each_Epoch>0) {
	if((unsigned int)fmod(Epo, Generate_Pat_Each_Epoch) == 0)
	  GeneratePattern(sTrainData, Pat_FuncName);
      }
      if(MixTrainSeqs) MixTrainSequences();
#endif
      ClassesWrong=0; MSEEpo=0; PatCorrect=0; PatCount=0; 
      SeqOnline=0; SeqOnlinePatCount=0;
      // Seq loop.
      for(Seq=0;Seq<sTrainData.NbSeq;FreezeSeqLoop ? Seq=0 : Seq++) {
	MSESeq=0; PatWrong=0; 
#ifndef DO_ONLINE_PAT_PRODUCTION
 	ResetNet();
#else
	if(!SeqOnline) ResetNet(); //For sequential online
#endif
#ifdef DO_ONLINE_PAT_PRODUCTION
#if defined(NO_RESET_AFTER_ONLINE_SEQ) || defined(SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE)
	if(SeqOnline) ONLINE_PAT_FUNCNAME(sTrainData, false);
	else ONLINE_PAT_FUNCNAME(sTrainData, true);
#else
	ONLINE_PAT_FUNCNAME(sTrainData, true);
#endif
#endif
	//	if(PatCorrectTest>10)
	//	  {cout<< "\n";cout.flush();}//Debug on-line.
	//DumpAll("dump.init");
	//DisplayNet(sTrainData); DisplayWeights(); KeyCheck();
	//AlphaPredict = Alpha / sTrainData.NbPat[Seq]; // AlphaPredict
 	for(Pat=0;Pat<sTrainData.NbPat[Seq];Pat++) { // Pat loop.
//        if(PatCorrectTest>10) { // Debug on-line pat generation. // ddd
// 	  cout << Epo  << "-" << Seq << "-" << SeqOnline 
// 	       << "-" << Pat << " :";
// 	  cout << "[" << sTrainData.NbPat[0]<< "] ";
//        	  for(int Val=0;Val<sTrainData.NbVal;Val++) 
//        	    cout << ((sTrainData.SeqList[Seq])[Pat])[Val] << " ";
//       	  cout << " = " << sTrainData.NbClass[Seq];
//        	  cout << " -> (" << MSEPat << ") " << PatCorrect;
//        	  cout << "\n"; cout.flush();
  // 	  }
	  ForwardPass(sTrainData,FreezeEndBlock,NbMemoBlocks);
	  //WriteOutLogFile(OUT_LOGFILE);
#ifdef USE_LOCAL_ALPHA_SEQUENCEWISE
	  BackwardPass(FreezeEndBlock,NbMemoBlocks);
#endif
	  if((Pat==sTrainData.NbPat[Seq]-1) || SetStepTarget ||
	     (NbPredictNextIn>0) || (NbPredictClass>0)) {
#ifdef USE_ALPHA_DECAY_IN_SEQ
	    if(SetStepTarget) {
#ifdef ALPHA_DECAY_IN_SEQ_LINEAR
	      AlphaDecayInSeq = USE_ALPHA_DECAY_IN_SEQ/
		(Pat+USE_ALPHA_DECAY_IN_SEQ);
#elif
	      if(Pat*USE_ALPHA_DECAY_IN_SEQ>709) AlphaDecayInSeq=0;
	      else AlphaDecayInSeq = EXP(-(Pat*USE_ALPHA_DECAY_IN_SEQ));
#endif
	    } else {
#ifdef ALPHA_DECAY_IN_SEQ_LINEAR
	      AlphaDecayInSeq = USE_ALPHA_DECAY_IN_SEQ/
		(SeqOnline+USE_ALPHA_DECAY_IN_SEQ);
#elif
	      if(SeqOnline*USE_ALPHA_DECAY_IN_SEQ>709) AlphaDecayInSeq=0;
	      else AlphaDecayInSeq =
		     EXP(-(SeqOnline*USE_ALPHA_DECAY_IN_SEQ)); 
#endif
	    }
	    //     cout << Pat << "-"  << Seq <<  "-" << Epo << ":" 
	    // 	 << AlphaDecayInSeq << " ";
#endif
#ifndef USE_LOCAL_ALPHA_SEQUENCEWISE
	    BackwardPass(FreezeEndBlock,NbMemoBlocks);
#endif
	    PatStatistics(sTrainData);
#ifdef DO_ONLINE_PAT_PRODUCTION
	    if(SetStepTarget) {
	      if(!PatWrong) 
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
		ONLINE_PAT_FUNCNAME(sTrainData, false); 
#else
	        ; // Generate seq only before Pat loop.
#endif
	      else break; // PatWrong.
	      //if(ReberGrammarState==-1) ResetNet();
	    }
#endif
	  }
// 	  if(Epo==7384) {
// 	    sprintf(cBuf, "w.end%d.%d",Epo,Pat);WriteWeightFile(cBuf); 
// 	    if(Pat>840) { 
// 	      sprintf(cBuf, "dump.%d.%d",Epo,Pat);DumpAll(cBuf); }
// 	  }
	  //sprintf(cBuf, "w.Pat.%d",Pat); WriteWeightFile(cBuf); exit(0);
	  //sprintf(cBuf, "dump.Pat.%d",Pat);DumpAll(cBuf);
	  //if(Pat>=3) exit(0);
	  //DisplayNet(sTrainData); DisplayWeights(); KeyCheck();
	  //if(PatCorrectTest>90) Dump_Inner_States(); //ddd
	  //printf(cBuf, "dump.%d.%d",Epo,Pat);DumpAll(cBuf);
	  //if(Epo>1000) exit(0);
	  //Dump_Inner_States();
	} // End Pat loop.
#ifdef UPDATE_WEIGHTS_AFTER_SEQ
	ExecuteWeightChanges(FreezeEndBlock,NbMemoBlocks);
#endif
	SeqStatistics(sTrainData);
	//sprintf(cBuf, "w.Seq.%d",Seq); WriteWeightFile(cBuf); exit(0);
	//DisplayNet(sTrainData); DisplayWeights(); KeyCheck();
#ifdef DO_ONLINE_PAT_PRODUCTION
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
	if(!SetStepTarget)
#endif
	  if(!PatWrong) { 
	    SeqOnline++; 
	    if(PatCorrect>=MaxOnlineStreamLengthTrain) break; 
	  } else break;
#endif
      } // End Seq loop.
      //DumpSomeAlphas();
      EpoStatistics(sTrainData);
      //ExecuteWeightChanges(FreezeEndBlock,NbMemoBlocks);
      //sprintf(cBuf, "w.Epo.%d",Epo); WriteWeightFile(cBuf);
      //sprintf(cBuf, "%s.%d", OUT_LOGFILE,Epo); rename(OUT_LOGFILE,cBuf);
      //if(Epo==7384) DumpAll("dump.end.log");
      //if((unsigned int)fmod(Epo,1000) == 0) {
      //sprintf(cBuf, "w.end%d.%d",Tri,Epo);WriteWeightFile(cBuf); }
      //WriteWeightFile("w");)
      if(StopLern) EpochsAfterLearned--;
#ifdef DO_ONLINE_PAT_PRODUCTION
      if((Epo>=MaxEpochs)||(PatCorrect>=MaxOnlineStreamLengthTrain)
	 ||(MSEEpo<MSEStop)) StopLern=true;
      // StopLearn when the taing set is leerand -> test.
      if(StopLern && (Epo<MaxEpochs)) { 
	if(LastTestEpo+TestMaxEach_Epo<Epo) // Not test too much.
	  Test(); // If not just tested during the EpoStatistics.
	// Here StopLern only then trainig requirements met..
	//StopLern=((PatCorrectTest>=MaxOnlineStreamLengthTest) || 
	//	  (TestMSEEpo<MSEStop));
      }
      // StopLearn also when the test is learnd but the trainig not.
      // The first or is for the case that we give up (MaxEpochs).
      // Test results may come from test during EpoStatistics.
      StopLern=(Epo>=MaxEpochs) ||
	((PatCorrectTest>=MaxOnlineStreamLengthTest) || 
	 (TestMSEEpo<MSEStop));
#else
      if(!ClassesWrong||(Epo>=MaxEpochs)||(MSEEpo<MSEStop)) StopLern=true;
      // Test when Traing set learned.
      //if(LastTestEpo+TestMaxEach_Epo<Epo) { // Not test too much
	//if(StopLern&&(Epo<MaxEpochs)) StopLern = !Test();
      //} else StopLern = false;
#endif
    } while (!StopLern || (EpochsAfterLearned>0));// End Epo loop.
    Test();// Final Test
    //if(Epo>=MaxEpochs) { Test();}// Final test if unsolved.
    //if(Epo>=MaxEpochs) {OutputDebug=true; Test();}// ddd
    //if(Epo<MaxEpochs){OutputDebug=true; Test();}//ddd
    OutputDebug=false; //ddd
#ifdef BUFFER_EPO_LOG
    // Flush the buffers to files.
    if(TrainMSEEpoLogBuf.NbVal) 
      FlushEpoLogBuf(MSELOGFILE, TrainMSEEpoLogBuf);
    if(TrainErrEpoLogBuf.NbVal) 
      FlushEpoLogBuf(ERRORLOGFILE, TrainErrEpoLogBuf);
    if(TestErrEpoLogBuf.NbVal) 
      FlushEpoLogBuf(TEST_ERRORLOGFILE, TestErrEpoLogBuf);
    if(TestMSEEpoLogBuf.NbVal) 
      FlushEpoLogBuf(TEST_MSELOGFILE, TestMSEEpoLogBuf);
#endif
    sprintf(cBuf, "w.end%d.%d",Tri,Epo); WriteWeightFile(cBuf);
    if(NbTrials>1) {
      sprintf(cBuf, "%s.%d", ERRORLOGFILE,Tri); rename(ERRORLOGFILE,cBuf);
      sprintf(cBuf, "%s.%d", MSELOGFILE,Tri); rename(MSELOGFILE,cBuf);
      if(DirContainsFile(true, TEST_ERRORLOGFILE)) {
	sprintf(cBuf, "%s.%d", TEST_ERRORLOGFILE,Tri); 
	rename(TEST_ERRORLOGFILE,cBuf); }
      if(DirContainsFile(true, TEST_MSELOGFILE)) {
	sprintf(cBuf, "%s.%d", TEST_MSELOGFILE,Tri); 
	rename(TEST_MSELOGFILE,cBuf); }
      if(GrowNet) {
	sprintf(cBuf, "%s.%d", GROWLOGFILE,Tri); rename(GROWLOGFILE,cBuf); }
      if(DirContainsFile(true, WEIGHTMEAN_LOGFILE)) {
	sprintf(cBuf, "%s.%d", WEIGHTMEAN_LOGFILE,Tri); 
	rename(WEIGHTMEAN_LOGFILE,cBuf); }
      if(DirContainsFile(true, ALPHAMEAN_LOGFILE)) {
	sprintf(cBuf, "%s.%d", ALPHAMEAN_LOGFILE,Tri); 
	rename(ALPHAMEAN_LOGFILE,cBuf); }
      if(DirContainsFile(true, INNERSTATE_DUMP_LOGFILE)) {
	sprintf(cBuf, "%s.%d", INNERSTATE_DUMP_LOGFILE,Tri); 
	rename(INNERSTATE_DUMP_LOGFILE,cBuf); }
      if(DirContainsFile(true, "AlphaDump.log")) {
	sprintf(cBuf, "%s.%d", "AlphaDump.log",Tri); 
	rename("AlphaDump.log",cBuf); }
    }
    DeleteNet();
  } // End Tri loop.
  cout << "LSTM Done." << endl;
}

void  TLSTM::InitNet() {
  InitMemoBlocks(0,NbMemoBlocks);
  // Init other units.
  // Init Out units.
  Out = new TOut[NbOut];
  for(int iO=0;iO<NbOut;iO++) {
    NewWeight(Out[iO].w_Bias,OutUnitBiasInitWeight);
#ifndef NO_IN_OUT_SHORTCUTS
    NewWeight(Out[iO].w_In,NbIn);
#endif
    NewWeight(Out[iO].w_Hidden,NbHidden);
    Out[iO].w_Cell = new (TWeight *)[MAX_MEMOBLOCKS];
    for(int iB=0;iB<NbMemoBlocks;iB++) 
      NewWeight(Out[iO].w_Cell[iB], MemoBlock[iB].MemoBlockSize);
  }
  // Init Hidden units.
  Hidden = new THidden[NbHidden];
  for(int iH=0;iH<NbHidden;iH++) {
    NewWeight(Hidden[iH].w_Bias,HiddenUnitBiasInitWeight);
    NewWeight(Hidden[iH].w_In, NbIn);
    Hidden[iH].w_Cell = new (TWeight *)[MAX_MEMOBLOCKS];
    for(int iB=0;iB<NbMemoBlocks;iB++) 
      NewWeight(Hidden[iH].w_Cell[iB], MemoBlock[iB].MemoBlockSize);
  }
  // Init NextIn predict.
  // Init PredictNextIn units.
  PredictNextIn = new TPredictNextIn[NbPredictNextIn];
  for(int iP=0;iP<NbPredictNextIn;iP++) {
    NewWeight(PredictNextIn[iP].w_In, NbIn);
    NewWeight(PredictNextIn[iP].w_InClass, NbOut);
    PredictNextIn[iP].w_Cell = new (TWeight *)[MAX_MEMOBLOCKS];
    for(int iB=0;iB<NbMemoBlocks;iB++) 
      NewWeight(PredictNextIn[iP].w_Cell[iB], MemoBlock[iB].MemoBlockSize);
  }
  // Init NextIn units.
  if(NbPredictNextIn>0) {
    NextIn = new TNextIn[NbIn];
    for(int iO=0;iO<NbIn;iO++)
      NewWeight(NextIn[iO].w_PredictNextIn, NbPredictNextIn);
  }
  // Init Class predict.
  // Init PredictClass units.
  PredictClass = new TPredictClass[NbPredictClass];
  for(int iP=0;iP<NbPredictClass;iP++) {
    PredictClass[iP].w_Cell = new (TWeight *)[MAX_MEMOBLOCKS];
    for(int iB=0;iB<NbMemoBlocks;iB++) 
      NewWeight(PredictClass[iP].w_Cell[iB], MemoBlock[iB].MemoBlockSize);
  }
  // Init PredictClassOut units.
  if(NbPredictClass>0) {
    PredictClassOut = new TPredictClassOut[NbOut];
    for(int iO=0;iO<NbOut;iO++)
      NewWeight(PredictClassOut[iO].w_PredictClass, NbPredictClass);
  }
  // Init statistics and other working variables 
  // (only what is not done in RUN).
  PatCorrectTest=0; TestMSEEpo=DBL_MAX; // First test result for StopLern.
  LastTestEpo=0;
  FreezeEndBlock=0; // Worm start.
  Epo=0;
  LogErrRecMean=0; PartLogErrRecMean=0; NbLogedErr=0; 
  WeightMean=0; AlphaMean=0; AlphaStd=0; //WPC=0;
  StopLern=false; MaxPatCorrectTrain=0;
#ifdef BUFFER_EPO_LOG
  TrainMSEEpoLogBuf.Reset(); TrainErrEpoLogBuf.Reset(); 
  TestErrEpoLogBuf.Reset(); TestMSEEpoLogBuf.Reset();
#endif
  //Alpha = AlphaBase + AlphaError;
  NbPredictOut =0;
  if(NbPredictNextIn>0) NbPredictOut +=NbIn;
  if(NbPredictClass>0)  NbPredictOut +=NbOut;
  SetTopoStatistics();
}

char TLSTM::LoadPar() {
  double NotUsed;
  if (TNeuralNetBase::LoadPar()) return 1;
  if (OpenFile(cParFilename, ios::in)) return 1;
  if (ReadComment()) return 1;
  // Scan for the starting point in the parameter file.
  pFile->scan("NbPredictNextIn:"); 
  while(!pFile->good()) { 
    pFile->seekg(1, pFile->cur);
    if(pFile->eof()) { cout << "eof\n"; return 1;}
    pFile->clear();    
    pFile->scan("NbPredictNextIn:"); // First parameter of LSTM.
  }
  *pFile >> NbPredictNextIn;
  pFile->scan("NbPredictClass:");  *pFile >> NbPredictClass;
  pFile->scan("InternalBlockConect:");  *pFile >> InternalBlockConect;
  pFile->scan("NbMemoBlocks:");  *pFile >> NbMemoBlocks;
  if(NbMemoBlocks>MAX_MEMOBLOCKS) { 
    cerr << "NbMemoBlocks>MAX_MEMOBLOCKS\n"; exit(1); }
  MemoBlock = new (TMemoBlock)[NbMemoBlocks];
  //MemoBlock = new (TMemoBlock)[MAX_MEMOBLOCKS];
  pFile->scan("# parameters for each memo block");
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    pFile->scan("# memo block number"); pFile->ignore(1000, '\n');
    pFile->scan("MemoBlockSize:");
    *pFile >> MemoBlock[iB].MemoBlockSize;
    pFile->scan("BegSrcBlockNb:");
    *pFile >> MemoBlock[iB].BegSrcBlockNb;
    pFile->scan("EndSrcBlockNb:");
    *pFile >> MemoBlock[iB].EndSrcBlockNb;
    if(MemoBlock[iB].EndSrcBlockNb==-1) 
      MemoBlock[iB].EndSrcBlockNb = NbMemoBlocks;
    pFile->scan("InputGateBias:");
    *pFile >> MemoBlock[iB].InGate.w_Bias.w;
    pFile->scan("OutputGateBias:");
    *pFile >> MemoBlock[iB].OutGate.w_Bias.w;
    pFile->scan("ForgetGateBias:");
#ifdef USE_FORGET_GATES
    *pFile >> MemoBlock[iB].FgGate.w_Bias.w;
#else
    *pFile >> NotUsed;
#endif
  }
  return CloseFile();
}

void TLSTM::InitMemoBlocks(unsigned int BegBlock, unsigned int EndBlock) {
  // MemoBlocks (newed in LoadPar() and partial inited).
  // For easier indexing all arrays are newed from zero instead from
  // MemoBlock[iB].BegSrcBlockNb. For the **variables, never
  // used *var[] indices, are not newed (no difference in indexing).
  // BegSrcBlockNb will be zero(or small) in most cases anyway. 
  // But instead of newing until NbMemoBlocks we will use 
  // MemoBlock[iB].EndSrcBlockNb (not any more for full Add).
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    // InGate.
    NewWeight(MemoBlock[iB].InGate.w_Bias,MemoBlock[iB].InGate.w_Bias.w);
    NewWeight(MemoBlock[iB].InGate.w_In, NbIn);
#ifdef CONNECT_GATES_TO_S
    NewWeight(MemoBlock[iB].InGate.w_s, MemoBlock[iB].MemoBlockSize);
#endif
    MemoBlock[iB].InGate.w_Cell = 
      new (TWeight *)[MAX_MEMOBLOCKS];
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
      NewWeight(MemoBlock[iB].InGate.w_Cell[iiB],
		MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
    NewWeight(MemoBlock[iB].InGate.w_InGate, MAX_MEMOBLOCKS);
    NewWeight(MemoBlock[iB].InGate.w_OutGate ,MAX_MEMOBLOCKS);
#endif
    // OutGate.
    NewWeight(MemoBlock[iB].OutGate.w_Bias,MemoBlock[iB].OutGate.w_Bias.w);
    NewWeight(MemoBlock[iB].OutGate.w_In ,NbIn);
#ifdef CONNECT_GATES_TO_S
    NewWeight(MemoBlock[iB].OutGate.w_s, MemoBlock[iB].MemoBlockSize);
#endif
    MemoBlock[iB].OutGate.w_Cell = 
      new (TWeight *)[MAX_MEMOBLOCKS]; //[MemoBlock[iB].EndSrcBlockNb];
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
      NewWeight(MemoBlock[iB].OutGate.w_Cell[iiB], 
		MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
    NewWeight(MemoBlock[iB].OutGate.w_InGate ,MAX_MEMOBLOCKS);
    NewWeight(MemoBlock[iB].OutGate.w_OutGate ,MAX_MEMOBLOCKS);
#endif
#ifdef USE_FORGET_GATES
    // FgGate.
    NewWeight(MemoBlock[iB].FgGate.w_Bias,MemoBlock[iB].FgGate.w_Bias.w);
    NewWeight(MemoBlock[iB].FgGate.w_In, NbIn);
#ifdef CONNECT_GATES_TO_S
    NewWeight(MemoBlock[iB].FgGate.w_s, MemoBlock[iB].MemoBlockSize);
#endif
    MemoBlock[iB].FgGate.w_Cell = 
      new (TWeight *)[MAX_MEMOBLOCKS];
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
      NewWeight(MemoBlock[iB].FgGate.w_Cell[iiB],
		MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
    NewWeight(MemoBlock[iB].FgGate.w_InGate, MAX_MEMOBLOCKS);
    NewWeight(MemoBlock[iB].FgGate.w_OutGate ,MAX_MEMOBLOCKS);
#endif
#endif
    // Cells.
    MemoBlock[iB].Cell = new (TCell)[MemoBlock[iB].MemoBlockSize];
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
#ifdef USE_CELL_BIAS
      NewWeight(MemoBlock[iB].Cell[iC].w_Bias,0); // Random init.
#endif
      NewWeight(MemoBlock[iB].Cell[iC].w_In ,NbIn);
      MemoBlock[iB].Cell[iC].w_Cell = 
	new (TWeight *)[MAX_MEMOBLOCKS]; //[MemoBlock[iB].EndSrcBlockNb];
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
	NewWeight(MemoBlock[iB].Cell[iC].w_Cell[iiB],
		  MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
      NewWeight(MemoBlock[iB].Cell[iC].w_InGate, MAX_MEMOBLOCKS);
      NewWeight(MemoBlock[iB].Cell[iC].w_OutGate, MAX_MEMOBLOCKS);
#endif
    }
  }
  // Init the derivatives s_d.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    // For Cells.
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      MemoBlock[iB].Cell[iC].s_d_In = new double[NbIn];
      MemoBlock[iB].Cell[iC].s_d_Cell = new (double *)[MAX_MEMOBLOCKS];
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
        MemoBlock[iB].Cell[iC].s_d_Cell[iiB] = 
          new double[MemoBlock[iiB].MemoBlockSize];
#ifdef CONNECT_TO_GATES
      MemoBlock[iB].Cell[iC].s_d_InGate = 
        new double[MAX_MEMOBLOCKS];
      MemoBlock[iB].Cell[iC].s_d_OutGate = 
        new double[MAX_MEMOBLOCKS];
#endif
      // For InGate.
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_Bias);
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_In,NbIn);
#ifdef CONNECT_GATES_TO_S
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_s,
		   MemoBlock[iB].MemoBlockSize);
#endif
      MemoBlock[iB].Cell[iC].s_d_InGate_Cell = 
	new (Ts_d_Gate *)[MAX_MEMOBLOCKS];
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
	NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB],
		       MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_InGate,
		     MAX_MEMOBLOCKS);
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_OutGate,
		     MAX_MEMOBLOCKS);
#endif
#ifdef USE_FORGET_GATES
      // For FgGate.
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_Bias);
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_In,NbIn);
#ifdef CONNECT_GATES_TO_S
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_s,
		   MemoBlock[iB].MemoBlockSize);
#endif
      MemoBlock[iB].Cell[iC].s_d_FgGate_Cell = 
	new (Ts_d_Gate *)[MAX_MEMOBLOCKS];
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
	NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB],
		       MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_InGate,
		     MAX_MEMOBLOCKS);
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate,
		     MAX_MEMOBLOCKS);
#endif
#endif
    }
  }
}

void TLSTM::AddMemoBlockToExistingBlocks(unsigned int BegBlock, 
					 unsigned int EndBlock) {
  unsigned int Ml = NbMemoBlocks-1; // last Block shorthand.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    // Always fully connected.
    MemoBlock[iB].EndSrcBlockNb = NbMemoBlocks;
    // InGate.	
    NewWeight(MemoBlock[iB].InGate.w_Cell[Ml], MemoBlock[Ml].MemoBlockSize);
    // OutGate.
    NewWeight(MemoBlock[iB].OutGate.w_Cell[Ml], MemoBlock[Ml].MemoBlockSize);
#ifdef USE_FORGET_GATES
    // FgGate.	
    NewWeight(MemoBlock[iB].FgGate.w_Cell[Ml], MemoBlock[Ml].MemoBlockSize);
#endif
    // Cells.
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
      NewWeight(MemoBlock[iB].Cell[iC].w_Cell[Ml],
		MemoBlock[Ml].MemoBlockSize);
  }
  // Init the derivatives s_d.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    // For Cells.
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      MemoBlock[iB].Cell[iC].s_d_Cell[Ml] = 
        new double[MemoBlock[Ml].MemoBlockSize];
      // For InGate.	
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_Cell[Ml],
		     MemoBlock[Ml].MemoBlockSize);
#ifdef USE_FORGET_GATES
      //  For FgGate.	
      NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[Ml],
		     MemoBlock[Ml].MemoBlockSize);
#endif
    }
  }
  // Set Init Weights different than in NewWeight().
  // Set MemoBlock weights.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    // InGate weights. The bias was already set in LoadPar.
#ifdef CONNECT_TO_GATES
    MemoBlock[iB].InGate.w_InGate[Ml].w = 0;
    MemoBlock[iB].InGate.w_OutGate[Ml].w = 0;
#endif
    for(int iiC=0;iiC<MemoBlock[Ml].MemoBlockSize;iiC++)
      MemoBlock[iB].InGate.w_Cell[Ml][iiC].w = 0;
    // OutGate weights.
#ifdef CONNECT_TO_GATES
    MemoBlock[iB].OutGate.w_InGate[Ml].w = 0;
    MemoBlock[iB].OutGate.w_OutGate[Ml].w = 0;
#endif
    for(int iiC=0;iiC<MemoBlock[Ml].MemoBlockSize;iiC++)
      MemoBlock[iB].OutGate.w_Cell[Ml][iiC].w= 0;
#ifdef USE_FORGET_GATES
    // FgGate weights. The bias was already set in LoadPar.
#ifdef CONNECT_TO_GATES
    MemoBlock[iB].FgGate.w_InGate[Ml].w = 0;
    MemoBlock[iB].FgGate.w_OutGate[Ml].w = 0;
#endif
    for(int iiC=0;iiC<MemoBlock[Ml].MemoBlockSize;iiC++)
      MemoBlock[iB].FgGate.w_Cell[Ml][iiC].w = 0;
#endif
    // Cells weights.
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
#ifdef CONNECT_TO_GATES
      MemoBlock[iB].Cell[iC].w_InGate[Ml].w =0;
      MemoBlock[iB].Cell[iC].w_OutGate[Ml].w =0;
#endif
      for(int iiC=0;iiC<MemoBlock[Ml].MemoBlockSize;iiC++)
	MemoBlock[iB].Cell[iC].w_Cell[Ml][iiC].w = 0;
    }
  }
}
  
void TLSTM::DeleteNet() {
  // Delete MemoBlocks.
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    // InGate.	
    delete MemoBlock[iB].InGate.w_In;
#ifdef CONNECT_GATES_TO_S
    delete MemoBlock[iB].InGate.w_s;
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
      delete MemoBlock[iB].InGate.w_Cell[iiB];
    delete MemoBlock[iB].InGate.w_Cell;
#ifdef CONNECT_TO_GATES
    delete MemoBlock[iB].InGate.w_InGate;
    delete MemoBlock[iB].InGate.w_OutGate;
#endif
    // OutGate.
    delete MemoBlock[iB].OutGate.w_In;
#ifdef CONNECT_GATES_TO_S
    delete MemoBlock[iB].OutGate.w_s;
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
      delete MemoBlock[iB].OutGate.w_Cell[iiB];
    delete MemoBlock[iB].OutGate.w_Cell;
#ifdef CONNECT_TO_GATES
    delete MemoBlock[iB].OutGate.w_InGate;
    delete MemoBlock[iB].OutGate.w_OutGate;
#endif
#ifdef USE_FORGET_GATES
    // FgGate.	
    delete MemoBlock[iB].FgGate.w_In;
#ifdef CONNECT_GATES_TO_S
    delete MemoBlock[iB].FgGate.w_s;
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
      delete MemoBlock[iB].FgGate.w_Cell[iiB];
    delete MemoBlock[iB].FgGate.w_Cell;
#ifdef CONNECT_TO_GATES
    delete MemoBlock[iB].FgGate.w_InGate;
    delete MemoBlock[iB].FgGate.w_OutGate;
#endif
#endif
    // Cells.
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      delete MemoBlock[iB].Cell[iC].w_In;
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
	delete MemoBlock[iB].Cell[iC].w_Cell[iiB];
      delete MemoBlock[iB].Cell[iC].w_Cell;
#ifdef CONNECT_TO_GATES
      delete MemoBlock[iB].Cell[iC].w_InGate;
      delete MemoBlock[iB].Cell[iC].w_OutGate;
#endif
    }
    delete MemoBlock[iB].Cell;
  }
  // Delete the derivatives s_d.
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    // For Cells.
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      delete MemoBlock[iB].Cell[iC].s_d_In;
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
	delete MemoBlock[iB].Cell[iC].s_d_Cell[iiB];
      delete MemoBlock[iB].Cell[iC].s_d_Cell;
#ifdef CONNECT_TO_GATES
      delete MemoBlock[iB].Cell[iC].s_d_InGate;
      delete MemoBlock[iB].Cell[iC].s_d_OutGate;
#endif
      // For InGate.	
      delete MemoBlock[iB].Cell[iC].s_d_InGate_In;
#ifdef CONNECT_GATES_TO_S
      delete MemoBlock[iB].Cell[iC].s_d_InGate_s;
#endif
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
	delete MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB];
      delete MemoBlock[iB].Cell[iC].s_d_InGate_Cell;
#ifdef CONNECT_TO_GATES
      delete MemoBlock[iB].Cell[iC].s_d_InGate_InGate;
      delete MemoBlock[iB].Cell[iC].s_d_InGate_OutGate;
#endif
#ifdef USE_FORGET_GATES
      // FgGate.	
      delete MemoBlock[iB].Cell[iC].s_d_FgGate_In;
#ifdef CONNECT_GATES_TO_S
      delete MemoBlock[iB].Cell[iC].s_d_FgGate_s;
#endif
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) 
	delete MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB];
      delete MemoBlock[iB].Cell[iC].s_d_FgGate_Cell;
#ifdef CONNECT_TO_GATES
      delete MemoBlock[iB].Cell[iC].s_d_FgGate_InGate;
      delete MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate;
#endif
#endif
    }
  }
  delete MemoBlock;
  // Delete other units.
  // Delete Out units.
  for(int iO=0;iO<NbOut;iO++) {
#ifndef NO_IN_OUT_SHORTCUTS
    delete Out[iO].w_In;
#endif
    for(int iB=0;iB<NbMemoBlocks;iB++) 
      delete Out[iO].w_Cell[iB];
    delete Out[iO].w_Cell;
  }
  delete Out;
  // Delete Hidden units.
  for(int iH=0;iH<NbHidden;iH++) {
    delete Hidden[iH].w_In;
    for(int iB=0;iB<NbMemoBlocks;iB++) 
      delete Hidden[iH].w_Cell[iB];
    delete Hidden[iH].w_Cell;
  }
  delete Hidden;
  // Delete PredictNextIn units.
  for(int iP=0;iP<NbPredictNextIn;iP++) {
    delete PredictNextIn[iP].w_In;
    delete PredictNextIn[iP].w_InClass;
    for(int iB=0;iB<NbMemoBlocks;iB++) 
      delete PredictNextIn[iP].w_Cell[iB];
    delete PredictNextIn[iP].w_Cell;
  }
  delete PredictNextIn;
  // Delete NextIn units.
  if(NbPredictNextIn>0) {
    for(int iO=0;iO<NbIn;iO++) delete NextIn[iO].w_PredictNextIn;
    delete NextIn;
  }
  // Delete PredictClass units.
  for(int iP=0;iP<NbPredictClass;iP++) {
    for(int iB=0;iB<NbMemoBlocks;iB++) 
      delete PredictClass[iP].w_Cell[iB];
    delete PredictClass[iP].w_Cell;
  }
  delete PredictClass;
  // Delete PredictClassOut units.
  if(NbPredictClass>0) {
    for(int iO=0;iO<NbOut;iO++) 
      delete PredictClassOut[iO].w_PredictClass;
    delete PredictClassOut;
  }
}

char TLSTM::AddMemoBlock(int BegSrcBlockNb, int EndSrcBlockNb, 
			 int MemoBlockSize) {
  if(NbMemoBlocks>=MAX_MEMOBLOCKS) return 1;
  // Add Block.
  if(FreezeAndGrow) FreezeEndBlock=NbMemoBlocks; // Only the new block.
  NbMemoBlocks++; 
  if(BegSrcBlockNb==-1) BegSrcBlockNb = NbMemoBlocks-1; // Only self.
  if(EndSrcBlockNb==-1) EndSrcBlockNb = NbMemoBlocks;
  MemoBlock[NbMemoBlocks-1].MemoBlockSize = MemoBlockSize;
  MemoBlock[NbMemoBlocks-1].BegSrcBlockNb = BegSrcBlockNb;
  MemoBlock[NbMemoBlocks-1].EndSrcBlockNb = EndSrcBlockNb;
  MemoBlock[NbMemoBlocks-1].InGate.w_Bias.w = -1;
  MemoBlock[NbMemoBlocks-1].OutGate.w_Bias.w = -1;
#ifdef USE_FORGET_GATES
  MemoBlock[NbMemoBlocks-1].FgGate.w_Bias.w = +1;
#endif
  InitMemoBlocks(NbMemoBlocks-1, NbMemoBlocks);
  if(GrowFullyNotCascade) AddMemoBlockToExistingBlocks(0,NbMemoBlocks-1);
  // Add new block the Out src.
  for(int iO=0;iO<NbOut;iO++)
    NewWeight(Out[iO].w_Cell[NbMemoBlocks-1],
	      MemoBlock[NbMemoBlocks-1].MemoBlockSize);
  // Add new block the PredictNextIn src.
  for(int iP=0;iP<NbPredictNextIn;iP++)
    NewWeight(PredictNextIn[iP].w_Cell[NbMemoBlocks-1],
	      MemoBlock[NbMemoBlocks-1].MemoBlockSize);
  // Add new block the PredictClass src.
  for(int iP=0;iP<NbPredictClass;iP++)
    NewWeight(PredictClass[iP].w_Cell[NbMemoBlocks-1],
	      MemoBlock[NbMemoBlocks-1].MemoBlockSize);
  // Refresh statistics.
  SetTopoStatistics();
  return 0;
}

void TLSTM::ResetNet() {
  // Reset MemoBlock varables.
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    // InGate variables.
    MemoBlock[iB].InGate.y=0.0;
    // OutGate variables.
    MemoBlock[iB].OutGate.y=0.0;
    // FgGate variables.
#ifdef USE_FORGET_GATES
    MemoBlock[iB].FgGate.y=0.0; // Even though No connection form the gates.
#endif
    // Cells.
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      // Cell variables.
      MemoBlock[iB].Cell[iC].s=0;
      MemoBlock[iB].Cell[iC].y=0.0;
      // s_d for cells.
#ifdef USE_CELL_BIAS
      MemoBlock[iB].Cell[iC].s_d_Bias=0;
#endif
      for(int iI=0;iI<NbIn;iI++)     
	MemoBlock[iB].Cell[iC].s_d_In[iI]=0;
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
	if(!InternalBlockConect && (iiB==iB)) continue;
#ifdef CONNECT_TO_GATES
	MemoBlock[iB].Cell[iC].s_d_InGate[iiB]=0;
	MemoBlock[iB].Cell[iC].s_d_OutGate[iiB]=0;
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  MemoBlock[iB].Cell[iC].s_d_Cell[iiB][iiC]=0;
      }
      // s_d for InGates.
      MemoBlock[iB].Cell[iC].s_d_InGate_Bias.s_d=0;
      for(int iI=0;iI<NbIn;iI++)     
	MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].s_d=0;
#ifdef CONNECT_GATES_TO_S
      for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++)
	MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].s_d=0;
#endif
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].s_d=0;
	MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].s_d=0;
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].s_d=0;
      }
#ifdef USE_FORGET_GATES
      // s_d for FgGates.
      MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.s_d=0;
      for(int iI=0;iI<NbIn;iI++)     
	MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].s_d=0;
#ifdef CONNECT_GATES_TO_S
      for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++)
	MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].s_d=0;
#endif
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].s_d=0;
	MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].s_d=0;
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].s_d=0;
      }
#endif
    }
  }
  // Reset Out units varables. no.
  // Reset Hidden units varables.
  for(int iH=0;iH<NbHidden;iH++) Hidden[iH].y=0; 
  // Reset PredictNextIn units varables.
  for(int iP=0;iP<NbPredictNextIn;iP++) PredictNextIn[iP].y=0; 
  // Reset NextIn units varables not necessary.
  // Reset PredictClass units varables.
  for(int iP=0;iP<NbPredictClass;iP++)  PredictClass[iP].y=0; 
  // Reset PredictClassOut units varables not necessary.
}

void TLSTM::ForwardPass(struct TPatData &PatData, unsigned int BegBlockDeriv,
			unsigned int EndBlockDeriv) {
  //TopoCheck(); exit(0);
  // One step process: update Hidden, Predict, MemoBlocks than Out.
  // Temporal order within the MemoBlocks: 
  // i) net_InGate, net_FgGate, net_cell (gates see s(t-1)(if connected))
  // ii) y_InGate, y_FgGate -> s(t)
  // ii) net_OutGate (with s(t)(if connected)), y_OutGate -> y_cell.
  // Input to the Hidden units (net).
  for(int iH=0;iH<NbHidden;iH++) {
    //Hidden[iH].net=0; // Reset with bias.
    Hidden[iH].net = Hidden[iH].w_Bias.w;
    for(int iI=0;iI<NbIn;iI++)  // From Input.
      Hidden[iH].net += Hidden[iH].w_In[iI].w * 
	((PatData.SeqList[Seq])[Pat])[iI];
    for(int iB=0;iB<NbMemoBlocks;iB++) // From Blocks.
      for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
	Hidden[iH].net += Hidden[iH].w_Cell[iB][iC].w *
	  MemoBlock[iB].Cell[iC].y;
  }
  //   // Input to the PredictNextIn units (net).
  //   for(int iP=0;iP<NbPredictNextIn;iP++) {
  //     PredictNextIn[iP].net=0;    
  //     for(int iI=0;iI<NbIn;iI++) // From Input.
  //       PredictNextIn[iP].net += PredictNextIn[iP].w_In[iI].w * 
  // 	((PatData.SeqList[Seq])[Pat])[iI];
  //     for(int iI=0;iI<NbOut;iI++) // From constant InClass.
  //       PredictNextIn[iP].net += PredictNextIn[iP].w_InClass[iI].w *
  // 	((PatData.SeqList[Seq])[PatData.NbPat[Seq]-1])[NbIn+iI];
  //     for(int iB=0;iB<NbMemoBlocks;iB++)
  //       for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
  // 	PredictNextIn[iP].net += PredictNextIn[iP].w_Cell[iB][iC].w * 
  // 	  MemoBlock[iB].Cell[iC].y;
  //   }
  //   // Input to the NextIn units (net).
  //   if(NbPredictNextIn>0) {
  //     for(int iO=0;iO<NbIn;iO++) {
  //       NextIn[iO].net=0;
  //       for(int iP=0;iP<NbPredictNextIn;iP++)
  // 	NextIn[iO].net +=NextIn[iO].w_PredictNextIn[iP].w * 
  // 	  PredictNextIn[iP].y;
  //     }
  //   }
  //   // Input to the PredictClass units (net).
  //   for(int iP=0;iP<NbPredictClass;iP++) {
  //     PredictClass[iP].net=0;    
  //     for(int iB=0;iB<NbMemoBlocks;iB++)
  //       for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
  // 	PredictClass[iP].net += PredictClass[iP].w_Cell[iB][iC].w * 
  // 	  MemoBlock[iB].Cell[iC].y;
  //   }
  //   // Input to the PredictClassOut units (net).
  //   if(NbPredictClass>0) {
  //     for(int iO=0;iO<NbOut;iO++) {
  //       PredictClassOut[iO].net=0;
  //       for(int iP=0;iP<NbPredictClass;iP++)
  // 	PredictClassOut[iO].net += PredictClassOut[iO].w_PredictClass[iP].w * 
  // 	  PredictClass[iP].y;
  //     }
  //   }

  // Input to MemoBlocks (net).
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    // net InGate.
    //MemoBlock[iB].InGate.net=0; // Reset with Bias connection.
    MemoBlock[iB].InGate.net = MemoBlock[iB].InGate.w_Bias.w;
    for(int iI=0;iI<NbIn;iI++) // From Input.
      MemoBlock[iB].InGate.net += MemoBlock[iB].InGate.w_In[iI].w *
	((PatData.SeqList[Seq])[Pat])[iI];
#ifdef CONNECT_GATES_TO_S
    for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
      MemoBlock[iB].InGate.net += MemoBlock[iB].InGate.w_s[iiC].w * 
	MemoBlock[iB].Cell[iiC].s;
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	MemoBlock[iB].InGate.net += // From InGate.
	  MemoBlock[iB].InGate.w_InGate[iiB].w * MemoBlock[iiB].InGate.y;
	MemoBlock[iB].InGate.net += // From OutGate.
	  MemoBlock[iB].InGate.w_OutGate[iiB].w * MemoBlock[iiB].OutGate.y;
#endif      
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) 
	MemoBlock[iB].InGate.net += // From Cells.
	  MemoBlock[iB].InGate.w_Cell[iiB][iiC].w *
	  MemoBlock[iiB].Cell[iiC].y;
    }
    // net OutGate.
    //MemoBlock[iB].OutGate.net=0; // Reset with Bias connection.
    MemoBlock[iB].OutGate.net = MemoBlock[iB].OutGate.w_Bias.w;
    for(int iI=0;iI<NbIn;iI++) // From Input.
      MemoBlock[iB].OutGate.net += MemoBlock[iB].OutGate.w_In[iI].w *
	((PatData.SeqList[Seq])[Pat])[iI];
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	MemoBlock[iB].OutGate.net += // From InGate.
	  MemoBlock[iB].OutGate.w_InGate[iiB].w * MemoBlock[iiB].InGate.y;
	MemoBlock[iB].OutGate.net += // From OutGate.
	  MemoBlock[iB].OutGate.w_OutGate[iiB].w * MemoBlock[iiB].OutGate.y;
#endif
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	MemoBlock[iB].OutGate.net += // From Cells.
	  MemoBlock[iB].OutGate.w_Cell[iiB][iiC].w *
	  MemoBlock[iiB].Cell[iiC].y;
    }
#ifdef USE_FORGET_GATES
    // net FgGate.
    //MemoBlock[iB].FgGate.net=0; // Reset with Bias connection.
    MemoBlock[iB].FgGate.net = MemoBlock[iB].FgGate.w_Bias.w;
    for(int iI=0;iI<NbIn;iI++) // From Input.
      MemoBlock[iB].FgGate.net += MemoBlock[iB].FgGate.w_In[iI].w *
	((PatData.SeqList[Seq])[Pat])[iI];
#ifdef CONNECT_GATES_TO_S
    for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
      MemoBlock[iB].FgGate.net += MemoBlock[iB].FgGate.w_s[iiC].w * 
	MemoBlock[iB].Cell[iiC].s;
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	MemoBlock[iB].FgGate.net += // From InGate.
	  MemoBlock[iB].FgGate.w_InGate[iiB].w * MemoBlock[iiB].InGate.y;
	MemoBlock[iB].FgGate.net += // From OutGate.
	  MemoBlock[iB].FgGate.w_OutGate[iiB].w * MemoBlock[iiB].OutGate.y;
#endif
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) 
	MemoBlock[iB].FgGate.net += // From Cells.
	  MemoBlock[iB].FgGate.w_Cell[iiB][iiC].w *
	  MemoBlock[iiB].Cell[iiC].y;
    }
#ifdef RATIONAL_RECURSION
    // Do a range check for the exp() in the recursion.
    if(MemoBlock[iB].FgGate.net>709) MemoBlock[iB].FgGate.net=709;
    if(MemoBlock[iB].FgGate.net<-709) MemoBlock[iB].FgGate.net=-709;
#endif
#endif
    // net Cells.
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      MemoBlock[iB].Cell[iC].net=0;
#ifdef USE_CELL_BIAS
      MemoBlock[iB].Cell[iC].net = MemoBlock[iB].Cell[iC].w_Bias.w;
#endif
      for(int iI=0;iI<NbIn;iI++) // From Input.
	MemoBlock[iB].Cell[iC].net += MemoBlock[iB].Cell[iC].w_In[iI].w *
	  ((PatData.SeqList[Seq])[Pat])[iI];
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
	if(!InternalBlockConect && (iiB==iB)) continue;
#ifdef CONNECT_TO_GATES
	  MemoBlock[iB].Cell[iC].net += // From InGate.
	    MemoBlock[iB].Cell[iC].w_InGate[iiB].w * MemoBlock[iiB].InGate.y;
	  MemoBlock[iB].Cell[iC].net += // From OutGate.
	    MemoBlock[iB].Cell[iC].w_OutGate[iiB].w * 
	    MemoBlock[iiB].OutGate.y;
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  MemoBlock[iB].Cell[iC].net += // From Cells.
	    MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC].w *
	    MemoBlock[iiB].Cell[iiC].y;
      }
    }
  } // End first loop over MemoBlocks.
  // MemoBlocks (g,s,h,y).
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    // y InGate.
    MemoBlock[iB].InGate.y_t1 = MemoBlock[iB].InGate.y;
    log_sig(MemoBlock[iB].InGate.net, MemoBlock[iB].InGate.y);
#ifdef USE_FORGET_GATES
    // y FgGate.
#ifndef RATIONAL_RECURSION
    Yfg(MemoBlock[iB].FgGate.net, MemoBlock[iB].FgGate.y);
#else
    MemoBlock[iB].FgGate.y=0; // Not needed in this case (just for debug).
    MemoBlock[iB].FgGate.exp_NegNet=exp(-MemoBlock[iB].FgGate.net);
#endif
#endif
    // g,s,h,y Cell[iC].
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      MemoBlock[iB].Cell[iC].y_t1 = MemoBlock[iB].Cell[iC].y;
      // G Input squashing.
      G(MemoBlock[iB].Cell[iC].net, MemoBlock[iB].Cell[iC].g);
      //MemoBlock[iB].Cell[iC].g = tanh(MemoBlock[iB].Cell[iC].net);
#ifdef USE_FORGET_GATES
      // For the FgGate partials (and direct conection of the gates to s) 
      // we keep the last s.
      MemoBlock[iB].Cell[iC].s_t1 = MemoBlock[iB].Cell[iC].s;
#ifndef RATIONAL_RECURSION
      MemoBlock[iB].Cell[iC].s *= MemoBlock[iB].FgGate.y;
#else
      MemoBlock[iB].Cell[iC].s = MemoBlock[iB].Cell[iC].s/
	(1+fabs(MemoBlock[iB].Cell[iC].s)*MemoBlock[iB].FgGate.exp_NegNet);
#endif
#endif
      //MemoBlock[iB].Cell[iC].s *= 0.9;
      MemoBlock[iB].Cell[iC].s +=
	MemoBlock[iB].Cell[iC].g * MemoBlock[iB].InGate.y;
      // Do the overflow check for s (709 is max) in H() not here.
      // H Output squashing.
      H(MemoBlock[iB].Cell[iC].s, MemoBlock[iB].Cell[iC].h);
      //MemoBlock[iB].Cell[iC].h = tanh(MemoBlock[iB].Cell[iC].s);
      // Now we finish the OutGate net input, because it should see
      // the actual s before it lets it out.
#ifdef CONNECT_GATES_TO_S
      for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
	MemoBlock[iB].OutGate.net += MemoBlock[iB].OutGate.w_s[iiC].w * 
	  MemoBlock[iB].Cell[iiC].s;
#endif      
      // y OutGate (the outher gates'ys are updated before the s calc).
      MemoBlock[iB].OutGate.y_t1 = MemoBlock[iB].OutGate.y;
      log_sig(MemoBlock[iB].OutGate.net, MemoBlock[iB].OutGate.y);
      // Gating cell output.
      MemoBlock[iB].Cell[iC].y = MemoBlock[iB].Cell[iC].h *
	MemoBlock[iB].OutGate.y;
    }
  }
  // Derivatives s_d.
  for(int iB=BegBlockDeriv;iB<EndBlockDeriv;iB++) {
#ifndef LOG_LIKELIHOOD_GATES
    log_sig_d(MemoBlock[iB].InGate.y,InGate_df); //For the s_d for InGate.
#else
    InGate_df = 1;
#endif
#ifdef USE_FORGET_GATES
#if !defined(LOG_LIKELIHOOD_GATES) && !defined(RATIONAL_RECURSION)
    Yfg_d(MemoBlock[iB].FgGate.y,FgGate_df); //For the s_d for FgGate.
#else
    FgGate_df = 1;
#endif
#endif
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
#if defined(USE_FORGET_GATES) && defined(RATIONAL_RECURSION)
      frac_abs_s_NegNet = fabs(MemoBlock[iB].Cell[iC].s_t1)*
	MemoBlock[iB].FgGate.exp_NegNet;
      // For all.
      frac_1_Sqr = 1/SQR(1+frac_abs_s_NegNet);
      // For forget gates.
      frac_Sqr_s_NegNet = MemoBlock[iB].Cell[iC].s_t1*frac_abs_s_NegNet; 
#endif
#ifndef LOG_LIKELIHOOD_CELLS
      // s_d for Cells.
#if defined(G_SIGMOID) || defined(G_TANH)
      G_d(MemoBlock[iB].Cell[iC].g, g_d);
#elif defined(G_RATIONAL_FUNC) || defined(G_LINEAR)
      G_d(g_d);
#endif
#else
      g_d = 1;
#endif 
      g_d__y_in = g_d * MemoBlock[iB].InGate.y; // For all cells in block.
#ifdef USE_CELL_BIAS
      Update_Cell_s_d(MemoBlock[iB].Cell[iC].s_d_Bias,iB,1);
#endif
      for(int iI=0;iI<NbIn;iI++)
	Update_Cell_s_d(MemoBlock[iB].Cell[iC].s_d_In[iI],iB,
			((PatData.SeqList[Seq])[Pat])[iI]);
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
	if(!InternalBlockConect && (iiB==iB)) continue;
#ifdef CONNECT_TO_GATES
	Update_Cell_s_d(MemoBlock[iB].Cell[iC].s_d_InGate[iiB],iB,
			MemoBlock[iiB].InGate.y_t1);
	Update_Cell_s_d(MemoBlock[iB].Cell[iC].s_d_OutGate[iiB],iB,
			MemoBlock[iiB].OutGate.y_t1);
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	Update_Cell_s_d(MemoBlock[iB].Cell[iC].s_d_Cell[iiB][iiC],iB,
			MemoBlock[iiB].Cell[iiC].y_t1);
      }
      // s_d for InGate.
      y_in_d__g = InGate_df * MemoBlock[iB].Cell[iC].g;
	Update_InGate_s_d(MemoBlock[iB].Cell[iC].s_d_InGate_Bias.s_d,iB,1);
      for(int iI=0;iI<NbIn;iI++)
	Update_InGate_s_d(MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].s_d,iB,
			  ((PatData.SeqList[Seq])[Pat])[iI]);
#ifdef CONNECT_GATES_TO_S
      for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
	Update_InGate_s_d(MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].s_d,iB,
			  MemoBlock[iB].Cell[iiC].s_t1);
#endif
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	Update_InGate_s_d(MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].s_d,
			  iB,MemoBlock[iiB].InGate.y_t1);
	Update_InGate_s_d(MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].s_d,
			  iB,MemoBlock[iiB].OutGate.y_t1);
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  Update_InGate_s_d
	    (MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].s_d,
	     iB,MemoBlock[iiB].Cell[iiC].y_t1);
      }
#ifdef USE_FORGET_GATES
      // s_d for FgGate.
#ifndef RATIONAL_RECURSION
      y__fg_d__s_t1 = FgGate_df * MemoBlock[iB].Cell[iC].s_t1;
#endif
      Update_FgGate_s_d(MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.s_d,iB,1);
      for(int iI=0;iI<NbIn;iI++)
	  Update_FgGate_s_d(MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].s_d,
                            iB,((PatData.SeqList[Seq])[Pat])[iI]);
#ifdef CONNECT_GATES_TO_S
      for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
	Update_FgGate_s_d(MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].s_d,iB,
			  MemoBlock[iB].Cell[iiC].s_t1);
#endif
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	  Update_FgGate_s_d(MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].s_d,
                            iB,MemoBlock[iiB].InGate.y_t1);
	  Update_FgGate_s_d
	    (MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].s_d,
                            iB,MemoBlock[iiB].OutGate.y_t1);
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  Update_FgGate_s_d
	    (MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].s_d,
                            iB,MemoBlock[iiB].Cell[iiC].y_t1);
      }
#endif
    } // End iC loop.
  } // End iB loop.

  // Update the Hidden units (y).
  for(int iH=0;iH<NbHidden;iH++) {
    //Hidden[iH].net=0; // Reset with bias.
    Hidden[iH].net = Hidden[iH].w_Bias.w;
    for(int iI=0;iI<NbIn;iI++)  // From Input.
      Hidden[iH].net += Hidden[iH].w_In[iI].w * 
	((PatData.SeqList[Seq])[Pat])[iI];
    for(int iB=0;iB<NbMemoBlocks;iB++) // From Blocks.
      for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
	Hidden[iH].net += Hidden[iH].w_Cell[iB][iC].w *
	  MemoBlock[iB].Cell[iC].y;
    Hidden[iH].y = tanh(Hidden[iH].net);
  }
  //   // Update the PredictNextIn units (y).
  //   for(int iP=0;iP<NbPredictNextIn;iP++)
  //     log_sig(PredictNextIn[iP].net, PredictNextIn[iP].y);
  //   // Update the NextIn units (y).
  //   if(NbPredictNextIn>0)
  //     for(int iO=0;iO<NbIn;iO++)
  //       log_sig(NextIn[iO].net, NextIn[iO].y);
  //   // Update the PredictClass units (y).
  //   for(int iP=0;iP<NbPredictClass;iP++)
  //     log_sig(PredictClass[iP].net, PredictClass[iP].y);
  //   // Update the PredictClassOut units (y).
  //   if(NbPredictClass>0) 
  //     for(int iO=0;iO<NbOut;iO++)
  //       log_sig(PredictClassOut[iO].net, PredictClassOut[iO].y);

  // Input to and Update the out units (net,y).
  for(int iO=0;iO<NbOut;iO++) {
    //Out[iO].net=0; // Reset with bias.
    Out[iO].net=Out[iO].w_Bias.w;
#ifndef NO_IN_OUT_SHORTCUTS
    for(int iI=0;iI<NbIn;iI++) // From Input. (input shortcuts)
      Out[iO].net += Out[iO].w_In[iI].w*((PatData.SeqList[Seq])[Pat])[iI];
#endif
    for(int iB=0;iB<NbMemoBlocks;iB++)  // From Blocks.
      for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
	Out[iO].net += Out[iO].w_Cell[iB][iC].w * MemoBlock[iB].Cell[iC].y;
    for(int iH=0;iH<NbHidden;iH++) // From Hidden.
      Out[iO].net += Out[iO].w_Hidden[iH].w * Hidden[iH].y;
#ifndef LINEAR_OUTPUT
    log_sig(Out[iO].net, Out[iO].y);
#else
    Out[iO].y = Out[iO].net;
#endif
  }
}

void TLSTM::BackwardPass(unsigned int BegBlock, unsigned int EndBlock) {
  // Calculate the errors (e,delta).
  // e,delta out units.
  for(int iO=0;iO<NbOut;iO++) {
    if(SetStepTarget || (Pat==sTrainData.NbPat[Seq]-1)) { 
      Out[iO].e=((sTrainData.SeqList[Seq])[Pat])[NbIn+iO] - Out[iO].y;
      //if(Pat<sTrainData.NbPat[Seq]-1) // Not the last Pat in Seq.
      //Out[iO].e /= sTrainData.NbPat[Seq];
#if !defined(LOG_LIKELIHOOD_OUTPUT) && !defined(LINEAR_OUTPUT)
      Out[iO].delta = log_sig_d(Out[iO].y,Out[iO].df) * Out[iO].e;
#else
      Out[iO].delta = Out[iO].e; Out[iO].df=1;
#endif
    } else { 
      // Keep track for local learning rate adaptation.
      Out[iO].e=0; Out[iO].delta=0; 
#if !defined(LOG_LIKELIHOOD_OUTPUT) && !defined(LINEAR_OUTPUT)
      log_sig_d(Out[iO].y,Out[iO].df); 
#else
      Out[iO].df=1; 
#endif
    }
  }
  // e,delta Hidden units.
  for(int iH=0;iH<NbHidden;iH++) {
    Hidden[iH].e =0;
    for(int iO=0;iO<NbOut;iO++) // From Out.
      Hidden[iH].e += Out[iO].delta * Out[iO].w_Hidden[iH].w;
    Hidden[iH].delta *= tanh_d(Hidden[iH].y,Hidden[iH].df) * Hidden[iH].e;
  }

  //   // e,delta NextIn units.
  //   if(NbPredictNextIn>0)
  //     for(int iO=0;iO<NbIn;iO++) {
  //       if(Pat==sTrainData.NbPat[Seq]-1) NextIn[iO].e =0; // Last Pat in Seq.
  //       else NextIn[iO].e = // Difference to the next input.
  // 	     (((sTrainData.SeqList[Seq])[Pat+1])[iO] - NextIn[iO].y);
  //       NextIn[iO].delta = log_sig_d(NextIn[iO].y,NextIn[iO].df) 
  // 	* NextIn[iO].e;
  //     }
  //   // e,delta PredictNextIn units.
  //   for(int iP=0;iP<NbPredictNextIn;iP++) {
  //     PredictNextIn[iP].e =0;
  //     for(int iO=0;iO<NbIn;iO++) // Error From NextIn.
  //       PredictNextIn[iP].e += NextIn[iO].delta 
  // 	* NextIn[iO].w_PredictNextIn[iP].w;
  //     PredictNextIn[iP].delta *= 
  //       log_sig_d(PredictNextIn[iP].y,PredictNextIn[iP].df)
  //       * PredictNextIn[iP].e;
  //   }
  //   // e,delta PredictClassOut units.
  //   if(NbPredictClass>0)
  //     for(int iO=0;iO<NbOut;iO++) {
  //       PredictClassOut[iO].e = 
  // 	(((sTrainData.SeqList[Seq])[sTrainData.NbPat[Seq]-1])[NbIn+iO]
  // 	 - PredictClassOut[iO].y);
  //       PredictClassOut[iO].delta = 
  // 	log_sig_d(PredictClassOut[iO].y,PredictClassOut[iO].df)
  // 	* PredictClassOut[iO].e;
  //     }
  //   // e,delta PredictClass units.
  //   for(int iP=0;iP<NbPredictClass;iP++) {
  //     PredictClass[iP].e =0;
  //     for(int iO=0;iO<NbOut;iO++) // Error From PredictClassOut.
  //       PredictClass[iP].e += PredictClassOut[iO].delta * 
  // 	PredictClassOut[iO].w_PredictClass[iP].w;
  //     PredictClass[iP].delta = 
  //       log_sig_d(PredictClass[iP].y,PredictClass[iP].df)
  //       * PredictClass[iP].e;
  //   }
  // e_unscaled cells.
  for(int iB=BegBlock;iB<EndBlock;iB++)
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      MemoBlock[iB].Cell[iC].e_unscaled=0;
      for(int iO=0;iO<NbOut;iO++) // From regular Out units.
	MemoBlock[iB].Cell[iC].e_unscaled += Out[iO].w_Cell[iB][iC].w 
	  * Out[iO].delta;
      //       for(int iP=0;iP<NbPredictNextIn;iP++) // From PredictNextIn units.
      // 	MemoBlock[iB].Cell[iC].e_unscaled +=
      // 	  PredictNextIn[iP].w_Cell[iB][iC].w *
      // 	  PredictNextIn[iP].delta / sTrainData.NbPat[Seq];
      //       for(int iP=0;iP<NbPredictClass;iP++) // From PredictClass units.
      // 	MemoBlock[iB].Cell[iC].e_unscaled += 
      // 	  PredictClass[iP].w_Cell[iB][iC].w *
      // 	  PredictClass[iP].delta / sTrainData.NbPat[Seq];
    }
  // e cells (=e InGates, also for the FgGates).
  for(int iB=BegBlock;iB<EndBlock;iB++)
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
#ifndef LOG_LIKELIHOOD_CELLS
      MemoBlock[iB].Cell[iC].e = MemoBlock[iB].OutGate.y * 
#if defined(H_SIGMOID) || defined(H_TANH)
	H_d(MemoBlock[iB].Cell[iC].h) 
#elif defined(H_RATIONAL_FUNC) || defined(H_LINEAR)
	H_d()
#endif
	* MemoBlock[iB].Cell[iC].e_unscaled;
	// end of H_d (!! defines inside expression !!)
#else
	MemoBlock[iB].Cell[iC].e = MemoBlock[iB].OutGate.y
	* MemoBlock[iB].Cell[iC].e_unscaled;
#endif
  // e,delta OutGates.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    MemoBlock[iB].OutGate.e=0;
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
      MemoBlock[iB].OutGate.e += MemoBlock[iB].Cell[iC].h
	* MemoBlock[iB].Cell[iC].e_unscaled;
#ifndef LOG_LIKELIHOOD_GATES
    MemoBlock[iB].OutGate.delta =
      log_sig_d(MemoBlock[iB].OutGate.y,MemoBlock[iB].OutGate.df)
      * MemoBlock[iB].OutGate.e;
#else
    MemoBlock[iB].OutGate.delta = MemoBlock[iB].OutGate.e;
    MemoBlock[iB].OutGate.df = 1;
#endif

  }
  // Calcualte the local learning rate alpha and dw.
  // Three layes in one time step.
  // dw out units.
  for(int iO=0;iO<NbOut;iO++) {
    AdjustAlphaAndWeights(Out[iO].delta,Out[iO].df,1,Out[iO].w_Bias);
#ifndef NO_IN_OUT_SHORTCUTS
    for(int iI=0;iI<NbIn;iI++) // From Input.
      AdjustAlphaAndWeights(Out[iO].delta,Out[iO].df,
			    ((sTrainData.SeqList[Seq])[Pat])[iI],
			    Out[iO].w_In[iI]);
#endif
    for(int iB=BegBlock;iB<EndBlock;iB++) // From Blocks.
      for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
	AdjustAlphaAndWeights(Out[iO].delta,Out[iO].df,
			      MemoBlock[iB].Cell[iC].y,
			      Out[iO].w_Cell[iB][iC]);
  }
  // dw Hidden units.
  for(int iH=0;iH<NbHidden;iH++) {
    AdjustAlphaAndWeights(Hidden[iH].delta,Hidden[iH].df,1,
			  Hidden[iH].w_Bias);
    for(int iI=0;iI<NbIn;iI++) // From Input.
      AdjustAlphaAndWeights(Hidden[iH].delta,Hidden[iH].df,
			    ((sTrainData.SeqList[Seq])[Pat])[iI],
			    Hidden[iH].w_In[iI]);
    for(int iB=BegBlock;iB<EndBlock;iB++) // From Blocks.
      for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
	AdjustAlphaAndWeights(Hidden[iH].delta,Hidden[iH].df,
			      MemoBlock[iB].Cell[iC].y_t1,
			      Hidden[iH].w_Cell[iB][iC]);
  }

  //   // dw NextIn units. 
  //   if(NbPredictNextIn>0)
  //     for(int iO=0;iO<NbIn;iO++)
  //       for(int iP=0;iP<NbPredictNextIn;iP++) 
  // 	AdjustAlphaAndWeights(NextIn[iO].delta,NextIn[iO].df,
  // 			      PredictNextIn[iP].y,
  // 			      NextIn[iO].w_PredictNextIn[iP]);
  //   // dw PredicNextIn units.
  //   for(int iP=0;iP<NbPredictNextIn;iP++) {
  //     // PredictNextIn[iP].e *= AlphaPredict;
  //     for(int iI=0;iI<NbIn;iI++) // From Input.
  //       AdjustAlphaAndWeights(PredictNextIn[iP].delta,PredictNextIn[iP].df,
  // 			    ((sTrainData.SeqList[Seq])[Pat])[iI],
  // 			    PredictNextIn[iP].w_In[iI]);
  //     for(int iI=0;iI<NbOut;iI++) // From const class.
  //       AdjustAlphaAndWeights(PredictNextIn[iP].delta,PredictNextIn[iP].df,
  // 			    ((sTrainData.SeqList[Seq])
  // 			     [sTrainData.NbPat[Seq]-1])[NbIn+iI],
  // 			    PredictNextIn[iP].w_InClass[iI]);
  //     for(int iB=BegBlock;iB<EndBlock;iB++)
  //       for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
  // 	AdjustAlphaAndWeights(PredictNextIn[iP].delta,PredictNextIn[iP].df,
  // 			      MemoBlock[iB].Cell[iC].y_t1,
  // 			      PredictNextIn[iP].w_Cell[iB][iC]);
  //   } 
  //   // dw PredictClassOut units. 
  //   if(NbPredictClass>0)
  //     for(int iO=0;iO<NbOut;iO++)
  //       for(int iP=0;iP<NbPredictClass;iP++)
  // 	AdjustAlphaAndWeights(PredictClassOut[iO].delta,
  // 			      PredictClassOut[iO].df,
  // 			      PredictClass[iP].y,
  // 			      PredictClassOut[iO].w_PredictClass[iP]);

//   // dw PredicClass units. 
//   for(int iP=0;iP<NbPredictClass;iP++)
//     for(int iB=BegBlock;iB<EndBlock;iB++)
//       for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
// 	AdjustAlphaAndWeights(PredictClass[iP].delta,PredictClass[iP].df,
// 			      MemoBlock[iB].Cell[iC].y_t1,
//			      PredictClass[iP].w_Cell[iB][iC]);
  // dw OutGate.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
			  MemoBlock[iB].OutGate.df,
			  1,
			  MemoBlock[iB].OutGate.w_Bias);
    for(int iI=0;iI<NbIn;iI++) 
      AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
			    MemoBlock[iB].OutGate.df,
			    ((sTrainData.SeqList[Seq])[Pat])[iI],
			    MemoBlock[iB].OutGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
    for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
      AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
			    MemoBlock[iB].OutGate.df,
			    // OutGate see one step ahead,
			    // it used already s(t) not s(t-1).
			    MemoBlock[iB].Cell[iiC].s, 
			    MemoBlock[iB].OutGate.w_s[iiC]);
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
			      MemoBlock[iB].OutGate.df,
			      MemoBlock[iiB].InGate.y_t1,
			      MemoBlock[iB].OutGate.w_InGate[iiB]);
	AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
			      MemoBlock[iB].OutGate.df,
			      MemoBlock[iiB].OutGate.y_t1,
			      MemoBlock[iB].OutGate.w_OutGate[iiB]);
#endif
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
			      MemoBlock[iB].OutGate.df,
			      MemoBlock[iiB].Cell[iiC].y_t1,
			      MemoBlock[iB].OutGate.w_Cell[iiB][iiC]);
    }
  }
  // dw cells.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
#ifdef USE_CELL_BIAS
      RTRLAdjustAlphaAndWeights(MemoBlock[iB].Cell[iC].s_d_Bias,
				MemoBlock[iB].Cell[iC].e,
				MemoBlock[iB].Cell[iC].w_Bias.alpha,
				MemoBlock[iB].Cell[iC].w_Bias.h,
				MemoBlock[iB].Cell[iC].w_Bias);      
#endif
      for(int iI=0;iI<NbIn;iI++)     
	RTRLAdjustAlphaAndWeights(MemoBlock[iB].Cell[iC].s_d_In[iI],
				  MemoBlock[iB].Cell[iC].e,
				  MemoBlock[iB].Cell[iC].w_In[iI].alpha,
				  MemoBlock[iB].Cell[iC].w_In[iI].h,
				  MemoBlock[iB].Cell[iC].w_In[iI]);
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
	if(!InternalBlockConect && (iiB==iB)) continue;
#ifdef CONNECT_TO_GATES
	  RTRLAdjustAlphaAndWeights
	    (MemoBlock[iB].Cell[iC].s_d_InGate[iiB],
	     MemoBlock[iB].Cell[iC].e,
	     MemoBlock[iB].Cell[iC].w_InGate[iiB].alpha,
	     MemoBlock[iB].Cell[iC].w_InGate[iiB].h,
	     MemoBlock[iB].Cell[iC].w_InGate[iiB]);
	  RTRLAdjustAlphaAndWeights
	    (MemoBlock[iB].Cell[iC].s_d_OutGate[iiB],
	     MemoBlock[iB].Cell[iC].e,
	     MemoBlock[iB].Cell[iC].w_OutGate[iiB].alpha,
	     MemoBlock[iB].Cell[iC].w_OutGate[iiB].h,
	     MemoBlock[iB].Cell[iC].w_OutGate[iiB]);
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  RTRLAdjustAlphaAndWeights
	    (MemoBlock[iB].Cell[iC].s_d_Cell[iiB][iiC],
	     MemoBlock[iB].Cell[iC].e,
	     MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC].alpha,
	     MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC].h,
	     MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC]);
      }
    }
  }
  // dw InGate.
  // Here we could do two runs. One to update alpha and 
  // than one for the weights, otherwise we get changing alphas during one
  // weight update, ..ok.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      RTRLAdjustAlphaAndWeights(MemoBlock[iB].Cell[iC].s_d_InGate_Bias.s_d,
				MemoBlock[iB].Cell[iC].e,
				MemoBlock[iB].Cell[iC].s_d_InGate_Bias.alpha,
				MemoBlock[iB].Cell[iC].s_d_InGate_Bias.h,
				MemoBlock[iB].InGate.w_Bias);
      for(int iI=0;iI<NbIn;iI++)     
	RTRLAdjustAlphaAndWeights
	  (MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].s_d,
	   MemoBlock[iB].Cell[iC].e,
	   MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].alpha,
	   MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].h,
	   MemoBlock[iB].InGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
      for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
	RTRLAdjustAlphaAndWeights
	  (MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].s_d,
	   MemoBlock[iB].Cell[iC].e,
	   MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].alpha,
	   MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].h,
	   MemoBlock[iB].InGate.w_s[iiC]);
#endif
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	  RTRLAdjustAlphaAndWeights
	    (MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].s_d,
	     MemoBlock[iB].Cell[iC].e,
	     MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].alpha,
	     MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].h,
	     MemoBlock[iB].InGate.w_InGate[iiB]);
	  RTRLAdjustAlphaAndWeights
	    (MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].s_d,
	     MemoBlock[iB].Cell[iC].e,
	     MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].alpha,
	     MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].h,
	     MemoBlock[iB].InGate.w_OutGate[iiB]);
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  RTRLAdjustAlphaAndWeights
	    (MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].s_d,
	     MemoBlock[iB].Cell[iC].e,
	     MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].alpha,
	     MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].h,
	     MemoBlock[iB].InGate.w_Cell[iiB][iiC]);
      }
    }
  }
#ifdef USE_FORGET_GATES
  // dw FgGate.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      RTRLAdjustAlphaAndWeights(MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.s_d,
				MemoBlock[iB].Cell[iC].e,
				MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.alpha,
				MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.h,
				MemoBlock[iB].FgGate.w_Bias);
      for(int iI=0;iI<NbIn;iI++)     
	RTRLAdjustAlphaAndWeights
	  (MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].s_d,
	   MemoBlock[iB].Cell[iC].e,
	   MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].alpha,
	   MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].h,
	   MemoBlock[iB].FgGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
      for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
	RTRLAdjustAlphaAndWeights
	  (MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].s_d,
	   MemoBlock[iB].Cell[iC].e,
	   MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].alpha,
	   MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].h,
	   MemoBlock[iB].FgGate.w_s[iiC]);
#endif

      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	  RTRLAdjustAlphaAndWeights
	    (MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].s_d,
	     MemoBlock[iB].Cell[iC].e,
	     MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].alpha,
	     MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].h,
	     MemoBlock[iB].FgGate.w_InGate[iiB]);
	  RTRLAdjustAlphaAndWeights
	    (MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].s_d,
	     MemoBlock[iB].Cell[iC].e,
	     MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].alpha,
	     MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].h,
	     MemoBlock[iB].FgGate.w_OutGate[iiB]);
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  RTRLAdjustAlphaAndWeights
	    (MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].s_d,
	     MemoBlock[iB].Cell[iC].e,
	     MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].alpha,
	     MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].h,
	     MemoBlock[iB].FgGate.w_Cell[iiB][iiC]);
      }
    }
  }
#endif
}

#ifdef UPDATE_WEIGHTS_AFTER_SEQ 
void TLSTM::ExecuteWeightChanges(unsigned int BegBlock, 
				 unsigned int EndBlock) {
  // w out units.
  for(int iO=0;iO<NbOut;iO++) {
    ExecuteWeightChange(Out[iO].w_Bias); // From Bias.
#ifndef NO_IN_OUT_SHORTCUTS
    for(int iI=0;iI<NbIn;iI++) // From Input.
      ExecuteWeightChange(Out[iO].w_In[iI]);
#endif
    for(int iB=BegBlock;iB<EndBlock;iB++) // From Blocks.
      for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
	ExecuteWeightChange(Out[iO].w_Cell[iB][iC]);
  }
  // w Hidden units.
  for(int iH=0;iH<NbHidden;iH++) {
    ExecuteWeightChange(Hidden[iH].w_Bias); // From Bias.
    for(int iI=0;iI<NbIn;iI++) // From Input.
      ExecuteWeightChange(Hidden[iH].w_In[iI]);
    for(int iB=BegBlock;iB<EndBlock;iB++) // From Blocks.
      for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
	ExecuteWeightChange(Hidden[iH].w_Cell[iB][iC]);
  }
  // w OutGate.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    ExecuteWeightChange(MemoBlock[iB].OutGate.w_Bias); // From Bias.
    for(int iI=0;iI<NbIn;iI++) 
      ExecuteWeightChange(MemoBlock[iB].OutGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
    for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
      ExecuteWeightChange(MemoBlock[iB].OutGate.w_s[iiC]);
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	ExecuteWeightChange(MemoBlock[iB].OutGate.w_InGate[iiB]);
	ExecuteWeightChange(MemoBlock[iB].OutGate.w_OutGate[iiB]);
#endif
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	ExecuteWeightChange(MemoBlock[iB].OutGate.w_Cell[iiB][iiC]);
    }
  }
  // w cells.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
#ifdef USE_CELL_BIAS
      ExecuteWeightChange(MemoBlock[iB].Cell[iC].w_Bias); // From Bias.
#endif
      for(int iI=0;iI<NbIn;iI++)     
	ExecuteWeightChange(MemoBlock[iB].Cell[iC].w_In[iI]);
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
	if(!InternalBlockConect && (iiB==iB)) continue;
#ifdef CONNECT_TO_GATES
	  ExecuteWeightChange(MemoBlock[iB].Cell[iC].w_InGate[iiB]);
	  ExecuteWeightChange(MemoBlock[iB].Cell[iC].w_OutGate[iiB]);
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  ExecuteWeightChange(MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC]);
      }
    }
  }
  // w InGate.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    ExecuteWeightChange(MemoBlock[iB].InGate.w_Bias); // From Bias.
    for(int iI=0;iI<NbIn;iI++)     
      ExecuteWeightChange(MemoBlock[iB].InGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
    for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
      ExecuteWeightChange(MemoBlock[iB].InGate.w_s[iiC]);
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	ExecuteWeightChange(MemoBlock[iB].InGate.w_InGate[iiB]);
	ExecuteWeightChange(MemoBlock[iB].InGate.w_OutGate[iiB]);
#endif
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	ExecuteWeightChange(MemoBlock[iB].InGate.w_Cell[iiB][iiC]);
    }
  }
#ifdef USE_FORGET_GATES
  // w FgGate.
  for(int iB=BegBlock;iB<EndBlock;iB++) {
    ExecuteWeightChange(MemoBlock[iB].FgGate.w_Bias); // From Bias.
    for(int iI=0;iI<NbIn;iI++)     
      ExecuteWeightChange(MemoBlock[iB].FgGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
    for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
      ExecuteWeightChange(MemoBlock[iB].FgGate.w_s[iiC]);
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	ExecuteWeightChange(MemoBlock[iB].FgGate.w_InGate[iiB]);
	ExecuteWeightChange(MemoBlock[iB].FgGate.w_OutGate[iiB]);
#endif   
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	ExecuteWeightChange(MemoBlock[iB].FgGate.w_Cell[iiB][iiC]);
    }
  }
#endif
}
#endif

void TLSTM::PatStatistics(struct TPatData &PatData) {
  // Next-In PredictNextIn statistics.
  //   MSEPat=0;
  //   if(NbPredictNextIn>0)
  //     for(int iO=0;iO<NbIn;iO++) // Error From NextIn.
  //       MSEPat += NextIn[iO].e * NextIn[iO].e;
  //   if(NbPredictClass>0)
  //     for(int iO=0;iO<NbOut;iO++) // Error From PredictClassOut.
  // 	MSEPat += PredictClassOut[iO].e * PredictClassOut[iO].e;
  //   if(!SetStepTarget && NbPredictOut) MSEPat /= NbPredictOut;
  //   else {
  //     // Find the first two winners for RebberGrammar.
  //     // Find correct class.
  //     while(((PatData.SeqList[Seq])[Pat])[NbIn+FirstClassNb]!=1) 
  //       FirstClassNb++; // At least one 1 should be there.
  //     SecondClassNb = FirstClassNb;
  //     do {
  //       SecondClassNb++; // Second winner.
  //       if(SecondClassNb>=NbOut) {
  // 	SecondClassNb=FirstClassNb; break; } // No second.
  //     } while(((PatData.SeqList[Seq])[Pat])[NbIn+SecondClassNb]!=1);
  //     // Find winner and second.
  //     for(int iO=1;iO<NbOut;iO++) 
  //       if(Out[iO].y>Out[SecondWinnerNb].y)
  // 	if(Out[iO].y>Out[FirstWinnerNb].y) {
  // 	  SecondWinnerNb=FirstWinnerNb; FirstWinnerNb=iO; 
  // 	} else SecondWinnerNb=iO;
  //     if(FirstClassNb==SecondClassNb) {
  //       if(FirstWinnerNb!=FirstClassNb) PatWrong++;
  //     } else if((FirstWinnerNb!=FirstClassNb || SecondWinnerNb!=SecondClassNb)&&
  // 	     (FirstWinnerNb!=SecondClassNb || SecondWinnerNb!=FirstClassNb))
  //       PatWrong++;

  // Patten is wrongly classified when error for at least one out-unit
  // is over MaxMSE.
  bool ThisPatWrong=false;
  for(int iO=0;iO<NbOut;iO++)
    if(fabs(Out[iO].e)>MaxMSE) {ThisPatWrong=true; break;}
  if(ThisPatWrong) PatWrong++;
#ifdef DO_ONLINE_PAT_PRODUCTION
  else PatCorrect++;
#endif
  //#ifndef DO_ONLINE_PAT_PRODUCTION
  // MSE
  MSEPat=0;
  for(int iO=0;iO<NbOut;iO++)MSEPat+=fabs(Out[iO].e);//Out[iO].e * Out[iO].e;
  MSEPat /= NbOut;
  //  } // End else.
  //MSEPat = sqrt(MSEPat); // Or just take the absolute error.
  MSESeq += MSEPat;
  //#endif
// //    //ddd
//            cout <<Epo  << "-" << Seq << "-" << SeqOnline 
//                 << "-" << Pat << " : ";
//            for(int Val=0;Val<PatData.NbVal;Val++)
//              cout << ((PatData.SeqList[Seq])[Pat])[Val] << " ";
//            cout << "->" << PatCorrect << "(" << PatWrong << ")"
//  	       << "(" << Out[0].y<<" -> "<< Out[0].e << ")"
//                 << " MSEPat:" << MSEPat
//                 << " MSESeq:" << MSESeq
//                 << " MSEEpo:" << MSEEpo;
//            cout << "\n"; cout.flush();
 	 ////if(SeqOnline>=2) exit(1);
  //ddd
}

void TLSTM::SeqStatistics(struct TPatData &PatData) {
#ifndef DO_ONLINE_PAT_PRODUCTION
  /*
    // Find correct class.
    ClassNb=0;
    while(((PatData.SeqList[Seq])[PatData.NbPat[Seq]-1])
    [NbIn+ClassNb]!=1) ClassNb++;
    // Find winner.
    WinnerNb=0;
    for(int iO=1;iO<NbOut;iO++) 
    if(Out[iO].y>Out[WinnerNb].y) WinnerNb=iO;
    // Neterror.
    if(WinnerNb!=ClassNb) ClassesWrong++;
    #ifdef DEBUG
    cerr << WinnerNb << " " << ClassNb << endl;
    #endif
    // MSE
    if((NbPredictNextIn>0) || (NbPredictClass>0))
    MSESeq /= PatData.NbPat[Seq]; // Last pattern for free.
    else {
    for(int iO=0;iO<NbOut;iO++) MSESeq += Out[iO].e * Out[iO].e;
    MSESeq /= NbOut; MSESeq = sqrt(MSESeq); 
    }
    */
  ClassesWrong+=PatWrong;//if(PatWrong) ClassesWrong++;
#endif
  if(SetStepTarget) MSESeq /= PatData.NbPat[Seq]; MSEEpo += MSESeq;
}

void TLSTM::EpoStatistics(struct TPatData &PatData) {
#ifdef DO_ONLINE_PAT_PRODUCTION
  if(SetStepTarget) PatCorrect/=PatData.NbSeq;
#ifdef BUFFER_EPO_LOG
  TrainErrEpoLogBuf.AddValue(Epo, PatCorrect);
#else
  WriteLogfileEpo(ERRORLOGFILE,PatCorrect);
#endif
  if(PatCorrect>MaxPatCorrectTrain) MaxPatCorrectTrain=PatCorrect;
#else
#ifdef BUFFER_EPO_LOG
  TrainErrEpoLogBuf.AddValue(Epo, ClassesWrong);
#else
  WriteLogfileEpo(ERRORLOGFILE, ClassesWrong);
#endif
  //ClassesWrongRel = ClassesWrong/PatData.NbSeq;
  // Alpha = AlphaBase + AlphaError * ClassesWrongRel;
  // Record the dLogError to decide when to add an other MemoBlock.
  if(GrowNet) {
    NbLogedErr++;
    unsigned int devisor;
    if(NbLogedErr>LogErrRecSize) devisor = LogErrRecSize; 
    else devisor = NbLogedErr;
    LogErrRecMean = ((devisor-1)*LogErrRecMean+ClassesWrong)/devisor;
    if(NbLogedErr>PartLogErrRecSize) devisor = PartLogErrRecSize; 
    else devisor = NbLogedErr;
    PartLogErrRecMean =((devisor-1)*PartLogErrRecMean+ClassesWrong)/devisor;
    if((NbLogedErr>LogErrRecSize) && (NbLogedErr>LogErrRecSize)) 
      // Decide when to add an other MemoBlock.
      if(PartLogErrRecMean>=LogErrRecMean) {
	// Add memoblock, all same size.
	if(GrowFirstOrder) 
	  AddMemoBlock(-1,-1,MemoBlock[0].MemoBlockSize);
	else // Grow cascate or fully connected.
	  AddMemoBlock(0,-1,MemoBlock[0].MemoBlockSize);
	LogErrRecMean=0; PartLogErrRecMean=0; NbLogedErr=0;
      }
    WriteLogfileEpo(GROWLOGFILE, LogErrRecMean, PartLogErrRecMean);
  }
#endif
  // MSE for the training has to be compleated before a test is run.
#ifdef DO_ONLINE_PAT_PRODUCTION
  MSEEpo /= (SeqOnline+1);
#else
  MSEEpo /= PatData.NbSeq;
#endif
#ifdef BUFFER_EPO_LOG
  TrainMSEEpoLogBuf.AddValue(Epo, MSEEpo);
#else
  WriteLogfileEpo(MSELOGFILE, MSEEpo);
#endif
  // Some more statistics.
#ifdef DO_WEIGHT_STATISTICS
#ifdef DO_ONLINE_PAT_PRODUCTION
  WeightMean/=PatCount*NbWeights;
#else
  WeightMean/=PatData.NbPatTotal*NbWeights;
#endif
  //AlphaMean/=PatData.NbPatTotal*NbWeights;
  //AlphaStd/=PatData.NbPatTotal*NbWeights; 
  //AlphaStd-=SQR(AlphaMean); AlphaStd=sqrt(AlphaStd);
  WriteLogfileEpo(WEIGHTMEAN_LOGFILE, WeightMean);
  //WriteLogfileEpo(ALPHAMEAN_LOGFILE, AlphaMean, AlphaStd);
  WeightMean=0; AlphaMean=0; AlphaStd=0;
#endif
  // Test the net on the Test-set. At the end not to disturb statistics.
  if(TestEach_Epo>0) {
    if((unsigned int)fmod(Epo, TestEach_Epo) == 0) Test(); }
#ifdef DO_ONLINE_PAT_PRODUCTION
    else {
      if(PatCorrect==MaxPatCorrectTrain && // Better in training.
	 LastTestEpo+TestMaxEach_Epo<Epo)  { // Not test too much.
	Test();
	// Write the weight file for good test results.
// 	if(PatCorrectTest>500) { //ddd
// 	  sprintf(cBuf, "w.Test%d.%d-%d",
// 		  Tri,Epo,(unsigned int)PatCorrectTest); 
// 	  WriteWeightFile(cBuf);
// 	}
      }
    }
#endif
}

unsigned int TLSTM::Test() {
  //  cout << "Test...." << endl; //ddd
  if (TestEpochs==0) return 0;
  LastTestEpo=Epo;
  unsigned int ClassesWrongTmp = ClassesWrong; // Save the train value.
  double PatCorrectTmp = PatCorrect; // Save the train value.
  unsigned int SeqOnlineTmp = SeqOnline;
  unsigned int SeqOnlinePatCountTmp = SeqOnlinePatCount;
  double MSEEpoTmp = MSEEpo; 
  ClassesWrong=0; TestMSEEpo=0; PatCorrectTest=0;
  for(TestEpo=0;TestEpo<TestEpochs;TestEpo++) { // Test Epo loop.
    PatCorrect=0; SeqOnline=0; SeqOnlinePatCount=0; MSEEpo=0;
#ifdef DO_ONLINE_PAT_PRODUCTION
    // This Seq will be overwritten, we init the Nb variables here.
    ONLINE_PAT_FUNCNAME(sTestData, true);
#endif
    for(Seq=0;Seq<sTestData.NbSeq;FreezeSeqLoop ? Seq=0 : Seq++) {//Seq loop.
      MSESeq=0; PatWrong=0; 
#ifndef DO_ONLINE_PAT_PRODUCTION
      ResetNet();
#else
      if(!SeqOnline) ResetNet(); //For sequential online
#endif
#ifdef DO_ONLINE_PAT_PRODUCTION
#ifdef NO_RESET_AFTER_ONLINE_SEQ
	if(SeqOnline) ONLINE_PAT_FUNCNAME(sTestData, false);
	else ONLINE_PAT_FUNCNAME(sTestData, true);
#else
	ONLINE_PAT_FUNCNAME(sTestData, true);
#endif
#endif
      for(Pat=0;Pat<sTestData.NbPat[Seq];Pat++) {
	ForwardPass(sTestData,0,0); // Pat loop, don't calc s_d.
	// Calc output units' error instead of doing the Backpass.
	if((Pat==sTestData.NbPat[Seq]-1) || SetStepTarget ||
	   (NbPredictNextIn>0) || (NbPredictClass>0)) {
	  for(int iO=0;iO<NbOut;iO++)
	    Out[iO].e=((sTestData.SeqList[Seq])[Pat])[NbIn+iO] - Out[iO].y;
	  PatStatistics(sTestData);
	  //ddd
//   	  cout <<Epo<<"-"<<TestEpo<<"-"<<Seq<<"-"
// 	       <<SeqOnline<<"-"<<Pat<<":"; 
//   	  for(int Val=0;Val<sTestData.NbVal;Val++) 
//   	    cout << ((sTestData.SeqList[Seq])[Pat])[Val] << " ";
//   	  cout << "->" << PatCorrect << "(" << PatWrong << ")"
// 	       << "PCT:" << PatCorrectTest
//   	       << "MSEPat:" << MSEPat << "TestMSEEpo:" << TestMSEEpo;
//   	  cout << "\n"; //cout.flush();
	  //ddd
  	  //if(/*Epo<MaxEpochs && StopLern /*&& Seq<10*/) { //ddd
	    if(OutputDebug && !TestEpo) { Dump_Inner_States(); //ddd
 	    //sprintf(cBuf,"DumpAll.log.%d-%d-%d-%d-%d",//ddd
	    //    Tri,Epo,Seq,SeqOnline,Pat);//ddd
 	    //DumpAll(cBuf); //ddd
 	  } //ddd
#ifdef DO_ONLINE_PAT_PRODUCTION
	  if(SetStepTarget) {
	    if(!PatWrong) 
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
	      ONLINE_PAT_FUNCNAME(sTestData, false); 
#else
	      ; // Generate seq only before Pat loop.
#endif
	    else break; // PatWrong.
	    //if(ReberGrammarState==-1) ResetNet();
	  }
#endif
	}
//         if(OutputDebug/*Epo<MaxEpochs && StopLern /*&& Seq<10*/) { //ddd
// 	  Dump_Inner_States(); //ddd
// 	  sprintf(cBuf,"DumpAll.log.%d-%d-%d-%d-%d",//ddd
// 		  Tri,Epo,Seq,SeqOnline,Pat);//ddd
// 	  DumpAll(cBuf); //ddd
// 	} //ddd
      } // End Pat loop.
      SeqStatistics(sTestData);
#ifdef DO_ONLINE_PAT_PRODUCTION
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
	if(!SetStepTarget)
#endif
	if(!PatWrong) {
	  SeqOnline++;
	  if(PatCorrect>=MaxOnlineStreamLengthTest) break; 
	} else break;
#endif
    } // End Seq loop.
    TestMSEEpo += MSEEpo/(SeqOnline+1);
    PatCorrectTest+=PatCorrect; 
  } // End TestEpochs loop.
  //cout << "End of Test." << endl; //ddd
  TestMSEEpo /= TestEpochs;
#ifdef DO_ONLINE_PAT_PRODUCTION
  PatCorrectTest/=sTestData.NbSeq*TestEpochs;
  //PatCorrectTest=PatCorrect; // Remember the test value.
#ifdef BUFFER_EPO_LOG
  TestErrEpoLogBuf.AddValue(Epo, PatCorrectTest);
  TestMSEEpoLogBuf.AddValue(Epo, TestMSEEpo);
#else
  WriteLogfileEpo(TEST_ERRORLOGFILE,PatCorrectTest);
  WriteLogfileEpo(TEST_MSELOGFILE,TestMSEEpo);
#endif
#else // We leave out the TestEpochs for the non online case.
  TestClassesWrong = ClassesWrong; // Remember the test value.
#ifdef BUFFER_EPO_LOG
  TestErrEpoLogBuf.AddValue(Epo, TestClassesWrong);
#else
  WriteLogfileEpo(TEST_ERRORLOGFILE, TestClassesWrong);
#endif
#endif
  // Reconstruct the train values.
  ClassesWrong = ClassesWrongTmp; // Reconstruct the train value for stop.
  PatCorrect = PatCorrectTmp; // Reconstruct the train value for stop.
  SeqOnline = SeqOnlineTmp;
  SeqOnlinePatCount = SeqOnlinePatCountTmp;
  MSEEpo = MSEEpoTmp;
#ifdef DO_ONLINE_PAT_PRODUCTION
  return((unsigned int) PatCorrectTest);
#else
  return TestClassesWrong;
#endif
}

char TLSTM::WriteWeightFile(char *FileName) {
  if (OpenFile(FileName, ios::out)) return 1;
  pFile->precision(PRECISION); 
  WriteWeightStream(pFile,(void (*)(class iostream *, double &))&d2s,true);
  return CloseFile();
}

char TLSTM::LoadWeightFile(char *FileName) {
  if (OpenFile(FileName, ios::in)) return 1;
  WriteWeightStream(pFile,(void (*)(class iostream *, double &))&s2d,false);
  return CloseFile();
}

char TLSTM::DumpAll(char *FileName) {
  if (OpenFile(FileName, ios::out)) return 1;
  pFile->precision(PRECISION);
  *pFile << "weights\n\n";
  WriteWeightStream(pFile,(void (*)(class iostream *, double &))&d2s,true);
  WriteNetStream(pFile,sTrainData);
  return CloseFile();
}


char TLSTM::WriteWeightStream(iostream *s,
			      void(*f)(iostream *s, double &d),
			      bool WriteWhiteSpace) {
  double zero=0; // Dummy zeros to fill the weight matrix with not exist. w..
  // Freds Weight Matrix(row=to;column=from): 
  // 1xBias(1 bias cell to all),(s),In,Hidden,
  // MemoBlocks(Ingate,Outgate,FgGate,cells),Out.
  // Zero rows or columns are left out.
  // To Bias. (nothing)
  // To In. (nothing)
  // To s. (nothing)
  // To Hidden. (!!!nothing from the Gates)
  // Without CONNECT_TO_GATES defined  these weights are not printed.
  // Forget gates never have outgoing connections, so never printed.
  for(int iH=0;iH<NbHidden;iH++) {
    f(s,Hidden[iH].w_Bias.w); if(WriteWhiteSpace) *s << " "; // From Bias. 
    for(int iI=0;iI<NbIn;iI++) {  // From In.
      f(s,Hidden[iH].w_In[iI].w); if(WriteWhiteSpace) *s << " "; }
    for(int iB=0;iB<NbMemoBlocks;iB++) {// From Blocks.
      for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // From Cells.
#ifdef CONNECT_TO_GATES
	// Add zeros for the missing connections from the gates.
	f(s,zero); if(WriteWhiteSpace) *s << " "; // From InGate.
	f(s,zero); if(WriteWhiteSpace) *s << " "; // From OutGate.
#ifdef USE_FORGET_GATES
	f(s,zero); if(WriteWhiteSpace) *s << " "; // From FgGate.
#endif
#endif
	f(s,Hidden[iH].w_Cell[iB][iC].w); 
	if(WriteWhiteSpace) *s << " "; }
    }
  } //if(WriteWhiteSpace) *s << "\n";
  // To MemoBlocks.
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    // To InGate.
    f(s,MemoBlock[iB].InGate.w_Bias.w); 
    if(WriteWhiteSpace) *s << " "; // From Bias. 
    for(int iI=0;iI<NbIn;iI++) {
      f(s,MemoBlock[iB].InGate.w_In[iI].w);  
      if(WriteWhiteSpace) *s << " "; } // From In.
#ifdef CONNECT_GATES_TO_S
    for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) { // From s.
      f(s,MemoBlock[iB].InGate.w_s[iiC].w);  
      if(WriteWhiteSpace) *s << " "; } 
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) { // From Blocks.
#ifdef CONNECT_TO_GATES
      f(s,MemoBlock[iB].InGate.w_InGate[iiB].w);  
      if(WriteWhiteSpace) *s << " ";  // From InGate.
      f(s,MemoBlock[iB].InGate.w_OutGate[iiB].w);  
      if(WriteWhiteSpace) *s << " "; // From OutGate.
#endif
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) {
	f(s,MemoBlock[iB].InGate.w_Cell[iiB][iiC].w);  
	if(WriteWhiteSpace) *s << " "; }// From Cells.
    } if(WriteWhiteSpace) *s << "\n";
    // OutGate weights.
    f(s,MemoBlock[iB].OutGate.w_Bias.w);  
    if(WriteWhiteSpace) *s << " "; // From Bias. 
    for(int iI=0;iI<NbIn;iI++) {     
      f(s,MemoBlock[iB].OutGate.w_In[iI].w);  
      if(WriteWhiteSpace) *s << " "; } // From In.
#ifdef CONNECT_GATES_TO_S
    for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) { // From s.
      f(s,MemoBlock[iB].OutGate.w_s[iiC].w);  
      if(WriteWhiteSpace) *s << " "; } 
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) { // From Blocks.
#ifdef CONNECT_TO_GATES
      f(s,MemoBlock[iB].OutGate.w_InGate[iiB].w);  
      if(WriteWhiteSpace) *s << " "; // From InGate.
      f(s,MemoBlock[iB].OutGate.w_OutGate[iiB].w);  
      if(WriteWhiteSpace) *s << " "; // From OutGate.
#endif
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) {
	f(s,MemoBlock[iB].OutGate.w_Cell[iiB][iiC].w);  
	if(WriteWhiteSpace) *s << " "; } // From Cells.
    } if(WriteWhiteSpace) *s << "\n";
#ifdef USE_FORGET_GATES
    // To FgGate.
    f(s,MemoBlock[iB].FgGate.w_Bias.w); 
    if(WriteWhiteSpace) *s << " "; // From Bias. 
    for(int iI=0;iI<NbIn;iI++) {
      f(s,MemoBlock[iB].FgGate.w_In[iI].w);  
      if(WriteWhiteSpace) *s << " "; } // From In.
#ifdef CONNECT_GATES_TO_S
    for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) { // From s.
      f(s,MemoBlock[iB].FgGate.w_s[iiC].w);  
      if(WriteWhiteSpace) *s << " "; } 
#endif
    for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) { // From Blocks.
#ifdef CONNECT_TO_GATES
      f(s,MemoBlock[iB].FgGate.w_InGate[iiB].w);  
      if(WriteWhiteSpace) *s << " ";  // From InGate.
      f(s,MemoBlock[iB].FgGate.w_OutGate[iiB].w);  
      if(WriteWhiteSpace) *s << " "; // From OutGate.
#endif
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) {
	f(s,MemoBlock[iB].FgGate.w_Cell[iiB][iiC].w);  
	if(WriteWhiteSpace) *s << " "; }// From Cells.
    } if(WriteWhiteSpace) *s << "\n";
#endif   
    // To Cells.
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      // Cell weights.
#ifdef USE_CELL_BIAS
      f(s,MemoBlock[iB].Cell[iC].w_Bias.w);  
#else
      f(s,zero); 
#endif
      if(WriteWhiteSpace) *s << " "; // From Bias. 
      for(int iI=0;iI<NbIn;iI++) {     
	f(s,MemoBlock[iB].Cell[iC].w_In[iI].w);  
	if(WriteWhiteSpace) *s << " "; }// From In.
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {// From Blocks.
#ifdef CONNECT_TO_GATES
	f(s,MemoBlock[iB].Cell[iC].w_InGate[iiB].w);  
	if(WriteWhiteSpace) *s << " ";// From InGate.
	f(s,MemoBlock[iB].Cell[iC].w_OutGate[iiB].w);  
	if(WriteWhiteSpace) *s << " ";// From OutGate.
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) {
	  f(s,MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC].w);  
	  if(WriteWhiteSpace) *s << " "; } //From Cells
      } if(WriteWhiteSpace) *s << "\n";
    } //if(WriteWhiteSpace) *s << "\n";
  }
  // Out units' weights.
  for(int iO=0;iO<NbOut;iO++) { 
    //if(WriteWhiteSpace) *s << "\n";
    f(s,Out[iO].w_Bias.w); if(WriteWhiteSpace) *s << " "; // From Bias.
#ifndef NO_IN_OUT_SHORTCUTS
    for(int iI=0;iI<NbIn;iI++) {     
      f(s,Out[iO].w_In[iI].w); if(WriteWhiteSpace) *s << " "; }// From In.
#endif
    for(int iB=0;iB<NbMemoBlocks;iB++) { // From Blocks.
#ifdef CONNECT_TO_GATES
      // Add zeros for the missing connections to the gates.
      f(s,zero); if(WriteWhiteSpace) *s << " "; // From InGate.
      f(s,zero); if(WriteWhiteSpace) *s << " "; // From OutGate.   
#endif
      for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
	f(s,Out[iO].w_Cell[iB][iC].w);  
	if(WriteWhiteSpace) *s << " "; }//From Cells
    }if(WriteWhiteSpace) *s << "\n";
  } 
  if(WriteWhiteSpace) *s << "\n"; // For matrix2X tool.
}

void TLSTM::DisplayNet(struct TPatData &PatData) {
  WriteNetStream(NULL,PatData);
}

void TLSTM::WriteNetStream(iostream *s, struct TPatData &PatData) {
  strstream DisplayBuf;
  // Flags for DisplayBuf
  DisplayBuf.precision(PRECISION);
  cWidth = cPrecision+6;
  if(!s) {
    DisplayBuf.fill(' ');
    DisplayBuf.setf(ios::showpos);
    DisplayBuf.setf(ios::left, ios::adjustfield);
    // Start writing into the buffer.
    DisplayBuf << " Tri:" << Tri << "/" << NbTrials
	       << " Epo:" << Epo << "/" << MaxEpochs
	       << " Seq:" << Seq << "/" << PatData.NbSeq
	       << " Pat:" << Pat << "/" << PatData.NbPat[Seq] << "\n"
	       << " ClassesWrong:" << ClassesWrong
	       << " PatWrong: " << PatWrong
	       << " MSEEpo: " << MSEEpo
	       << " MSESeq: " << setw(cWidth) << MSESeq
	       << " MSEPat: " << setw(cWidth) << MSEPat
	       << "\n";
    if(Pat>PatData.NbPat[Seq]-1) Pat=PatData.NbPat[Seq]-1;
    DisplayBuf << "   "; 
    for(int iO=0;iO<NbOut;iO++) // Targets
      DisplayBuf << ((PatData.SeqList[Seq])[Pat])[NbIn+iO] 
		 << " "; 
    DisplayBuf << "\n";
  }
  DisplayBuf << "\nactivations\n\n"; // activations.
  // No Hidden yet.
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    //DisplayBuf << setw(cWidth); 
    DisplayBuf << MemoBlock[iB].InGate.y << " "; // InGate.y
    //DisplayBuf << setw(cWidth); 
    DisplayBuf << MemoBlock[iB].OutGate.y << " "; // OutGate.y
#ifdef USE_FORGET_GATES
    //DisplayBuf << setw(cWidth); 
    DisplayBuf << MemoBlock[iB].FgGate.y << " "; // FgGate.y
#endif
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cell.y
      //DisplayBuf << setw(cWidth); 
      DisplayBuf << MemoBlock[iB].Cell[iC].y << " "; 
    }
    DisplayBuf << "\n";
  }
  for(int iO=0;iO<NbOut;iO++) DisplayBuf << Out[iO].y << " "; // Out.y
  DisplayBuf << "\n\ncellstates\n\n"; // cellstates.
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cell.s
      //DisplayBuf << setw(cWidth); 
      DisplayBuf << MemoBlock[iB].Cell[iC].s << " "; 
    }
    DisplayBuf << "\n";
  }
  DisplayBuf << "\ng\n\n"; // g
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cell.g
      //DisplayBuf << setw(cWidth); 
      DisplayBuf << MemoBlock[iB].Cell[iC].g << " "; 
    }
    DisplayBuf << "\n";
  }
  DisplayBuf << "\nh\n\n"; // h
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cell.h
      //DisplayBuf << setw(cWidth); 
      DisplayBuf << MemoBlock[iB].Cell[iC].h << " "; 
    }
    DisplayBuf << "\n";
  }
  // Errors.
  DisplayBuf << "\nouterror\n\n";
  //for(int iO=0;iO<NbOut;iO++)
  //  DisplayBuf << Out[iO].e << " "; // Out.e
  for(int iO=0;iO<NbOut;iO++)
    DisplayBuf << Out[iO].delta << " "; // Out.delta
  DisplayBuf << "\n\noutgateerror\n\n";
  for(int iB=0;iB<NbMemoBlocks;iB++) { 
    //DisplayBuf << setw(cWidth); 
    //DisplayBuf <<  MemoBlock[iB].OutGate.e << " "; // OutGate.e
    //DisplayBuf << setw(cWidth);
    DisplayBuf << MemoBlock[iB].OutGate.delta; // OutGate.delta
    DisplayBuf << "\n";
  }
  DisplayBuf << "\n\ncellerror\n\n";
  for(int iB=0;iB<NbMemoBlocks;iB++) { 
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cells.e_unscaled
      //DisplayBuf << setw(cWidth); 
      DisplayBuf << MemoBlock[iB].Cell[iC].e_unscaled << " ";
    }
    DisplayBuf << "\n";
  } 
  DisplayBuf << "\nstateerror\n\n";
  for(int iB=0;iB<NbMemoBlocks;iB++) { 
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cells.e
      //DisplayBuf << setw(cWidth); 
      DisplayBuf << MemoBlock[iB].Cell[iC].e << " "; 
    }
    DisplayBuf << "\n";
  } 
  // s_d for the InGate.
  DisplayBuf << "\ningatepartials\n\n";  
  for(int iB=0;iB<NbMemoBlocks;iB++) { // MemoBlocks
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      DisplayBuf << MemoBlock[iB].Cell[iC].s_d_InGate_Bias.s_d << " ";
      for(int iI=0;iI<NbIn;iI++)     
	DisplayBuf << MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].s_d << " ";
#ifdef CONNECT_GATES_TO_S
    for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
	DisplayBuf << MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].s_d << " ";
#endif
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	DisplayBuf << 
	  MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].s_d << " ";
	DisplayBuf << 
	  MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].s_d << " ";
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  DisplayBuf 
	    << MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].s_d << " ";
      }
      DisplayBuf << "\n";
    }
  }  
#ifdef USE_FORGET_GATES
  // s_d for the FgGate.
  DisplayBuf << "\nFgGatepartials\n\n";  
  for(int iB=0;iB<NbMemoBlocks;iB++) { // MemoBlocks
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      DisplayBuf << MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.s_d << " ";
      for(int iI=0;iI<NbIn;iI++)     
	DisplayBuf << MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].s_d << " ";
#ifdef CONNECT_GATES_TO_S
    for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
	DisplayBuf << MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].s_d << " ";
#endif
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	DisplayBuf << 
	  MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].s_d << " ";
	DisplayBuf << 
	  MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].s_d << " ";
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  DisplayBuf 
	    << MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].s_d << " ";
      }
      DisplayBuf << "\n";
    }
  }  
#endif
  // s_d for the cells.
  DisplayBuf << "\ncellpartials\n\n";  
  for(int iB=0;iB<NbMemoBlocks;iB++) { // MemoBlocks
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      DisplayBuf << MemoBlock[iB].Cell[iC].s_d_Bias << " ";
      for(int iI=0;iI<NbIn;iI++)     
	DisplayBuf << MemoBlock[iB].Cell[iC].s_d_In[iI] << " ";
      for(int iiB=MemoBlock[iB].BegSrcBlockNb;
	  iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
	DisplayBuf << MemoBlock[iB].Cell[iC].s_d_InGate[iiB] << " ";
	DisplayBuf << MemoBlock[iB].Cell[iC].s_d_OutGate[iiB] << " ";
#endif
	for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	  DisplayBuf << MemoBlock[iB].Cell[iC].s_d_Cell[iiB][iiC] << " ";
      }
      DisplayBuf << "\n";
    }
  }
  DisplayBuf << "\n";
  if(!s) {
    for(int iI=0;iI<NbIn;iI++) // In
      DisplayBuf << ((PatData.SeqList[Seq])[Pat])[iI] << " ";  
    DisplayBuf << "\n\n";
  }
  // Print the buffer.
  char* DisplayStr = DisplayBuf.str();
  DisplayBuf << ends;
  if(!s) { // Print to stdout.
    printf("\033[0;H\007"); // position curser (0,0) (row,colm)
    printf("\033[0J\007"); // Clear Below
    cout << DisplayStr; cout.flush(); 
  } else *s << DisplayStr;
  free(DisplayStr);
  //DisplayBuf.clear(); DisplayBuf.seekg(0); DisplayBuf.seekp(0);
}

void TLSTM::DisplayWeights() {
  strstream WeightBuf; WeightBuf.precision(4);
  WriteWeightStream(&WeightBuf,(void (*)(class iostream *, double &))&d2s,
		    true);
  // Print the buffer.
  WeightBuf << ends;
  char *WeightBufStr = WeightBuf.str(); 
  cout << WeightBufStr; free(WeightBufStr);
  cout.flush(); // Finally flush all out (including the buffers).
  //WeightBuf.clear(); WeightBuf.seekg(0); WeightBuf.seekp(0);
}

char TLSTM::Dump_Inner_States() {
#ifdef DO_ONLINE_PAT_PRODUCTION
  // Log seq beginnings and endings.
  if(!SetStepTarget) {
    if(Pat==0) {
      sprintf(cBuf,"SeqBegEnd.log.%d-%d-%d",Tri,Epo,Seq); 
      if(OpenFile(cBuf, ios::app)) return 1;
      // ddd TempOrderEx6
//       *pFile << SeqOnlinePatCount << " " <<  sTestData.NbClass[0] << "\n";
//       *pFile << SeqOnlinePatCount+t1 << " " <<  XNotYt1 << "\n";
//       *pFile << SeqOnlinePatCount+t2 << " " <<  XNotYt2 << "\n";
//       *pFile << SeqOnlinePatCount+t3 << " " <<  XNotYt3 << "\n";
      CloseFile();
    }
  }
#endif
  // Dump the activation of the out units.
  sprintf(cBuf,"%s.%d", OUT_LOGFILE,Tri); 
  if(OpenFile(cBuf, ios::app)) return 1;
  pFile->precision(4);
#ifndef DO_ONLINE_PAT_PRODUCTION
  *pFile << Pat << " ";
#else
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
  if(SetStepTarget) *pFile << Pat << " "; 
  else  *pFile << SeqOnlinePatCount++ << " ";
#else
  if(SetStepTarget) *pFile << PatCorrect << " "; 
#endif
#endif
  for(int iO=0;iO<NbOut;iO++){ *pFile << Out[iO].y << " "; }
  *pFile << "\n";
  CloseFile(); // No return, no check here.
  //
  // Dump the cells' inner states s.
  //sprintf(cBuf,"%s.%d-%d-%d", INNERSTATE_DUMP_LOGFILE,Tri,Epo,Seq); 
  //sprintf(cBuf,"%s.%d-%d", INNERSTATE_DUMP_LOGFILE,Tri,Seq); 
  sprintf(cBuf,"%s.%d", INNERSTATE_DUMP_LOGFILE,Tri); 
  //sprintf(cBuf,"%s.%d-%d", INNERSTATE_DUMP_LOGFILE,Tri,Epo); 
  if(OpenFile(cBuf, ios::app)) return 1;
  pFile->precision(4);
#ifndef DO_ONLINE_PAT_PRODUCTION
  *pFile << Pat << " ";
#else
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
  if(SetStepTarget) *pFile << Pat << " "; 
  else  *pFile << SeqOnlinePatCount++ << " ";
#else
  if(SetStepTarget) *pFile << PatCorrect << " "; 
#endif
#endif
  for(int iB=0;iB<NbMemoBlocks;iB++)
    for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
      *pFile << MemoBlock[iB].Cell[iC].s << " ";
    }
  *pFile << "\n";
#ifndef USE_FORGET_GATES
  return CloseFile();
#else
  bool FirstFileClose = CloseFile();
  //sprintf(cBuf, "%s.%d-%d-%d",FGGATE_Y_LOGFILE,Tri,Seq); 
  //sprintf(cBuf, "%s.%d-%d",FGGATE_Y_LOGFILE,Tri,Epo,Seq); 
  sprintf(cBuf, "%s.%d",FGGATE_Y_LOGFILE,Tri); 
  //sprintf(cBuf, "%s.%d-%d",FGGATE_Y_LOGFILE,Tri,Epo); 
  if(OpenFile(cBuf, ios::app)) return 1;
  pFile->precision(6);
#ifndef DO_ONLINE_PAT_PRODUCTION
  *pFile << Pat << " ";
#else
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
  if(SetStepTarget) *pFile << Pat << " "; 
  else  *pFile << SeqOnlinePatCount << " ";
#else
  if(SetStepTarget) *pFile << PatCorrect << " "; 
#endif
#endif
  for(int iB=0;iB<NbMemoBlocks;iB++)
#ifndef RATIONAL_RECURSION
      *pFile << MemoBlock[iB].FgGate.y << " ";
#else
      *pFile << MemoBlock[iB].FgGate.net << " ";
#endif
  *pFile << "\n";
  return (CloseFile() && FirstFileClose);
#endif
}

char TLSTM::DumpSomeAlphas() {
  if (OpenFile("AlphaDump.log", ios::app)) return 1;
  pFile->precision(4);
  *pFile << Epo << " ";
  // Alpha for the InGate.
  for(int iC=0;iC<MemoBlock[3].MemoBlockSize;iC++) {
    *pFile << MemoBlock[3].Cell[iC].s_d_InGate_Bias.alpha << " ";
    for(int iI=0;iI<NbIn;iI++)     
      *pFile << MemoBlock[3].Cell[iC].s_d_InGate_In[iI].alpha << " ";
    for(int iiB=2;iiB<3;iiB++) {
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
	*pFile 
	  << MemoBlock[3].Cell[iC].s_d_InGate_Cell[iiB][iiC].alpha << " ";
    }
  }
  // OutGate weights alphas.
  *pFile << MemoBlock[3].OutGate.w_Bias.alpha;  
  *pFile << " "; // From Bias. 
  for(int iI=0;iI<NbIn;iI++) {     
    *pFile << MemoBlock[3].OutGate.w_In[iI].alpha;  
    *pFile << " "; } // From In.
  // Cells.
  for(int iC=0;iC<MemoBlock[3].MemoBlockSize;iC++) {
    // Cell weights alphas.
    *pFile << " "; // From Bias. 
    for(int iI=0;iI<NbIn;iI++) {     
      *pFile << MemoBlock[3].Cell[iC].w_In[iI].alpha;  
      *pFile << " "; }// From In.
    for(int iiB=2;iiB<3;iiB++) {// From Blocks.
      *pFile << MemoBlock[3].Cell[iC].w_InGate[iiB].alpha;  
      *pFile << " ";// From InGate.
      *pFile << MemoBlock[3].Cell[iC].w_OutGate[iiB].alpha;  
      *pFile << " ";// From OutGate.
      for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) {
	*pFile << MemoBlock[3].Cell[iC].w_Cell[iiB][iiC].alpha;  
	*pFile << " "; } //From Cells
    }
  }
  *pFile << "\n";
  return CloseFile();
}


char TLSTM::WriteOutLogFile(char *FileName) {
  if (OpenFile(FileName, ios::app)) return 1;
  *pFile << Pat << " ";
  for(int iO=0;iO<NbOut;iO++) *pFile << Out[iO].y << " ";
  *pFile << "\n";
  return CloseFile();
}

void TLSTM::TopoCheck() {
  cout << " "<< NbCells << " " << NbMemUnits << " " << NbWeights << endl;
  for(int iB=0;iB<NbMemoBlocks;iB++) { // MemoBlocks
    cout << iB << " : "
	 << MemoBlock[iB].BegSrcBlockNb << "-" 
	 << MemoBlock[iB].EndSrcBlockNb << endl;
  }
  cout << FreezeEndBlock << endl;
}

void TLSTM::SetTopoStatistics() {
  // Count the number of gates.
  // For the input gates we count NbCells*Weights, 
  // so a weight to the gate is counted NbCell times.
  unsigned int NbGates=NbMemoBlocks*2;
  NbWeights=0; NbCells=0;
  for(int iB=0;iB<NbMemoBlocks;iB++) NbCells+=MemoBlock[iB].MemoBlockSize;
  NbMemUnits = NbCells+NbGates;
  // Bias.
  NbWeights += NbMemUnits+NbOut;
#ifndef USE_CELL_BIAS
  NbWeights -= NbCells;
#endif
#ifdef CONNECT_GATES_TO_S
  for(int iB=0;iB<NbMemoBlocks;iB++) {
    NbWeights += MemoBlock[iB].MemoBlockSize *2;
#ifdef USE_FORGET_GATES
    NbWeights += MemoBlock[iB].MemoBlockSize;
#endif
  }
#endif
  // hidden to out.
  NbWeights += NbCells * NbOut;
  // Start with fully connected memblocks then substract.
  NbWeights += SQR(NbMemUnits);
#ifdef CONNECT_TO_GATES
  NbWeights-=NbGates*NbMemUnits;
#endif
  if(!InternalBlockConect) 
    for(int iB=0;iB<NbMemoBlocks;iB++) 
      NbWeights-=SQR(MemoBlock[iB].MemoBlockSize);
  // Connetion from the input.
  NbWeights+=(NbMemUnits+NbOut)*NbIn;
  // Now count the weight to the InGate * NbCells as extra weights.
  unsigned int NbWeightsToInGate = NbIn+NbCells+1; // 1 for bias.
#ifdef CONNECT_TO_GATES
    NbWeightsToInGate += NbGates;
#endif
  for(int iB=0;iB<NbMemoBlocks;iB++)
    NbWeights+=(MemoBlock[iB].MemoBlockSize-1)*NbWeightsToInGate;
#ifdef USE_FORGET_GATES
  // Same as to the InGates only that they are not yet counted at all.
  for(int iB=0;iB<NbMemoBlocks;iB++)
    NbWeights+=(MemoBlock[iB].MemoBlockSize)*NbWeightsToInGate;
#endif
}

char TLSTM::GetOnlineProblemStatistics() {
  unsigned int NbCyl, CylLength;
  if(OpenFile("ONLINE_PAT_FUNCNAME.log", ios::out)) return 1;
  ONLINE_PAT_FUNCNAME(sTrainData, true);
  NbCyl=0;CylLength=0;
  while(NbCyl<0) {
    ONLINE_PAT_FUNCNAME(sTrainData, false);
    CylLength++;
//     if(ReberGrammarState==0) { //ddd
//       NbCyl++;
//       *pFile << CylLength  << "\n";
//       CylLength=0;
//     }
  }
  return CloseFile();
}

///////////////////main
main(int argc, char **argv)
{
  //Global memory error management called by new
  set_new_handler(MemoryFull); // __new_handler = &MemoryFull;
  // Log stderr (cerr) into a file.
  //dup2(open("cerr.log", O_WRONLY | O_CREAT | O_TRUNC, 0777),STDERR_FILENO);
  int fdCerr = open("cerr.log", O_WRONLY | O_CREAT | O_TRUNC, 0666);
  dup2(fdCerr, STDERR_FILENO);
  // The User Interface (UI) is the only static object.
  TLSTM LSTM;
  return LSTM.Run();
  // clean up
  close(fdCerr);
}